From 29799b6348fc42b80d0f1ee0009463c1ab7afec1 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Tue, 4 Nov 2025 22:39:37 +0100 Subject: [PATCH] migrate codebase to use semver::Version & VersionReq everywhere --- .sqlx/README.md | 12 - ...5f997fccf6a563f654c7943b4593598acccbc.json | 22 ++ ...0cb25441e8e65b273e01ed86d2d3ecebfe84.json} | 6 +- ...cfffb0d87d017a75e82b070ea493023a8d0f6.json | 22 -- ...c810678cd8bbee215532ce26f2c4e76e54a67.json | 17 ++ ...48d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json | 34 --- ...a6b69fd0f7baf3d82b0bc4807852315ec63cf.json | 34 +++ ...911074a74b38084b93dbe62c23d9348b8e609.json | 34 --- ...1d69412ea5af6d0b6562644fe913bac2e1df.json} | 6 +- ...3747632ee3144ebe16cfa11d852f73a4ebbff.json | 17 -- ...9fd12acd6a57e29bf2ecdef5a99ab2186ce0.json} | 6 +- ...f66fc2f46a8b5caa5473278123403ebbd613.json} | 6 +- ...71111accba3638f90abb39e20484d034dfe9.json} | 8 +- ...1c9d032b39b373f2db2713365c2efb03d408.json} | 6 +- ...8d3a4ebef2018ccefb61af8ed2dde44148ccc.json | 34 +++ Cargo.toml | 2 +- clippy.toml | 4 + src/bin/cratesfyi.rs | 8 +- src/build_queue.rs | 223 ++++++++++-------- src/db/add_package.rs | 57 +++-- src/db/delete.rs | 66 +++--- src/db/mod.rs | 2 +- src/db/{types.rs => types/mod.rs} | 2 + src/db/types/version.rs | 100 ++++++++ src/docbuilder/rustwide_builder.rs | 192 ++++++++------- src/registry_api.rs | 14 +- src/storage/mod.rs | 25 +- src/test/fakes.rs | 58 +++-- src/test/mod.rs | 22 +- src/utils/cargo_metadata.rs | 11 +- src/utils/consistency/data.rs | 4 +- src/utils/consistency/db.rs | 18 +- src/utils/consistency/diff.rs | 41 ++-- src/utils/consistency/index.rs | 23 +- src/utils/consistency/mod.rs | 53 ++--- src/utils/html.rs | 16 +- src/utils/version.rs | 0 src/web/build_details.rs | 6 +- src/web/builds.rs | 54 +++-- src/web/crate_details.rs | 98 ++++---- src/web/extractors/rustdoc.rs | 6 +- src/web/mod.rs | 8 +- src/web/releases.rs | 125 +++++----- src/web/rustdoc.rs | 30 +-- src/web/source.rs | 9 +- src/web/status.rs | 28 ++- templates/macros.html | 2 +- 47 files changed, 873 insertions(+), 698 deletions(-) delete mode 100644 .sqlx/README.md create mode 100644 .sqlx/query-45dce653c51bb8a13c9b091b1aa5f997fccf6a563f654c7943b4593598acccbc.json rename .sqlx/{query-7072759134a3abceaa6d3105b1f7986fd2ec3e76ff4df7c8204f2d4eaebba8f1.json => query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json} (70%) delete mode 100644 .sqlx/query-682dd9adacb3841d81633bb918bcfffb0d87d017a75e82b070ea493023a8d0f6.json create mode 100644 .sqlx/query-743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67.json delete mode 100644 .sqlx/query-85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json create mode 100644 .sqlx/query-aebd890b300726ef82aad7137c2a6b69fd0f7baf3d82b0bc4807852315ec63cf.json delete mode 100644 .sqlx/query-aee84ef8b568045ef4928498578911074a74b38084b93dbe62c23d9348b8e609.json rename .sqlx/{query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json => query-af663241b6e7d77402847d57ef9a1d69412ea5af6d0b6562644fe913bac2e1df.json} (51%) delete mode 100644 .sqlx/query-b2266a1d32ffedec1b35d94404d3747632ee3144ebe16cfa11d852f73a4ebbff.json rename .sqlx/{query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json => query-bb6a174647b07fd68bb2e1010bab9fd12acd6a57e29bf2ecdef5a99ab2186ce0.json} (55%) rename .sqlx/{query-3cb4dcb5778c77148aeb8dfc7f942ad45269ad7aab9e9c08fdaa9cb218dbc752.json => query-db8654a53a9914ebfaa15a9bcc0af66fc2f46a8b5caa5473278123403ebbd613.json} (50%) rename .sqlx/{query-4504bbfe4e4b21c82c4f56562d42e904426f9899fbf95f177e61b1f6be226ae4.json => query-e95856a4991ea37e23ee3f28fbc171111accba3638f90abb39e20484d034dfe9.json} (62%) rename .sqlx/{query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json => query-e9e7696f5b32df8981648f3c0ca71c9d032b39b373f2db2713365c2efb03d408.json} (64%) create mode 100644 .sqlx/query-f0bdee589c06dea8b71c5dd09cf8d3a4ebef2018ccefb61af8ed2dde44148ccc.json rename src/db/{types.rs => types/mod.rs} (98%) create mode 100644 src/db/types/version.rs create mode 100644 src/utils/version.rs diff --git a/.sqlx/README.md b/.sqlx/README.md deleted file mode 100644 index acb362aa2..000000000 --- a/.sqlx/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# stored queries for sqlx offline mode - -We want to have both nice local development with sqlx and a local database, and -also possible builds without having a db, like in CI or on the server. - -So we need to store the queries here. - -See also: - -- [how to compile without a database](https://github.com/launchbadge/sqlx/blob/main/FAQ.md#how-do-i-compile-with-the-macros-without-needing-a-database-eg-in-ci) -- [`sqlx::query!` offline mode](https://docs.rs/sqlx/latest/sqlx/macro.query.html#offline-mode) -- [`cargo sqlx prepare`](https://github.com/launchbadge/sqlx/blob/main/sqlx-cli/README.md#enable-building-in-offline-mode-with-query) diff --git a/.sqlx/query-45dce653c51bb8a13c9b091b1aa5f997fccf6a563f654c7943b4593598acccbc.json b/.sqlx/query-45dce653c51bb8a13c9b091b1aa5f997fccf6a563f654c7943b4593598acccbc.json new file mode 100644 index 000000000..ca87c1cb2 --- /dev/null +++ b/.sqlx/query-45dce653c51bb8a13c9b091b1aa5f997fccf6a563f654c7943b4593598acccbc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n version as \"version: Version\"\n FROM releases\n WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version: Version", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "45dce653c51bb8a13c9b091b1aa5f997fccf6a563f654c7943b4593598acccbc" +} diff --git a/.sqlx/query-7072759134a3abceaa6d3105b1f7986fd2ec3e76ff4df7c8204f2d4eaebba8f1.json b/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json similarity index 70% rename from .sqlx/query-7072759134a3abceaa6d3105b1f7986fd2ec3e76ff4df7c8204f2d4eaebba8f1.json rename to .sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json index 80cb41e33..ca0a44f4b 100644 --- a/.sqlx/query-7072759134a3abceaa6d3105b1f7986fd2ec3e76ff4df7c8204f2d4eaebba8f1.json +++ b/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n releases.id as \"id: ReleaseId\",\n releases.version,\n release_build_status.build_status as \"build_status!: BuildStatus\",\n releases.yanked,\n releases.is_library,\n releases.rustdoc_status,\n releases.release_time,\n releases.target_name,\n releases.default_target,\n releases.doc_targets\n FROM releases\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n WHERE\n releases.crate_id = $1", + "query": "SELECT\n releases.id as \"id: ReleaseId\",\n releases.version as \"version: Version\",\n release_build_status.build_status as \"build_status!: BuildStatus\",\n releases.yanked,\n releases.is_library,\n releases.rustdoc_status,\n releases.release_time,\n releases.target_name,\n releases.default_target,\n releases.doc_targets\n FROM releases\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n WHERE\n releases.crate_id = $1", "describe": { "columns": [ { @@ -10,7 +10,7 @@ }, { "ordinal": 1, - "name": "version", + "name": "version: Version", "type_info": "Text" }, { @@ -83,5 +83,5 @@ true ] }, - "hash": "7072759134a3abceaa6d3105b1f7986fd2ec3e76ff4df7c8204f2d4eaebba8f1" + "hash": "4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84" } diff --git a/.sqlx/query-682dd9adacb3841d81633bb918bcfffb0d87d017a75e82b070ea493023a8d0f6.json b/.sqlx/query-682dd9adacb3841d81633bb918bcfffb0d87d017a75e82b070ea493023a8d0f6.json deleted file mode 100644 index 74ec96372..000000000 --- a/.sqlx/query-682dd9adacb3841d81633bb918bcfffb0d87d017a75e82b070ea493023a8d0f6.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT version FROM releases WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "version", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "682dd9adacb3841d81633bb918bcfffb0d87d017a75e82b070ea493023a8d0f6" -} diff --git a/.sqlx/query-743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67.json b/.sqlx/query-743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67.json new file mode 100644 index 000000000..55b209a3f --- /dev/null +++ b/.sqlx/query-743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO queue (name, version, priority, registry)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (name, version) DO UPDATE\n SET priority = EXCLUDED.priority,\n registry = EXCLUDED.registry,\n attempt = 0,\n last_attempt = NULL\n ;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67" +} diff --git a/.sqlx/query-85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json b/.sqlx/query-85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json deleted file mode 100644 index 24c575ac8..000000000 --- a/.sqlx/query-85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n name as \"name!\",\n version as \"version!\",\n yanked\n FROM (\n SELECT\n crates.name,\n releases.version,\n releases.yanked\n FROM crates\n INNER JOIN releases ON releases.crate_id = crates.id\n UNION ALL\n -- crates & releases that are already queued\n -- don't have to be requeued.\n SELECT\n queue.name,\n queue.version,\n NULL as yanked\n FROM queue\n LEFT OUTER JOIN crates ON crates.name = queue.name\n LEFT OUTER JOIN releases ON (\n releases.crate_id = crates.id AND\n releases.version = queue.version\n )\n WHERE queue.attempt < $1 AND (\n crates.id IS NULL OR\n releases.id IS NULL\n )\n ) AS inp\n ORDER BY name, version", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name!", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "version!", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "yanked", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [ - null, - null, - null - ] - }, - "hash": "85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95" -} diff --git a/.sqlx/query-aebd890b300726ef82aad7137c2a6b69fd0f7baf3d82b0bc4807852315ec63cf.json b/.sqlx/query-aebd890b300726ef82aad7137c2a6b69fd0f7baf3d82b0bc4807852315ec63cf.json new file mode 100644 index 000000000..945a85e93 --- /dev/null +++ b/.sqlx/query-aebd890b300726ef82aad7137c2a6b69fd0f7baf3d82b0bc4807852315ec63cf.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT i.* FROM (\n SELECT\n c.name,\n r.version as \"version: Version\",\n (\n SELECT MAX(COALESCE(b.build_finished, b.build_started))\n FROM builds AS b\n WHERE b.rid = r.id\n ) AS last_build_attempt\n FROM crates AS c\n INNER JOIN releases AS r ON c.latest_version_id = r.id\n\n WHERE\n r.rustdoc_status = TRUE\n ) as i\n ORDER BY i.last_build_attempt ASC\n LIMIT $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "version: Version", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "last_build_attempt", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + null + ] + }, + "hash": "aebd890b300726ef82aad7137c2a6b69fd0f7baf3d82b0bc4807852315ec63cf" +} diff --git a/.sqlx/query-aee84ef8b568045ef4928498578911074a74b38084b93dbe62c23d9348b8e609.json b/.sqlx/query-aee84ef8b568045ef4928498578911074a74b38084b93dbe62c23d9348b8e609.json deleted file mode 100644 index 6f3b89e08..000000000 --- a/.sqlx/query-aee84ef8b568045ef4928498578911074a74b38084b93dbe62c23d9348b8e609.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT i.* FROM (\n SELECT\n c.name,\n r.version,\n (\n SELECT MAX(COALESCE(b.build_finished, b.build_started))\n FROM builds AS b\n WHERE b.rid = r.id\n ) AS last_build_attempt\n FROM crates AS c\n INNER JOIN releases AS r ON c.latest_version_id = r.id\n\n WHERE\n r.rustdoc_status = TRUE\n ) as i\n ORDER BY i.last_build_attempt ASC\n LIMIT $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "version", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "last_build_attempt", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false, - false, - null - ] - }, - "hash": "aee84ef8b568045ef4928498578911074a74b38084b93dbe62c23d9348b8e609" -} diff --git a/.sqlx/query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json b/.sqlx/query-af663241b6e7d77402847d57ef9a1d69412ea5af6d0b6562644fe913bac2e1df.json similarity index 51% rename from .sqlx/query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json rename to .sqlx/query-af663241b6e7d77402847d57ef9a1d69412ea5af6d0b6562644fe913bac2e1df.json index d9cbc56c1..ea4fc500a 100644 --- a/.sqlx/query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json +++ b/.sqlx/query-af663241b6e7d77402847d57ef9a1d69412ea5af6d0b6562644fe913bac2e1df.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n crates.name,\n releases.version,\n releases.description,\n release_build_status.last_build_time,\n releases.target_name,\n releases.rustdoc_status,\n repositories.stars as \"stars?\",\n EXISTS (\n SELECT 1\n FROM releases AS all_releases\n WHERE\n all_releases.crate_id = crates.id AND\n all_releases.yanked = false\n ) AS has_unyanked_releases\n\n FROM crates\n INNER JOIN releases ON crates.latest_version_id = releases.id\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n LEFT JOIN repositories ON releases.repository_id = repositories.id\n\n WHERE\n crates.name = ANY($1) AND\n release_build_status.build_status <> 'in_progress'", + "query": "SELECT\n crates.name,\n releases.version as \"version: Version\",\n releases.description,\n release_build_status.last_build_time,\n releases.target_name,\n releases.rustdoc_status,\n repositories.stars as \"stars?\",\n EXISTS (\n SELECT 1\n FROM releases AS all_releases\n WHERE\n all_releases.crate_id = crates.id AND\n all_releases.yanked = false\n ) AS has_unyanked_releases\n\n FROM crates\n INNER JOIN releases ON crates.latest_version_id = releases.id\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n LEFT JOIN repositories ON releases.repository_id = repositories.id\n\n WHERE\n crates.name = ANY($1) AND\n release_build_status.build_status <> 'in_progress'", "describe": { "columns": [ { @@ -10,7 +10,7 @@ }, { "ordinal": 1, - "name": "version", + "name": "version: Version", "type_info": "Text" }, { @@ -60,5 +60,5 @@ null ] }, - "hash": "bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7" + "hash": "af663241b6e7d77402847d57ef9a1d69412ea5af6d0b6562644fe913bac2e1df" } diff --git a/.sqlx/query-b2266a1d32ffedec1b35d94404d3747632ee3144ebe16cfa11d852f73a4ebbff.json b/.sqlx/query-b2266a1d32ffedec1b35d94404d3747632ee3144ebe16cfa11d852f73a4ebbff.json deleted file mode 100644 index d26a5fc1c..000000000 --- a/.sqlx/query-b2266a1d32ffedec1b35d94404d3747632ee3144ebe16cfa11d852f73a4ebbff.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO queue (name, version, priority, registry)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (name, version) DO UPDATE\n SET priority = EXCLUDED.priority,\n registry = EXCLUDED.registry,\n attempt = 0,\n last_attempt = NULL\n ;", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Int4", - "Text" - ] - }, - "nullable": [] - }, - "hash": "b2266a1d32ffedec1b35d94404d3747632ee3144ebe16cfa11d852f73a4ebbff" -} diff --git a/.sqlx/query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json b/.sqlx/query-bb6a174647b07fd68bb2e1010bab9fd12acd6a57e29bf2ecdef5a99ab2186ce0.json similarity index 55% rename from .sqlx/query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json rename to .sqlx/query-bb6a174647b07fd68bb2e1010bab9fd12acd6a57e29bf2ecdef5a99ab2186ce0.json index 00e60e300..189f1c888 100644 --- a/.sqlx/query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json +++ b/.sqlx/query-bb6a174647b07fd68bb2e1010bab9fd12acd6a57e29bf2ecdef5a99ab2186ce0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, name, version, priority, registry, attempt\n FROM queue\n WHERE\n attempt < $1 AND\n (last_attempt IS NULL OR last_attempt < NOW() - make_interval(secs => $2))\n ORDER BY priority ASC, attempt ASC, id ASC\n LIMIT 1\n FOR UPDATE SKIP LOCKED", + "query": "SELECT\n id,\n name,\n version as \"version: Version\",\n priority,\n registry,\n attempt\n FROM queue\n WHERE\n attempt < $1 AND\n (last_attempt IS NULL OR last_attempt < NOW() - make_interval(secs => $2))\n ORDER BY priority ASC, attempt ASC, id ASC\n LIMIT 1\n FOR UPDATE SKIP LOCKED", "describe": { "columns": [ { @@ -15,7 +15,7 @@ }, { "ordinal": 2, - "name": "version", + "name": "version: Version", "type_info": "Text" }, { @@ -49,5 +49,5 @@ false ] }, - "hash": "4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b" + "hash": "bb6a174647b07fd68bb2e1010bab9fd12acd6a57e29bf2ecdef5a99ab2186ce0" } diff --git a/.sqlx/query-3cb4dcb5778c77148aeb8dfc7f942ad45269ad7aab9e9c08fdaa9cb218dbc752.json b/.sqlx/query-db8654a53a9914ebfaa15a9bcc0af66fc2f46a8b5caa5473278123403ebbd613.json similarity index 50% rename from .sqlx/query-3cb4dcb5778c77148aeb8dfc7f942ad45269ad7aab9e9c08fdaa9cb218dbc752.json rename to .sqlx/query-db8654a53a9914ebfaa15a9bcc0af66fc2f46a8b5caa5473278123403ebbd613.json index 0d8bee347..b35981b73 100644 --- a/.sqlx/query-3cb4dcb5778c77148aeb8dfc7f942ad45269ad7aab9e9c08fdaa9cb218dbc752.json +++ b/.sqlx/query-db8654a53a9914ebfaa15a9bcc0af66fc2f46a8b5caa5473278123403ebbd613.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n crates.name,\n releases.version\n FROM builds\n INNER JOIN releases ON releases.id = builds.rid\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE\n builds.build_status = 'in_progress'\n ORDER BY builds.id ASC", + "query": "SELECT\n crates.name,\n releases.version as \"version: Version\"\n FROM builds\n INNER JOIN releases ON releases.id = builds.rid\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE\n builds.build_status = 'in_progress'\n ORDER BY builds.id ASC", "describe": { "columns": [ { @@ -10,7 +10,7 @@ }, { "ordinal": 1, - "name": "version", + "name": "version: Version", "type_info": "Text" } ], @@ -22,5 +22,5 @@ false ] }, - "hash": "3cb4dcb5778c77148aeb8dfc7f942ad45269ad7aab9e9c08fdaa9cb218dbc752" + "hash": "db8654a53a9914ebfaa15a9bcc0af66fc2f46a8b5caa5473278123403ebbd613" } diff --git a/.sqlx/query-4504bbfe4e4b21c82c4f56562d42e904426f9899fbf95f177e61b1f6be226ae4.json b/.sqlx/query-e95856a4991ea37e23ee3f28fbc171111accba3638f90abb39e20484d034dfe9.json similarity index 62% rename from .sqlx/query-4504bbfe4e4b21c82c4f56562d42e904426f9899fbf95f177e61b1f6be226ae4.json rename to .sqlx/query-e95856a4991ea37e23ee3f28fbc171111accba3638f90abb39e20484d034dfe9.json index 3a6e5b742..fdc235e64 100644 --- a/.sqlx/query-4504bbfe4e4b21c82c4f56562d42e904426f9899fbf95f177e61b1f6be226ae4.json +++ b/.sqlx/query-e95856a4991ea37e23ee3f28fbc171111accba3638f90abb39e20484d034dfe9.json @@ -1,12 +1,14 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO queue (name, version, priority, attempt, last_attempt )\n VALUES ('failed_crate', '0.1.1', 0, 99, NOW())", + "query": "\n INSERT INTO queue (name, version, priority, attempt, last_attempt )\n VALUES ('failed_crate', $1, 0, 99, NOW())", "describe": { "columns": [], "parameters": { - "Left": [] + "Left": [ + "Text" + ] }, "nullable": [] }, - "hash": "4504bbfe4e4b21c82c4f56562d42e904426f9899fbf95f177e61b1f6be226ae4" + "hash": "e95856a4991ea37e23ee3f28fbc171111accba3638f90abb39e20484d034dfe9" } diff --git a/.sqlx/query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json b/.sqlx/query-e9e7696f5b32df8981648f3c0ca71c9d032b39b373f2db2713365c2efb03d408.json similarity index 64% rename from .sqlx/query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json rename to .sqlx/query-e9e7696f5b32df8981648f3c0ca71c9d032b39b373f2db2713365c2efb03d408.json index cbd84ed72..baa946d77 100644 --- a/.sqlx/query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json +++ b/.sqlx/query-e9e7696f5b32df8981648f3c0ca71c9d032b39b373f2db2713365c2efb03d408.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, name, version, priority, registry, attempt\n FROM queue\n WHERE attempt < $1\n ORDER BY priority ASC, attempt ASC, id ASC", + "query": "SELECT\n id,\n name,\n version as \"version: Version\",\n priority,\n registry,\n attempt\n FROM queue\n WHERE attempt < $1\n ORDER BY priority ASC, attempt ASC, id ASC", "describe": { "columns": [ { @@ -15,7 +15,7 @@ }, { "ordinal": 2, - "name": "version", + "name": "version: Version", "type_info": "Text" }, { @@ -48,5 +48,5 @@ false ] }, - "hash": "9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132" + "hash": "e9e7696f5b32df8981648f3c0ca71c9d032b39b373f2db2713365c2efb03d408" } diff --git a/.sqlx/query-f0bdee589c06dea8b71c5dd09cf8d3a4ebef2018ccefb61af8ed2dde44148ccc.json b/.sqlx/query-f0bdee589c06dea8b71c5dd09cf8d3a4ebef2018ccefb61af8ed2dde44148ccc.json new file mode 100644 index 000000000..3951f398f --- /dev/null +++ b/.sqlx/query-f0bdee589c06dea8b71c5dd09cf8d3a4ebef2018ccefb61af8ed2dde44148ccc.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n name as \"name!\",\n version as \"version!: Version\",\n yanked\n FROM (\n SELECT\n crates.name,\n releases.version,\n releases.yanked\n FROM crates\n INNER JOIN releases ON releases.crate_id = crates.id\n UNION ALL\n -- crates & releases that are already queued\n -- don't have to be requeued.\n SELECT\n queue.name,\n queue.version,\n NULL as yanked\n FROM queue\n LEFT OUTER JOIN crates ON crates.name = queue.name\n LEFT OUTER JOIN releases ON (\n releases.crate_id = crates.id AND\n releases.version = queue.version\n )\n WHERE queue.attempt < $1 AND (\n crates.id IS NULL OR\n releases.id IS NULL\n )\n ) AS inp\n ORDER BY name, version", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "version!: Version", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "yanked", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "f0bdee589c06dea8b71c5dd09cf8d3a4ebef2018ccefb61af8ed2dde44148ccc" +} diff --git a/Cargo.toml b/Cargo.toml index 9982ae064..0059cd3fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ bzip2 = "0.6.0" getrandom = "0.3.1" itertools = { version = "0.14.0" } hex = "0.4.3" -derive_more = { version = "2.0.0", features = ["display", "deref"] } +derive_more = { version = "2.0.0", features = ["display", "deref", "from", "into"] } sysinfo = { version = "0.37.2", default-features = false, features = ["system"] } derive_builder = "0.20.2" diff --git a/clippy.toml b/clippy.toml index 221f59182..00aa4ed8e 100644 --- a/clippy.toml +++ b/clippy.toml @@ -10,3 +10,7 @@ reason = """ We have our own wrapper struct (`crate::index::Index`) that is async, and should be used instead. """ + +[[disallowed-types]] +path = "semver::Version" +reason = "use our own custom db::types::version::Version so you can use it with sqlx" diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index c6cac3673..6337379f9 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -2,7 +2,7 @@ use anyhow::{Context as _, Result, anyhow}; use clap::{Parser, Subcommand, ValueEnum}; use docs_rs::{ Config, Context, PackageKind, RustwideBuilder, - db::{self, CrateId, Overrides, add_path_into_database}, + db::{self, CrateId, Overrides, add_path_into_database, types::version::Version}, start_background_metrics_webserver, start_web_server, utils::{ ConfigName, get_config, get_crate_pattern_and_priority, list_crate_priorities, @@ -237,7 +237,7 @@ enum QueueSubcommand { crate_name: String, /// Version of crate to build #[arg(name = "CRATE_VERSION")] - crate_version: String, + crate_version: Version, /// Priority of build (new crate builds get priority 0) #[arg( name = "BUILD_PRIORITY", @@ -395,7 +395,7 @@ enum BuildSubcommand { /// Version of crate #[arg(name = "CRATE_VERSION")] - crate_version: Option, + crate_version: Option, /// Build a crate at a specific path #[arg(short = 'l', long = "local", conflicts_with_all(&["CRATE_NAME", "CRATE_VERSION"]))] @@ -793,6 +793,6 @@ enum DeleteSubcommand { /// The version of the crate to delete #[arg(name = "VERSION")] - version: String, + version: Version, }, } diff --git a/src/build_queue.rs b/src/build_queue.rs index 146544c55..d20213712 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -1,17 +1,19 @@ -use crate::Context; -use crate::db::{CrateId, Pool, delete_crate, delete_version, update_latest_version_id}; -use crate::docbuilder::PackageKind; -use crate::error::Result; -use crate::storage::AsyncStorage; -use crate::utils::{ConfigName, get_config, get_crate_priority, report_error, retry, set_config}; -use crate::{BuildPackageSummary, cdn}; -use crate::{Config, Index, InstanceMetrics, RustwideBuilder}; +use crate::{ + BuildPackageSummary, Config, Context, Index, InstanceMetrics, RustwideBuilder, cdn, + db::{ + CrateId, Pool, delete_crate, delete_version, types::version::Version, + update_latest_version_id, + }, + docbuilder::PackageKind, + error::Result, + storage::AsyncStorage, + utils::{ConfigName, get_config, get_crate_priority, report_error, retry, set_config}, +}; use anyhow::Context as _; use fn_error_context::context; use futures_util::{StreamExt, stream::TryStreamExt}; use sqlx::Connection as _; -use std::collections::HashMap; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use tokio::runtime; use tracing::{debug, error, info, instrument}; @@ -26,7 +28,7 @@ pub(crate) struct QueuedCrate { #[serde(skip)] id: i32, pub(crate) name: String, - pub(crate) version: String, + pub(crate) version: Version, pub(crate) priority: i32, pub(crate) registry: Option, pub(crate) attempt: i32, @@ -87,7 +89,7 @@ impl AsyncBuildQueue { pub async fn add_crate( &self, name: &str, - version: &str, + version: &Version, priority: i32, registry: Option<&str>, ) -> Result<()> { @@ -95,15 +97,15 @@ impl AsyncBuildQueue { sqlx::query!( "INSERT INTO queue (name, version, priority, registry) - VALUES ($1, $2, $3, $4) - ON CONFLICT (name, version) DO UPDATE - SET priority = EXCLUDED.priority, - registry = EXCLUDED.registry, - attempt = 0, - last_attempt = NULL - ;", + VALUES ($1, $2, $3, $4) + ON CONFLICT (name, version) DO UPDATE + SET priority = EXCLUDED.priority, + registry = EXCLUDED.registry, + attempt = 0, + last_attempt = NULL + ;", name, - version, + version as _, priority, registry, ) @@ -166,17 +168,23 @@ impl AsyncBuildQueue { Ok(sqlx::query_as!( QueuedCrate, - "SELECT id, name, version, priority, registry, attempt - FROM queue - WHERE attempt < $1 - ORDER BY priority ASC, attempt ASC, id ASC", + r#"SELECT + id, + name, + version as "version: Version", + priority, + registry, + attempt + FROM queue + WHERE attempt < $1 + ORDER BY priority ASC, attempt ASC, id ASC"#, self.max_attempts ) .fetch_all(&mut *conn) .await?) } - pub(crate) async fn has_build_queued(&self, name: &str, version: &str) -> Result { + pub(crate) async fn has_build_queued(&self, name: &str, version: &Version) -> Result { let mut conn = self.db.get_async().await?; Ok(sqlx::query_scalar!( "SELECT id @@ -188,7 +196,7 @@ impl AsyncBuildQueue { ", self.max_attempts, name, - version, + version as _, ) .fetch_optional(&mut *conn) .await? @@ -265,7 +273,10 @@ impl AsyncBuildQueue { &self.storage, &self.config, &release.name, - &release.version, + &release + .version + .parse() + .context("couldn't parse release version as semver")?, ) .await .with_context(|| { @@ -294,7 +305,10 @@ impl AsyncBuildQueue { match self .add_crate( &release.name, - &release.version, + &release + .version + .parse() + .context("couldn't parse release version as semver")?, priority, index.repository_url(), ) @@ -322,14 +336,15 @@ impl AsyncBuildQueue { if let Some(release) = yanked.or(unyanked) { // FIXME: delay yanks of crates that have not yet finished building // https://github.com/rust-lang/docs.rs/issues/1934 - if let Err(err) = self - .set_yanked_inner( - &mut conn, - release.name.as_str(), - release.version.as_str(), - yanked.is_some(), - ) - .await + if let Ok(release_version) = Version::parse(&release.version) + && let Err(err) = self + .set_yanked_inner( + &mut conn, + release.name.as_str(), + &release_version, + yanked.is_some(), + ) + .await { report_error(&err); } @@ -350,7 +365,7 @@ impl AsyncBuildQueue { Ok(crates_added) } - pub async fn set_yanked(&self, name: &str, version: &str, yanked: bool) -> Result<()> { + pub async fn set_yanked(&self, name: &str, version: &Version, yanked: bool) -> Result<()> { let mut conn = self.db.get_async().await?; self.set_yanked_inner(&mut conn, name, version, yanked) .await @@ -361,7 +376,7 @@ impl AsyncBuildQueue { &self, conn: &mut sqlx::PgConnection, name: &str, - version: &str, + version: &Version, yanked: bool, ) -> Result<()> { let activity = if yanked { "yanked" } else { "unyanked" }; @@ -376,7 +391,7 @@ impl AsyncBuildQueue { RETURNING crates.id as "id: CrateId" "#, name, - version, + version as _, yanked, ) .fetch_optional(&mut *conn) @@ -422,7 +437,7 @@ impl BuildQueue { pub fn add_crate( &self, name: &str, - version: &str, + version: &Version, priority: i32, registry: Option<&str>, ) -> Result<()> { @@ -430,7 +445,7 @@ impl BuildQueue { .block_on(self.inner.add_crate(name, version, priority, registry)) } - pub fn set_yanked(&self, name: &str, version: &str, yanked: bool) -> Result<()> { + pub fn set_yanked(&self, name: &str, version: &Version, yanked: bool) -> Result<()> { self.runtime .block_on(self.inner.set_yanked(name, version, yanked)) } @@ -494,14 +509,20 @@ impl BuildQueue { let to_process = match self.runtime.block_on( sqlx::query_as!( QueuedCrate, - "SELECT id, name, version, priority, registry, attempt + r#"SELECT + id, + name, + version as "version: Version", + priority, + registry, + attempt FROM queue WHERE attempt < $1 AND (last_attempt IS NULL OR last_attempt < NOW() - make_interval(secs => $2)) ORDER BY priority ASC, attempt ASC, id ASC LIMIT 1 - FOR UPDATE SKIP LOCKED", + FOR UPDATE SKIP LOCKED"#, self.inner.max_attempts, self.inner.config.delay_between_build_attempts.as_secs_f64(), ) @@ -688,10 +709,10 @@ pub async fn queue_rebuilds( } let mut results = sqlx::query!( - "SELECT i.* FROM ( + r#"SELECT i.* FROM ( SELECT c.name, - r.version, + r.version as "version: Version", ( SELECT MAX(COALESCE(b.build_finished, b.build_started)) FROM builds AS b @@ -704,7 +725,7 @@ pub async fn queue_rebuilds( r.rustdoc_status = TRUE ) as i ORDER BY i.last_build_attempt ASC - LIMIT $1", + LIMIT $1"#, rebuilds_to_queue, ) .fetch(&mut *conn); @@ -728,9 +749,8 @@ pub async fn queue_rebuilds( #[cfg(test)] mod tests { - use crate::test::{FakeBuild, TestEnvironment}; - use super::*; + use crate::test::{FakeBuild, TestEnvironment, V1, V2}; use chrono::Utc; use std::time::Duration; @@ -746,7 +766,7 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .builds(vec![ FakeBuild::default().rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), ]) @@ -762,7 +782,7 @@ mod tests { let queue = build_queue.queued_crates().await?; assert_eq!(queue.len(), 1); assert_eq!(queue[0].name, "foo"); - assert_eq!(queue[0].version, "0.1.0"); + assert_eq!(queue[0].version, V1); assert_eq!(queue[0].priority, REBUILD_PRIORITY); Ok(()) @@ -779,10 +799,10 @@ mod tests { let build_queue = env.async_build_queue(); build_queue - .add_crate("foo1", "0.1.0", REBUILD_PRIORITY, None) + .add_crate("foo1", &V1, REBUILD_PRIORITY, None) .await?; build_queue - .add_crate("foo2", "0.1.0", REBUILD_PRIORITY, None) + .add_crate("foo2", &V1, REBUILD_PRIORITY, None) .await?; let mut conn = env.async_db().async_conn().await; @@ -795,7 +815,7 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .builds(vec![ FakeBuild::default().rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), ]) @@ -821,16 +841,16 @@ mod tests { let build_queue = env.async_build_queue(); build_queue - .add_crate("foo1", "0.1.0", REBUILD_PRIORITY, None) + .add_crate("foo1", &V1, REBUILD_PRIORITY, None) .await?; build_queue - .add_crate("foo2", "0.1.0", REBUILD_PRIORITY, None) + .add_crate("foo2", &V1, REBUILD_PRIORITY, None) .await?; env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .builds(vec![ FakeBuild::default().rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), ]) @@ -854,8 +874,8 @@ mod tests { let queue = env.async_build_queue(); - queue.add_crate("some_crate", "0.1.1", 0, None).await?; - queue.add_crate("some_crate", "0.1.1", 9, None).await?; + queue.add_crate("some_crate", &V1, 0, None).await?; + queue.add_crate("some_crate", &V1, 9, None).await?; let queued_crates = queue.queued_crates().await?; assert_eq!(queued_crates.len(), 1); @@ -876,14 +896,15 @@ mod tests { sqlx::query!( " INSERT INTO queue (name, version, priority, attempt, last_attempt ) - VALUES ('failed_crate', '0.1.1', 0, 99, NOW())", + VALUES ('failed_crate', $1, 0, 99, NOW())", + V1 as _ ) .execute(&mut *conn) .await?; assert_eq!(queue.pending_count().await?, 0); - queue.add_crate("failed_crate", "0.1.1", 9, None).await?; + queue.add_crate("failed_crate", &V1, 9, None).await?; assert_eq!(queue.pending_count().await?, 1); @@ -892,7 +913,7 @@ mod tests { FROM queue WHERE name = $1 AND version = $2", "failed_crate", - "0.1.1", + V1 as _ ) .fetch_one(&mut *conn) .await?; @@ -909,17 +930,17 @@ mod tests { let queue = env.async_build_queue(); - queue.add_crate("dummy", "0.1.1", 0, None).await?; + queue.add_crate("dummy", &V1, 0, None).await?; let mut conn = env.async_db().async_conn().await; - assert!(queue.has_build_queued("dummy", "0.1.1").await.unwrap()); + assert!(queue.has_build_queued("dummy", &V1).await.unwrap()); sqlx::query!("UPDATE queue SET attempt = 6") .execute(&mut *conn) .await .unwrap(); - assert!(!queue.has_build_queued("dummy", "0.1.1").await.unwrap()); + assert!(!queue.has_build_queued("dummy", &V1).await.unwrap()); Ok(()) } @@ -937,7 +958,7 @@ mod tests { let queue = env.build_queue(); - queue.add_crate("krate", "1.0.0", 0, None)?; + queue.add_crate("krate", &V1, 0, None)?; // first let it fail queue.process_next_crate(|krate| { @@ -987,15 +1008,15 @@ mod tests { let queue = env.build_queue(); let test_crates = [ - ("low-priority", "1.0.0", 1000), - ("high-priority-foo", "1.0.0", -1000), - ("medium-priority", "1.0.0", -10), - ("high-priority-bar", "1.0.0", -1000), - ("standard-priority", "1.0.0", 0), - ("high-priority-baz", "1.0.0", -1000), + ("low-priority", 1000), + ("high-priority-foo", -1000), + ("medium-priority", -10), + ("high-priority-bar", -1000), + ("standard-priority", 0), + ("high-priority-baz", -1000), ]; for krate in &test_crates { - queue.add_crate(krate.0, krate.1, krate.2, None)?; + queue.add_crate(krate.0, &V1, krate.1, None)?; } let assert_next = |name| -> Result<()> { @@ -1077,8 +1098,8 @@ mod tests { let queue = env.build_queue(); - queue.add_crate("will_succeed", "1.0.0", -1, None)?; - queue.add_crate("will_fail", "1.0.0", 0, None)?; + queue.add_crate("will_succeed", &V1, -1, None)?; + queue.add_crate("will_fail", &V1, 0, None)?; let fetch_invalidations = || { env.runtime() @@ -1128,9 +1149,9 @@ mod tests { let queue = env.build_queue(); assert_eq!(queue.pending_count()?, 0); - queue.add_crate("foo", "1.0.0", 0, None)?; + queue.add_crate("foo", &V1, 0, None)?; assert_eq!(queue.pending_count()?, 1); - queue.add_crate("bar", "1.0.0", 0, None)?; + queue.add_crate("bar", &V1, 0, None)?; assert_eq!(queue.pending_count()?, 2); queue.process_next_crate(|krate| { @@ -1151,11 +1172,11 @@ mod tests { let queue = env.build_queue(); assert_eq!(queue.prioritized_count()?, 0); - queue.add_crate("foo", "1.0.0", 0, None)?; + queue.add_crate("foo", &V1, 0, None)?; assert_eq!(queue.prioritized_count()?, 1); - queue.add_crate("bar", "1.0.0", -100, None)?; + queue.add_crate("bar", &V1, -100, None)?; assert_eq!(queue.prioritized_count()?, 2); - queue.add_crate("baz", "1.0.0", 100, None)?; + queue.add_crate("baz", &V1, 100, None)?; assert_eq!(queue.prioritized_count()?, 2); queue.process_next_crate(|krate| { @@ -1175,9 +1196,9 @@ mod tests { assert!(queue.pending_count_by_priority()?.is_empty()); - queue.add_crate("one", "1.0.0", 1, None)?; - queue.add_crate("two", "2.0.0", 2, None)?; - queue.add_crate("two_more", "2.0.0", 2, None)?; + queue.add_crate("one", &V1, 1, None)?; + queue.add_crate("two", &V2, 2, None)?; + queue.add_crate("two_more", &V2, 2, None)?; assert_eq!( queue.pending_count_by_priority()?, @@ -1206,9 +1227,9 @@ mod tests { let queue = env.build_queue(); assert_eq!(queue.failed_count()?, 0); - queue.add_crate("foo", "1.0.0", -100, None)?; + queue.add_crate("foo", &V1, -100, None)?; assert_eq!(queue.failed_count()?, 0); - queue.add_crate("bar", "1.0.0", 0, None)?; + queue.add_crate("bar", &V1, 0, None)?; for _ in 0..MAX_ATTEMPTS { assert_eq!(queue.failed_count()?, 0); @@ -1245,9 +1266,9 @@ mod tests { let queue = env.build_queue(); assert_eq!(queue.failed_count()?, 0); - queue.add_crate("foo", "1.0.0", -100, None)?; + queue.add_crate("foo", &V1, -100, None)?; assert_eq!(queue.failed_count()?, 0); - queue.add_crate("bar", "1.0.0", 0, None)?; + queue.add_crate("bar", &V1, 0, None)?; for _ in 0..MAX_ATTEMPTS { assert_eq!(queue.failed_count()?, 0); @@ -1273,25 +1294,21 @@ mod tests { let queue = env.build_queue(); - let test_crates = [ - ("bar", "1.0.0", 0), - ("foo", "1.0.0", -10), - ("baz", "1.0.0", 10), - ]; + let test_crates = [("bar", 0), ("foo", -10), ("baz", 10)]; for krate in &test_crates { - queue.add_crate(krate.0, krate.1, krate.2, None)?; + queue.add_crate(krate.0, &V1, krate.1, None)?; } assert_eq!( vec![ - ("foo", "1.0.0", -10), - ("bar", "1.0.0", 0), - ("baz", "1.0.0", 10), + ("foo".into(), V1, -10), + ("bar".into(), V1, 0), + ("baz".into(), V1, 10), ], queue .queued_crates()? - .iter() - .map(|c| (c.name.as_str(), c.version.as_str(), c.priority)) + .into_iter() + .map(|c| (c.name.clone(), c.version, c.priority)) .collect::>() ); @@ -1362,7 +1379,7 @@ mod tests { let name: String = "krate".repeat(100); - queue.add_crate(&name, "0.0.1", 0, None)?; + queue.add_crate(&name, &V1, 0, None)?; queue.process_next_crate(|krate| { assert_eq!(name, krate.name); @@ -1378,12 +1395,16 @@ mod tests { let queue = env.build_queue(); - let version: String = "version".repeat(100); + let long_version = Version::parse(&format!( + "1.2.3-{}+{}", + "prerelease".repeat(100), + "build".repeat(100) + ))?; - queue.add_crate("krate", &version, 0, None)?; + queue.add_crate("krate", &long_version, 0, None)?; queue.process_next_crate(|krate| { - assert_eq!(version, krate.version); + assert_eq!(long_version, krate.version); Ok(BuildPackageSummary::default()) })?; diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 42aa4bd24..d5ea0b0ec 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -1,5 +1,5 @@ use crate::{ - db::types::{BuildStatus, Feature}, + db::types::{BuildStatus, Feature, version::Version}, docbuilder::DocCoverage, error::Result, registry_api::{CrateData, CrateOwner, ReleaseData}, @@ -10,6 +10,7 @@ use crate::{ use anyhow::{Context, anyhow}; use derive_more::{Deref, Display}; use futures_util::stream::TryStreamExt; +use semver::VersionReq; use serde::{Deserialize, Serialize}; use serde_json::Value; use slug::slugify; @@ -34,7 +35,7 @@ pub struct ReleaseId(pub i32); pub struct BuildId(pub i32); type DepOut = (String, String, String, bool); -type DepIn = (String, String, Option, Option); +type DepIn = (String, VersionReq, Option, Option); /// A crate dependency in our internal representation for releases.dependencies json. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Deref)] @@ -389,7 +390,7 @@ pub(crate) async fn initialize_crate(conn: &mut sqlx::PgConnection, name: &str) pub(crate) async fn initialize_release( conn: &mut sqlx::PgConnection, crate_id: CrateId, - version: &str, + version: &Version, ) -> Result { let release_id = sqlx::query_scalar!( r#"INSERT INTO releases (crate_id, version, archive_storage) @@ -399,7 +400,7 @@ pub(crate) async fn initialize_release( version = EXCLUDED.version RETURNING id as "id: ReleaseId" "#, crate_id.0, - version + version as _, ) .fetch_one(&mut *conn) .await?; @@ -689,7 +690,7 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; - let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; + let release_id = initialize_release(&mut conn, crate_id, &V0_1).await?; let build_id = initialize_build(&mut conn, release_id).await?; update_build_with_error(&mut conn, build_id, Some("error message")).await?; @@ -723,7 +724,7 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; - let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; + let release_id = initialize_release(&mut conn, crate_id, &V0_1).await?; let build_id = initialize_build(&mut conn, release_id).await?; finish_build( @@ -772,7 +773,7 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; - let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; + let release_id = initialize_release(&mut conn, crate_id, &V0_1).await?; let build_id = initialize_build(&mut conn, release_id).await?; finish_build( @@ -817,7 +818,7 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; - let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; + let release_id = initialize_release(&mut conn, crate_id, &V0_1).await?; let build_id = initialize_build(&mut conn, release_id).await?; finish_build( @@ -864,7 +865,7 @@ mod test { .fake_release() .await .name("dummy") - .version("0.13.0") + .version(V0_1) .keywords(vec!["kw 1".into(), "kw 2".into()]) .create() .await?; @@ -907,7 +908,7 @@ mod test { env.fake_release() .await .name("dummy") - .version("0.13.0") + .version(V0_1) .keywords(vec!["kw 3".into(), "kw 4".into()]) .create() .await?; @@ -916,7 +917,7 @@ mod test { env.fake_release() .await .name("dummy") - .version("0.13.0") + .version(V0_1) .keywords(vec!["kw 3".into(), "kw 4".into()]) .create() .await?; @@ -931,7 +932,7 @@ mod test { env.fake_release() .await .name("dummy") - .version("0.13.0") + .version(V1) .keywords(vec!["kw 3".into(), "kw 4".into()]) .create() .await?; @@ -940,7 +941,7 @@ mod test { .fake_release() .await .name("dummy") - .version("0.13.0") + .version(V1) .keywords(vec!["kw 1".into(), "kw 2".into()]) .create() .await?; @@ -1261,22 +1262,21 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let name = "krate"; - let version = "0.1.0"; let crate_id = initialize_crate(&mut conn, name).await?; - let release_id = initialize_release(&mut conn, crate_id, version).await?; + let release_id = initialize_release(&mut conn, crate_id, &V1).await?; let id = sqlx::query_scalar!( r#"SELECT id as "id: ReleaseId" FROM releases WHERE crate_id = $1 and version = $2"#, crate_id.0, - version + V1 as _, ) .fetch_one(&mut *conn) .await?; assert_eq!(release_id, id); - let same_release_id = initialize_release(&mut conn, crate_id, version).await?; + let same_release_id = initialize_release(&mut conn, crate_id, &V1).await?; assert_eq!(release_id, same_release_id); Ok(()) @@ -1288,9 +1288,8 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let name = "krate"; - let version = "0.1.0"; let crate_id = initialize_crate(&mut conn, name).await?; - let release_id = initialize_release(&mut conn, crate_id, version).await?; + let release_id = initialize_release(&mut conn, crate_id, &V1).await?; let build_id = initialize_build(&mut conn, release_id).await?; @@ -1334,13 +1333,23 @@ mod test { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; - let version: String = "version".repeat(100); + let version = Version::parse(&format!( + "1.2.3-{}+{}", + "prerelease".repeat(100), + "build".repeat(100) + ))?; let release_id = initialize_release(&mut conn, crate_id, &version).await?; - let db_version = - sqlx::query_scalar!("SELECT version FROM releases WHERE id = $1", release_id.0) - .fetch_one(&mut *conn) - .await?; + let db_version = sqlx::query_scalar!( + r#" + SELECT + version as "version: Version" + FROM releases + WHERE id = $1"#, + release_id.0 + ) + .fetch_one(&mut *conn) + .await?; assert_eq!(db_version, version); diff --git a/src/db/delete.rs b/src/db/delete.rs index 11ba12019..94c896e2d 100644 --- a/src/db/delete.rs +++ b/src/db/delete.rs @@ -1,5 +1,6 @@ use crate::{ Config, + db::types::version::Version, error::Result, storage::{AsyncStorage, rustdoc_archive_path, source_archive_path}, }; @@ -65,7 +66,7 @@ pub async fn delete_version( storage: &AsyncStorage, config: &Config, name: &str, - version: &str, + version: &Version, ) -> Result<()> { let is_library = delete_version_from_database(conn, name, version).await?; let paths = if is_library { @@ -131,7 +132,7 @@ const METADATA: &[(&str, &str)] = &[ async fn delete_version_from_database( conn: &mut sqlx::PgConnection, name: &str, - version: &str, + version: &Version, ) -> Result { let crate_id = get_id(conn, name).await?; let mut transaction = conn.begin().await?; @@ -143,7 +144,7 @@ async fn delete_version_from_database( let is_library: bool = sqlx::query_scalar!( "DELETE FROM releases WHERE crate_id = $1 AND version = $2 RETURNING is_library", crate_id.0, - version, + version as _, ) .fetch_one(&mut *transaction) .await? @@ -223,7 +224,7 @@ mod tests { use crate::db::ReleaseId; use crate::registry_api::{CrateOwner, OwnerKind}; use crate::storage::{CompressionAlgorithm, rustdoc_json_path}; - use crate::test::{async_wrapper, fake_release_that_failed_before_build}; + use crate::test::{V1, V2, async_wrapper, fake_release_that_failed_before_build}; use test_case::test_case; async fn crate_exists(conn: &mut sqlx::PgConnection, name: &str) -> Result { @@ -246,7 +247,7 @@ mod tests { env.fake_release() .await .name("Some_Package") - .version("1.0.0") + .version(V1) .create() .await?; @@ -268,7 +269,7 @@ mod tests { .fake_release() .await .name("package-1") - .version("1.0.0") + .version(V1) .archive_storage(archive_storage) .create() .await?; @@ -276,7 +277,7 @@ mod tests { .fake_release() .await .name("package-1") - .version("2.0.0") + .version(V2) .archive_storage(archive_storage) .create() .await?; @@ -284,6 +285,7 @@ mod tests { .fake_release() .await .name("package-2") + .version(V1) .archive_storage(archive_storage) .create() .await?; @@ -293,11 +295,7 @@ mod tests { assert!(release_exists(&mut conn, pkg1_v1_id).await?); assert!(release_exists(&mut conn, pkg1_v2_id).await?); assert!(release_exists(&mut conn, pkg2_id).await?); - for (pkg, version) in &[ - ("package-1", "1.0.0"), - ("package-1", "2.0.0"), - ("package-2", "1.0.0"), - ] { + for (pkg, version) in &[("package-1", V1), ("package-1", V2), ("package-2", V1)] { assert!( env.async_storage() .rustdoc_file_exists( @@ -324,7 +322,7 @@ mod tests { env.async_storage() .rustdoc_file_exists( "package-2", - "1.0.0", + &V1, None, "package-2/index.html", archive_storage @@ -336,12 +334,12 @@ mod tests { if archive_storage { assert!( !env.async_storage() - .exists(&rustdoc_archive_path("package-1", "1.0.0")) + .exists(&rustdoc_archive_path("package-1", &V1)) .await? ); assert!( !env.async_storage() - .exists(&rustdoc_archive_path("package-1", "2.0.0")) + .exists(&rustdoc_archive_path("package-1", &V2)) .await? ); } else { @@ -349,7 +347,7 @@ mod tests { !env.async_storage() .rustdoc_file_exists( "package-1", - "1.0.0", + &V1, None, "package-1/index.html", archive_storage @@ -360,7 +358,7 @@ mod tests { !env.async_storage() .rustdoc_file_exists( "package-1", - "2.0.0", + &V2, None, "package-1/index.html", archive_storage @@ -394,7 +392,7 @@ mod tests { .collect()) } - async fn json_exists(storage: &AsyncStorage, version: &str) -> Result { + async fn json_exists(storage: &AsyncStorage, version: &Version) -> Result { storage .exists(&rustdoc_json_path( "a", @@ -411,7 +409,7 @@ mod tests { .fake_release() .await .name("a") - .version("1.0.0") + .version(V1) .archive_storage(archive_storage) .add_owner(CrateOwner { login: "malicious actor".into(), @@ -423,10 +421,10 @@ mod tests { assert!(release_exists(&mut conn, v1).await?); assert!( env.async_storage() - .rustdoc_file_exists("a", "1.0.0", None, "a/index.html", archive_storage) + .rustdoc_file_exists("a", &V1, None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(env.async_storage(), "1.0.0").await?); + assert!(json_exists(env.async_storage(), &V1).await?); let crate_id = sqlx::query_scalar!( r#"SELECT crate_id as "crate_id: CrateId" FROM releases WHERE id = $1"#, v1.0 @@ -442,7 +440,7 @@ mod tests { .fake_release() .await .name("a") - .version("2.0.0") + .version(V2) .archive_storage(archive_storage) .add_owner(CrateOwner { login: "Peter Rabbit".into(), @@ -454,21 +452,21 @@ mod tests { assert!(release_exists(&mut conn, v2).await?); assert!( env.async_storage() - .rustdoc_file_exists("a", "2.0.0", None, "a/index.html", archive_storage) + .rustdoc_file_exists("a", &V2, None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(env.async_storage(), "2.0.0").await?); + assert!(json_exists(env.async_storage(), &V2).await?); assert_eq!( owners(&mut conn, crate_id).await?, vec!["Peter Rabbit".to_string()] ); - delete_version(&mut conn, env.async_storage(), env.config(), "a", "1.0.0").await?; + delete_version(&mut conn, env.async_storage(), env.config(), "a", &V1).await?; assert!(!release_exists(&mut conn, v1).await?); if archive_storage { // for archive storage the archive and index files // need to be cleaned up. - let rustdoc_archive = rustdoc_archive_path("a", "1.0.0"); + let rustdoc_archive = rustdoc_archive_path("a", &V1); assert!(!env.async_storage().exists(&rustdoc_archive).await?); // local and remote index are gone too @@ -483,19 +481,19 @@ mod tests { } else { assert!( !env.async_storage() - .rustdoc_file_exists("a", "1.0.0", None, "a/index.html", archive_storage) + .rustdoc_file_exists("a", &V1, None, "a/index.html", archive_storage) .await? ); } - assert!(!json_exists(env.async_storage(), "1.0.0").await?); + assert!(!json_exists(env.async_storage(), &V1,).await?); assert!(release_exists(&mut conn, v2).await?); assert!( env.async_storage() - .rustdoc_file_exists("a", "2.0.0", None, "a/index.html", archive_storage) + .rustdoc_file_exists("a", &V2, None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(env.async_storage(), "2.0.0").await?); + assert!(json_exists(env.async_storage(), &V2).await?); assert_eq!( owners(&mut conn, crate_id).await?, vec!["Peter Rabbit".to_string()] @@ -517,10 +515,9 @@ mod tests { let mut conn = db.async_conn().await; let (release_id, _) = - fake_release_that_failed_before_build(&mut conn, "a", "1.0.0", "some-error") - .await?; + fake_release_that_failed_before_build(&mut conn, "a", V1, "some-error").await?; - delete_version(&mut conn, env.async_storage(), env.config(), "a", "1.0.0").await?; + delete_version(&mut conn, env.async_storage(), env.config(), "a", &V1).await?; assert!(!release_exists(&mut conn, release_id).await?); @@ -535,8 +532,7 @@ mod tests { let mut conn = db.async_conn().await; let (release_id, _) = - fake_release_that_failed_before_build(&mut conn, "a", "1.0.0", "some-error") - .await?; + fake_release_that_failed_before_build(&mut conn, "a", V1, "some-error").await?; delete_crate(&mut conn, env.async_storage(), env.config(), "a").await?; diff --git a/src/db/mod.rs b/src/db/mod.rs index eb874347e..2a808bdde 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -24,7 +24,7 @@ pub(crate) mod file; pub(crate) mod mimes; mod overrides; mod pool; -pub(crate) mod types; +pub mod types; static MIGRATOR: Migrator = sqlx::migrate!(); diff --git a/src/db/types.rs b/src/db/types/mod.rs similarity index 98% rename from src/db/types.rs rename to src/db/types/mod.rs index b3728ef1e..8d6b10f98 100644 --- a/src/db/types.rs +++ b/src/db/types/mod.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Serialize}; +pub mod version; + #[derive(Debug, Clone, PartialEq, Eq, Serialize, sqlx::Type)] #[sqlx(type_name = "feature")] pub struct Feature { diff --git a/src/db/types/version.rs b/src/db/types/version.rs new file mode 100644 index 000000000..6824c0ac2 --- /dev/null +++ b/src/db/types/version.rs @@ -0,0 +1,100 @@ +#[allow(clippy::disallowed_types)] +mod version_impl { + use crate::error::Result; + use derive_more::{Deref, Display, From, Into}; + use serde_with::{DeserializeFromStr, SerializeDisplay}; + use sqlx::{ + Postgres, + encode::IsNull, + error::BoxDynError, + postgres::{PgArgumentBuffer, PgTypeInfo, PgValueRef}, + prelude::*, + }; + use std::{io::Write, str::FromStr}; + + /// NewType around semver::Version to be able to use it with sqlx. + /// + /// Represented as string in the database. + #[derive( + Clone, + Debug, + Deref, + DeserializeFromStr, + Display, + Eq, + From, + Hash, + Into, + PartialEq, + SerializeDisplay, + )] + pub struct Version(pub semver::Version); + + impl Version { + pub const fn new(major: u64, minor: u64, patch: u64) -> Self { + Self(semver::Version::new(major, minor, patch)) + } + + pub fn parse(text: &str) -> Result { + Version::from_str(text) + } + } + + impl Type for Version { + fn type_info() -> PgTypeInfo { + >::type_info() + } + + fn compatible(ty: &PgTypeInfo) -> bool { + >::compatible(ty) + } + } + + impl<'q> Encode<'q, Postgres> for Version { + fn encode_by_ref(&self, buf: &mut PgArgumentBuffer) -> Result { + write!(**buf, "{}", self.0)?; + Ok(IsNull::No) + } + } + + impl<'r> Decode<'r, Postgres> for Version { + fn decode(value: PgValueRef<'r>) -> Result { + let s: &str = Decode::::decode(value)?; + Ok(Self(s.parse()?)) + } + } + + impl FromStr for Version { + type Err = semver::Error; + + fn from_str(s: &str) -> Result { + Ok(Version(semver::Version::from_str(s)?)) + } + } + + impl TryFrom<&str> for Version { + type Error = semver::Error; + + fn try_from(value: &str) -> Result { + Ok(Version(semver::Version::from_str(value)?)) + } + } + + impl TryFrom<&String> for Version { + type Error = semver::Error; + + fn try_from(value: &String) -> Result { + Ok(Version(semver::Version::from_str(value)?)) + } + } + + impl TryFrom for Version { + type Error = semver::Error; + + fn try_from(value: String) -> Result { + Ok(Version(semver::Version::from_str(&value)?)) + } + } +} + +pub use version_impl::Version; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 31b08209f..d02880a96 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -1,49 +1,52 @@ -use crate::RUSTDOC_STATIC_STORAGE_PREFIX; -use crate::db::{ - BuildId, - file::{add_path_into_database, file_list_to_json}, +use crate::{ + AsyncStorage, Config, Context, InstanceMetrics, RUSTDOC_STATIC_STORAGE_PREFIX, RegistryApi, + Storage, + db::{ + BuildId, CrateId, Pool, ReleaseId, add_doc_coverage, add_path_into_remote_archive, + blacklist::is_blacklisted, + file::{add_path_into_database, file_list_to_json}, + finish_build, finish_release, initialize_build, initialize_crate, initialize_release, + types::{BuildStatus, version::Version}, + update_build_with_error, update_crate_data_in_database, + }, + docbuilder::Limits, + error::Result, + repositories::RepositoryStatsUpdater, + storage::{ + CompressionAlgorithm, RustdocJsonFormatVersion, compress, get_file_list, + rustdoc_archive_path, rustdoc_json_path, source_archive_path, + }, + utils::{ + CargoMetadata, ConfigName, MetadataPackage, copy_dir_all, get_config, parse_rustc_version, + report_error, set_config, + }, }; -use crate::db::{CrateId, ReleaseId}; -use crate::db::{ - Pool, add_doc_coverage, add_path_into_remote_archive, finish_build, finish_release, - initialize_build, initialize_crate, initialize_release, types::BuildStatus, - update_build_with_error, update_crate_data_in_database, -}; -use crate::docbuilder::Limits; -use crate::error::Result; -use crate::repositories::RepositoryStatsUpdater; -use crate::storage::{ - CompressionAlgorithm, RustdocJsonFormatVersion, compress, get_file_list, rustdoc_archive_path, - rustdoc_json_path, source_archive_path, -}; -use crate::utils::{ - CargoMetadata, ConfigName, copy_dir_all, get_config, parse_rustc_version, report_error, - set_config, -}; -use crate::{AsyncStorage, Config, Context, InstanceMetrics, RegistryApi, Storage}; -use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage}; use anyhow::{Context as _, Error, anyhow, bail}; use docsrs_metadata::{BuildTargets, DEFAULT_TARGETS, HOST_TARGET, Metadata}; use itertools::Itertools as _; use regex::Regex; -use rustwide::cmd::{Command, CommandError, SandboxBuilder, SandboxImage}; -use rustwide::logging::{self, LogStorage}; -use rustwide::toolchain::ToolchainError; -use rustwide::{AlternativeRegistry, Build, Crate, Toolchain, Workspace, WorkspaceBuilder}; +use rustwide::{ + AlternativeRegistry, Build, Crate, Toolchain, Workspace, WorkspaceBuilder, + cmd::{Command, CommandError, SandboxBuilder, SandboxImage}, + logging::{self, LogStorage}, + toolchain::ToolchainError, +}; use serde::Deserialize; -use std::collections::{HashMap, HashSet}; -use std::fs::{self, File}; -use std::io::BufReader; -use std::path::Path; -use std::sync::Arc; -use std::time::Instant; +use std::{ + collections::{HashMap, HashSet}, + fs::{self, File}, + io::BufReader, + path::Path, + sync::Arc, + time::Instant, +}; use tokio::runtime; use tracing::{debug, error, info, info_span, instrument, warn}; const USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)"; const COMPONENTS: &[&str] = &["llvm-tools-preview", "rustc-dev", "rustfmt"]; const DUMMY_CRATE_NAME: &str = "empty-library"; -const DUMMY_CRATE_VERSION: &str = "1.0.0"; +const DUMMY_CRATE_VERSION: Version = Version::new(1, 0, 0); pub const RUSTDOC_JSON_COMPRESSION_ALGORITHMS: &[CompressionAlgorithm] = &[CompressionAlgorithm::Zstd, CompressionAlgorithm::Gzip]; @@ -314,7 +317,7 @@ impl RustwideBuilder { .build_dir(&format!("essential-files-{parsed_rustc_version}")); // This is an empty library crate that is supposed to always build. - let krate = Crate::crates_io(DUMMY_CRATE_NAME, DUMMY_CRATE_VERSION); + let krate = Crate::crates_io(DUMMY_CRATE_NAME, &DUMMY_CRATE_VERSION.to_string()); krate.fetch(&self.workspace)?; build_dir @@ -325,7 +328,7 @@ impl RustwideBuilder { let res = self.execute_build( BuildId(0), DUMMY_CRATE_NAME, - DUMMY_CRATE_VERSION, + &DUMMY_CRATE_VERSION, HOST_TARGET, true, build, @@ -393,7 +396,7 @@ impl RustwideBuilder { pub fn build_package( &mut self, name: &str, - version: &str, + version: &Version, kind: PackageKind<'_>, collect_metrics: bool, ) -> Result { @@ -439,7 +442,7 @@ impl RustwideBuilder { fn build_package_inner( &mut self, name: &str, - version: &str, + version: &Version, kind: PackageKind<'_>, crate_id: CrateId, release_id: ReleaseId, @@ -500,11 +503,12 @@ impl RustwideBuilder { let krate = { let _span = info_span!("krate.fetch").entered(); + let version = version.to_string(); let krate = match kind { PackageKind::Local(path) => Crate::local(path), - PackageKind::CratesIo => Crate::crates_io(name, version), + PackageKind::CratesIo => Crate::crates_io(name, &version), PackageKind::Registry(registry) => { - Crate::registry(AlternativeRegistry::new(registry), name, version) + Crate::registry(AlternativeRegistry::new(registry), name, &version) } }; krate.fetch(&self.workspace)?; @@ -784,7 +788,7 @@ impl RustwideBuilder { &self, build_id: BuildId, name: &str, - version: &str, + version: &Version, target: &str, build: &Build, limits: &Limits, @@ -829,7 +833,7 @@ impl RustwideBuilder { &self, build_id: BuildId, name: &str, - version: &str, + version: &Version, target: &str, is_default_target: bool, build: &Build, @@ -988,7 +992,7 @@ impl RustwideBuilder { &self, build_id: BuildId, name: &str, - version: &str, + version: &Version, target: &str, is_default_target: bool, build: &Build, @@ -1293,7 +1297,7 @@ mod tests { fn get_features( env: &TestEnvironment, name: &str, - version: &str, + version: &Version, ) -> Result>, sqlx::Error> { env.runtime().block_on(async { let mut conn = env.async_db().async_conn().await; @@ -1304,14 +1308,14 @@ mod tests { INNER JOIN crates ON crates.id = releases.crate_id WHERE crates.name = $1 AND releases.version = $2"#, name, - version, + version as _, ) .fetch_one(&mut *conn) .await }) } - fn remove_cache_files(env: &TestEnvironment, crate_: &str, version: &str) -> Result<()> { + fn remove_cache_files(env: &TestEnvironment, crate_: &str, version: &Version) -> Result<()> { let paths = [ format!("cache/index.crates.io-6f17d22bba15001f/{crate_}-{version}.crate"), format!("src/index.crates.io-6f17d22bba15001f/{crate_}-{version}"), @@ -1361,7 +1365,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); @@ -1390,7 +1394,7 @@ mod tests { c.name = $1 AND r.version = $2"#, crate_, - version, + version as _, ) .fetch_one(&mut *conn) .await @@ -1424,11 +1428,11 @@ mod tests { assert!(!storage.exists(&old_source_file)?); // doc archive exists - let doc_archive = rustdoc_archive_path(crate_, version); + let doc_archive = rustdoc_archive_path(crate_, &version); assert!(storage.exists(&doc_archive)?, "{}", doc_archive); // source archive exists - let source_archive = source_archive_path(crate_, version); + let source_archive = source_archive_path(crate_, &version); assert!(storage.exists(&source_archive)?, "{}", source_archive); // default target was built and is accessible @@ -1477,7 +1481,7 @@ mod tests { // check if rustdoc json files exist for all targets let path = rustdoc_json_path( crate_, - version, + &version, target, RustdocJsonFormatVersion::Latest, Some(*alg), @@ -1546,7 +1550,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, true)? + .build_package(crate_, &version, PackageKind::CratesIo, true)? .successful ); @@ -1569,7 +1573,7 @@ mod tests { // some binary crate let crate_ = "heater"; - let version = "0.2.3"; + let version = Version::new(0, 2, 3); let storage = env.storage(); let old_rustdoc_file = format!("rustdoc/{crate_}/{version}/some_doc_file"); @@ -1581,7 +1585,7 @@ mod tests { builder.update_toolchain()?; assert!( !builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); @@ -1600,7 +1604,7 @@ mod tests { c.name = $1 AND r.version = $2", crate_, - version + version as _ ) .fetch_one(&mut *conn) .await @@ -1610,11 +1614,11 @@ mod tests { assert_eq!(row.is_library, Some(false)); // doc archive exists - let doc_archive = rustdoc_archive_path(crate_, version); + let doc_archive = rustdoc_archive_path(crate_, &version); assert!(!storage.exists(&doc_archive)?); // source archive exists - let source_archive = source_archive_path(crate_, version); + let source_archive = source_archive_path(crate_, &version); assert!(storage.exists(&source_archive)?); // old rustdoc & source files still exist @@ -1632,13 +1636,13 @@ mod tests { // rand 0.8.5 fails to build with recent nightly versions // https://github.com/rust-lang/docs.rs/issues/26750 let crate_ = "rand"; - let version = "0.8.5"; + let version = Version::new(0, 8, 5); // create a successful release & build in the database let release_id = env.runtime().block_on(async { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, crate_).await?; - let release_id = initialize_release(&mut conn, crate_id, version).await?; + let release_id = initialize_release(&mut conn, crate_id, &version).await?; let build_id = initialize_build(&mut conn, release_id).await?; finish_build( &mut conn, @@ -1654,7 +1658,21 @@ mod tests { &mut conn, crate_id, release_id, - &MetadataPackage::default(), + &MetadataPackage { + name: crate_.into(), + version: version.clone(), + id: "".into(), + license: None, + repository: None, + homepage: None, + description: None, + documentation: None, + dependencies: vec![], + targets: vec![], + readme: None, + keywords: vec![], + features: HashMap::new(), + }, Path::new("/unknown/"), "x86_64-unknown-linux-gnu", serde_json::Value::Array(vec![]), @@ -1698,7 +1716,7 @@ mod tests { assert!( // not successful build !builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); @@ -1706,29 +1724,29 @@ mod tests { Ok(()) } - #[test_case("scsys-macros", "0.2.6")] - #[test_case("scsys-derive", "0.2.6")] - #[test_case("thiserror-impl", "1.0.26")] + #[test_case("scsys-macros", Version::new(0, 2, 6))] + #[test_case("scsys-derive", Version::new(0, 2, 6))] + #[test_case("thiserror-impl", Version::new(1, 0, 26))] #[ignore] - fn test_proc_macro(crate_: &str, version: &str) -> Result<()> { + fn test_proc_macro(crate_: &str, version: Version) -> Result<()> { let env = TestEnvironment::new_with_runtime()?; let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); let storage = env.storage(); // doc archive exists - let doc_archive = rustdoc_archive_path(crate_, version); + let doc_archive = rustdoc_archive_path(crate_, &version); assert!(storage.exists(&doc_archive)?); // source archive exists - let source_archive = source_archive_path(crate_, version); + let source_archive = source_archive_path(crate_, &version); assert!(storage.exists(&source_archive)?); Ok(()) @@ -1740,7 +1758,7 @@ mod tests { let env = TestEnvironment::new_with_runtime()?; let crate_ = "windows-win"; - let version = "2.4.1"; + let version = Version::new(2, 4, 1); let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; if builder.toolchain.as_ci().is_some() { @@ -1748,18 +1766,18 @@ mod tests { } assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); let storage = env.storage(); // doc archive exists - let doc_archive = rustdoc_archive_path(crate_, version); + let doc_archive = rustdoc_archive_path(crate_, &version); assert!(storage.exists(&doc_archive)?, "{}", doc_archive); // source archive exists - let source_archive = source_archive_path(crate_, version); + let source_archive = source_archive_path(crate_, &version); assert!(storage.exists(&source_archive)?, "{}", source_archive); let target = "x86_64-unknown-linux-gnu"; @@ -1791,7 +1809,7 @@ mod tests { )?; // if the corrected dependency of the crate was already downloaded we need to remove it - remove_cache_files(&env, "rand_core", "0.5.1")?; + remove_cache_files(&env, "rand_core", &Version::new(0, 5, 1))?; // Specific setup required: // * crate has a binary so that it is published with a lockfile @@ -1821,7 +1839,7 @@ mod tests { )?; // if the corrected dependency of the crate was already downloaded we need to remove it - remove_cache_files(&env, "value-bag-sval2", "1.4.1")?; + remove_cache_files(&env, "value-bag-sval2", &Version::new(1, 4, 1))?; // Similar to above, this crate fails to build with the published // lockfile, but generating a new working lockfile requires @@ -1844,12 +1862,12 @@ mod tests { let env = TestEnvironment::new_with_runtime()?; let crate_ = "proc-macro2"; - let version = "1.0.95"; + let version = Version::new(1, 0, 95); let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); Ok(()) @@ -1865,19 +1883,19 @@ mod tests { // Will succeed in the crate fetch step, so sources are // added. Will fail when we try to build. let crate_ = "simconnect-sys"; - let version = "0.23.1"; + let version = Version::new(0, 23, 1); let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; // `Result` is `Ok`, but the build-result is `false` assert!( !builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); // source archive exists - let source_archive = source_archive_path(crate_, version); + let source_archive = source_archive_path(crate_, &version); assert!( env.storage().exists(&source_archive)?, "archive doesnt exist: {source_archive}" @@ -1894,12 +1912,12 @@ mod tests { // https://github.com/rust-lang/docs.rs/issues/2491 // package without Cargo.toml, so fails directly in the fetch stage. let crate_ = "emheap"; - let version = "0.1.0"; + let version = Version::new(0, 1, 0); let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; // `Result` is `Ok`, but the build-result is `false` - let summary = builder.build_package(crate_, version, PackageKind::CratesIo, false)?; + let summary = builder.build_package(crate_, &version, PackageKind::CratesIo, false)?; assert!(!summary.successful); assert!(summary.should_reattempt); @@ -1918,7 +1936,7 @@ mod tests { INNER JOIN builds as b on b.rid = r.id WHERE c.name = $1 and r.version = $2"#, crate_, - version, + version as _, ) .fetch_one(&mut *conn) .await @@ -1938,17 +1956,17 @@ mod tests { let env = TestEnvironment::new_with_runtime()?; let crate_ = "serde"; - let version = "1.0.152"; + let version = Version::new(1, 0, 152); let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); assert!( - get_features(&env, crate_, version)? + get_features(&env, crate_, &version)? .unwrap() .iter() .any(|f| f.name == "serde_derive") @@ -1971,7 +1989,7 @@ mod tests { ); assert_eq!( - get_features(&env, "optional-dep", "0.0.1")? + get_features(&env, "optional-dep", &Version::new(0, 0, 1))? .unwrap() .iter() .map(|f| f.name.to_owned()) @@ -2063,7 +2081,7 @@ mod tests { builder .build_package( DUMMY_CRATE_NAME, - DUMMY_CRATE_VERSION, + &DUMMY_CRATE_VERSION, PackageKind::CratesIo, false )? diff --git a/src/registry_api.rs b/src/registry_api.rs index 74fe7f6dc..0631d7809 100644 --- a/src/registry_api.rs +++ b/src/registry_api.rs @@ -1,8 +1,7 @@ -use crate::{error::Result, utils::retry_async}; +use crate::{db::types::version::Version, error::Result, utils::retry_async}; use anyhow::{Context, anyhow, bail}; use chrono::{DateTime, Utc}; use reqwest::header::{ACCEPT, HeaderValue, USER_AGENT}; -use semver::Version; use serde::{Deserialize, Serialize}; use std::fmt; use tracing::instrument; @@ -119,7 +118,11 @@ impl RegistryApi { } #[instrument(skip(self))] - pub(crate) async fn get_release_data(&self, name: &str, version: &str) -> Result { + pub(crate) async fn get_release_data( + &self, + name: &str, + version: &Version, + ) -> Result { let (release_time, yanked, downloads) = self .get_release_time_yanked_downloads(name, version) .await @@ -136,7 +139,7 @@ impl RegistryApi { async fn get_release_time_yanked_downloads( &self, name: &str, - version: &str, + version: &Version, ) -> Result<(DateTime, bool, i32)> { let url = { let mut url = self.api_base.clone(); @@ -177,11 +180,10 @@ impl RegistryApi { .json() .await?; - let version = Version::parse(version)?; let version = response .versions .into_iter() - .find(|data| data.num == version) + .find(|data| data.num == *version) .with_context(|| anyhow!("Could not find version in response"))?; Ok((version.created_at, version.yanked, version.downloads)) diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 6925dfc58..5e4178825 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -4,15 +4,18 @@ mod database; mod s3; pub use self::compression::{CompressionAlgorithm, CompressionAlgorithms, compress, decompress}; -use self::compression::{wrap_reader_for_decompression, wrap_writer_for_compression}; -use self::database::DatabaseBackend; -use self::s3::S3Backend; +use self::{ + compression::{wrap_reader_for_decompression, wrap_writer_for_compression}, + database::DatabaseBackend, + s3::S3Backend, +}; use crate::{ Config, InstanceMetrics, db::{ BuildId, Pool, file::{FileEntry, detect_mime}, mimes, + types::version::Version, }, error::Result, utils::spawn_blocking, @@ -244,7 +247,7 @@ impl AsyncStorage { pub(crate) async fn stream_rustdoc_file( &self, name: &str, - version: &str, + version: &Version, latest_build_id: Option, path: &str, archive_storage: bool, @@ -264,7 +267,7 @@ impl AsyncStorage { pub(crate) async fn fetch_source_file( &self, name: &str, - version: &str, + version: &Version, latest_build_id: Option, path: &str, archive_storage: bool, @@ -287,7 +290,7 @@ impl AsyncStorage { pub(crate) async fn rustdoc_file_exists( &self, name: &str, - version: &str, + version: &Version, latest_build_id: Option, path: &str, archive_storage: bool, @@ -781,7 +784,7 @@ impl Storage { pub(crate) fn fetch_source_file( &self, name: &str, - version: &str, + version: &Version, latest_build_id: Option, path: &str, archive_storage: bool, @@ -798,7 +801,7 @@ impl Storage { pub(crate) fn rustdoc_file_exists( &self, name: &str, - version: &str, + version: &Version, latest_build_id: Option, path: &str, archive_storage: bool, @@ -959,7 +962,7 @@ impl std::fmt::Debug for Storage { } } -pub(crate) fn rustdoc_archive_path(name: &str, version: &str) -> String { +pub(crate) fn rustdoc_archive_path(name: &str, version: &Version) -> String { format!("rustdoc/{name}/{version}.zip") } @@ -984,7 +987,7 @@ impl FromStr for RustdocJsonFormatVersion { pub(crate) fn rustdoc_json_path( name: &str, - version: &str, + version: &Version, target: &str, format_version: RustdocJsonFormatVersion, compression_algorithm: Option, @@ -1001,7 +1004,7 @@ pub(crate) fn rustdoc_json_path( path } -pub(crate) fn source_archive_path(name: &str, version: &str) -> String { +pub(crate) fn source_archive_path(name: &str, version: &Version) -> String { format!("sources/{name}/{version}.zip") } diff --git a/src/test/fakes.rs b/src/test/fakes.rs index 4a84e9b86..4c4660089 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -1,37 +1,43 @@ use super::TestDatabase; - -use crate::db::file::{FileEntry, file_list_to_json}; -use crate::db::types::BuildStatus; -use crate::db::{ - BuildId, ReleaseId, initialize_build, initialize_crate, initialize_release, update_build_status, -}; -use crate::docbuilder::{DocCoverage, RUSTDOC_JSON_COMPRESSION_ALGORITHMS}; -use crate::error::Result; -use crate::registry_api::{CrateData, CrateOwner, ReleaseData}; -use crate::storage::{ - AsyncStorage, CompressionAlgorithm, RustdocJsonFormatVersion, compress, rustdoc_archive_path, - rustdoc_json_path, source_archive_path, +use crate::{ + db::{ + BuildId, ReleaseId, + file::{FileEntry, file_list_to_json}, + initialize_build, initialize_crate, initialize_release, + types::{BuildStatus, version::Version}, + update_build_status, + }, + docbuilder::{DocCoverage, RUSTDOC_JSON_COMPRESSION_ALGORITHMS}, + error::Result, + registry_api::{CrateData, CrateOwner, ReleaseData}, + storage::{ + AsyncStorage, CompressionAlgorithm, RustdocJsonFormatVersion, compress, + rustdoc_archive_path, rustdoc_json_path, source_archive_path, + }, + utils::{Dependency, MetadataPackage, cargo_metadata::Target}, }; -use crate::utils::{Dependency, MetadataPackage, cargo_metadata::Target}; use anyhow::{Context, bail}; use base64::{Engine, engine::general_purpose::STANDARD as b64}; use chrono::{DateTime, Utc}; -use std::collections::HashMap; -use std::iter; -use std::sync::Arc; +use std::{collections::HashMap, fmt, iter, sync::Arc}; use tracing::debug; /// Create a fake release in the database that failed before the build. /// This is a temporary small factory function only until we refactored the /// `FakeRelease` and `FakeBuild` factories to be more flexible. -pub(crate) async fn fake_release_that_failed_before_build( +pub(crate) async fn fake_release_that_failed_before_build( conn: &mut sqlx::PgConnection, name: &str, - version: &str, + version: V, errors: &str, -) -> Result<(ReleaseId, BuildId)> { +) -> Result<(ReleaseId, BuildId)> +where + V: TryInto, + V::Error: std::error::Error + Send + Sync + 'static, +{ + let version = version.try_into()?; let crate_id = initialize_crate(&mut *conn, name).await?; - let release_id = initialize_release(&mut *conn, crate_id, version).await?; + let release_id = initialize_release(&mut *conn, crate_id, &version).await?; let build_id = initialize_build(&mut *conn, release_id).await?; sqlx::query_scalar!( @@ -95,7 +101,7 @@ impl<'a> FakeRelease<'a> { package: MetadataPackage { id: "fake-package-id".into(), name: "fake-package".into(), - version: "1.0.0".into(), + version: Version::new(1, 0, 0), license: Some("MIT".into()), repository: Some("https://git.example.com".into()), homepage: Some("https://www.example.com".into()), @@ -103,7 +109,7 @@ impl<'a> FakeRelease<'a> { documentation: Some("https://docs.example.com".into()), dependencies: vec![Dependency { name: "fake-dependency".into(), - req: "^1.0.0".into(), + req: semver::VersionReq::parse("^1.0.0").unwrap(), kind: None, rename: None, optional: false, @@ -164,8 +170,12 @@ impl<'a> FakeRelease<'a> { self } - pub(crate) fn version(mut self, new: &str) -> Self { - self.package.version = new.into(); + pub(crate) fn version(mut self, new: V) -> Self + where + V: TryInto, + V::Error: fmt::Debug, + { + self.package.version = new.try_into().expect("invalid version"); self } diff --git a/src/test/mod.rs b/src/test/mod.rs index eabf7374d..3cde6e038 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -1,13 +1,15 @@ mod fakes; pub(crate) use self::fakes::{FakeBuild, fake_release_that_failed_before_build}; -use crate::cdn::CdnBackend; -use crate::config::ConfigBuilder; -use crate::db::{self, AsyncPoolClient, Pool}; -use crate::error::Result; -use crate::storage::{AsyncStorage, Storage, StorageKind}; -use crate::web::{build_axum_app, cache, page::TemplateData}; -use crate::{AsyncBuildQueue, BuildQueue, Config, Context, InstanceMetrics}; +use crate::{ + AsyncBuildQueue, BuildQueue, Config, Context, InstanceMetrics, + cdn::CdnBackend, + config::ConfigBuilder, + db::{self, AsyncPoolClient, Pool, types::version::Version}, + error::Result, + storage::{AsyncStorage, Storage, StorageKind}, + web::{build_axum_app, cache, page::TemplateData}, +}; use anyhow::Context as _; use axum::body::Bytes; use axum::{Router, body::Body, http::Request, response::Response as AxumResponse}; @@ -21,6 +23,12 @@ use tokio::{runtime, task::block_in_place}; use tower::ServiceExt; use tracing::error; +// some versions as constants for tests +pub(crate) const V0_1: Version = Version::new(0, 1, 0); +pub(crate) const V1: Version = Version::new(1, 0, 0); +pub(crate) const V2: Version = Version::new(2, 0, 0); +pub(crate) const V3: Version = Version::new(3, 0, 0); + pub(crate) fn async_wrapper(f: F) where F: FnOnce(Rc) -> Fut, diff --git a/src/utils/cargo_metadata.rs b/src/utils/cargo_metadata.rs index eea8e879f..f5327d686 100644 --- a/src/utils/cargo_metadata.rs +++ b/src/utils/cargo_metadata.rs @@ -1,6 +1,7 @@ -use crate::error::Result; +use crate::{db::types::version::Version, error::Result}; use anyhow::{Context, bail}; use rustwide::{Toolchain, Workspace, cmd::Command}; +use semver::VersionReq; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::path::Path; @@ -57,11 +58,11 @@ impl CargoMetadata { } } -#[derive(Debug, Deserialize, Serialize, Default)] +#[derive(Debug, Deserialize, Serialize)] pub(crate) struct Package { pub(crate) id: String, pub(crate) name: String, - pub(crate) version: String, + pub(crate) version: Version, pub(crate) license: Option, pub(crate) repository: Option, pub(crate) homepage: Option, @@ -128,7 +129,7 @@ impl Target { #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] pub(crate) struct Dependency { pub(crate) name: String, - pub(crate) req: String, + pub(crate) req: VersionReq, pub(crate) kind: Option, pub(crate) rename: Option, pub(crate) optional: bool, @@ -136,7 +137,7 @@ pub(crate) struct Dependency { impl Dependency { #[cfg(test)] - pub fn new(name: String, req: String) -> Dependency { + pub fn new(name: String, req: VersionReq) -> Dependency { Dependency { name, req, diff --git a/src/utils/consistency/data.rs b/src/utils/consistency/data.rs index 2c1e08d06..c38333f50 100644 --- a/src/utils/consistency/data.rs +++ b/src/utils/consistency/data.rs @@ -1,3 +1,5 @@ +use crate::db::types::version::Version; + #[derive(Clone, PartialEq, Debug)] pub(super) struct Crate { pub(super) name: String, @@ -10,6 +12,6 @@ pub(super) type Releases = Vec; #[derive(Clone, Debug, PartialEq)] pub(super) struct Release { - pub(super) version: String, + pub(super) version: Version, pub(super) yanked: Option, } diff --git a/src/utils/consistency/db.rs b/src/utils/consistency/db.rs index 42be527a8..c05c781e2 100644 --- a/src/utils/consistency/db.rs +++ b/src/utils/consistency/db.rs @@ -1,5 +1,5 @@ use super::data::{Crate, Crates, Release, Releases}; -use crate::Config; +use crate::{Config, db::types::version::Version}; use anyhow::Result; use itertools::Itertools; @@ -7,7 +7,7 @@ pub(super) async fn load(conn: &mut sqlx::PgConnection, config: &Config) -> Resu let rows = sqlx::query!( r#"SELECT name as "name!", - version as "version!", + version as "version!: Version", yanked FROM ( SELECT @@ -62,24 +62,24 @@ pub(super) async fn load(conn: &mut sqlx::PgConnection, config: &Config) -> Resu #[cfg(test)] mod tests { use super::*; - use crate::test::async_wrapper; + use crate::test::{V1, V2, V3, async_wrapper}; #[test] fn test_load() { async_wrapper(|env| async move { env.async_build_queue() - .add_crate("queued", "0.0.1", 0, None) + .add_crate("queued", &V1, 0, None) .await?; env.fake_release() .await .name("krate") - .version("0.0.2") + .version(V2) .create() .await?; env.fake_release() .await .name("krate") - .version("0.0.3") + .version(V3) .yanked(true) .create() .await?; @@ -94,11 +94,11 @@ mod tests { name: "krate".into(), releases: vec![ Release { - version: "0.0.2".into(), + version: V2, yanked: Some(false), }, Release { - version: "0.0.3".into(), + version: V3, yanked: Some(true), } ] @@ -106,7 +106,7 @@ mod tests { Crate { name: "queued".into(), releases: vec![Release { - version: "0.0.1".into(), + version: V1, yanked: None, }] }, diff --git a/src/utils/consistency/diff.rs b/src/utils/consistency/diff.rs index 78e929b60..efaa7fc9e 100644 --- a/src/utils/consistency/diff.rs +++ b/src/utils/consistency/diff.rs @@ -1,18 +1,18 @@ -use std::fmt::Display; - use super::data::Crate; +use crate::db::types::version::Version; use itertools::{ EitherOrBoth::{Both, Left, Right}, Itertools, }; +use std::fmt::Display; #[derive(Debug, PartialEq)] pub(super) enum Difference { CrateNotInIndex(String), - CrateNotInDb(String, Vec), - ReleaseNotInIndex(String, String), - ReleaseNotInDb(String, String), - ReleaseYank(String, String, bool), + CrateNotInDb(String, Vec), + ReleaseNotInIndex(String, Version), + ReleaseNotInDb(String, Version), + ReleaseYank(String, Version, bool), } impl Display for Difference { @@ -103,6 +103,8 @@ where #[cfg(test)] mod tests { + use crate::test::{V2, V3}; + use super::super::data::Release; use super::*; use std::iter; @@ -131,11 +133,11 @@ mod tests { name: "krate".into(), releases: vec![ Release { - version: "0.0.2".into(), + version: V2, yanked: Some(false), }, Release { - version: "0.0.3".into(), + version: V3, yanked: Some(true), }, ], @@ -143,10 +145,7 @@ mod tests { assert_eq!( calculate_diff([].iter(), index_releases.iter()), - vec![Difference::CrateNotInDb( - "krate".into(), - vec!["0.0.2".into(), "0.0.3".into()] - )] + vec![Difference::CrateNotInDb("krate".into(), vec![V2, V3])] ); } @@ -156,11 +155,11 @@ mod tests { name: "krate".into(), releases: vec![ Release { - version: "0.0.2".into(), + version: V2, yanked: Some(true), }, Release { - version: "0.0.3".into(), + version: V3, yanked: Some(true), }, ], @@ -169,11 +168,11 @@ mod tests { name: "krate".into(), releases: vec![ Release { - version: "0.0.2".into(), + version: V2, yanked: Some(false), }, Release { - version: "0.0.3".into(), + version: V3, yanked: Some(true), }, ], @@ -181,11 +180,7 @@ mod tests { assert_eq!( calculate_diff(db_releases.iter(), index_releases.iter()), - vec![Difference::ReleaseYank( - "krate".into(), - "0.0.2".into(), - false, - )] + vec![Difference::ReleaseYank("krate".into(), V2, false,)] ); } @@ -194,14 +189,14 @@ mod tests { let db_releases = [Crate { name: "krate".into(), releases: vec![Release { - version: "0.0.2".into(), + version: V2, yanked: None, }], }]; let index_releases = [Crate { name: "krate".into(), releases: vec![Release { - version: "0.0.2".into(), + version: V2, yanked: Some(false), }], }]; diff --git a/src/utils/consistency/index.rs b/src/utils/consistency/index.rs index 412f7a811..4370daefb 100644 --- a/src/utils/consistency/index.rs +++ b/src/utils/consistency/index.rs @@ -1,5 +1,5 @@ use super::data::{Crate, Crates, Release, Releases}; -use crate::{Config, utils::run_blocking}; +use crate::{Config, db::types::version::Version, utils::run_blocking}; use anyhow::Result; use rayon::iter::ParallelIterator; use tracing::debug; @@ -26,14 +26,19 @@ pub(super) async fn load(config: &Config) -> Result { .crates_parallel() .map(|krate| { krate.map(|krate| { - let mut releases: Releases = krate - .versions() - .iter() - .map(|version| Release { - version: version.version().into(), - yanked: Some(version.is_yanked()), - }) - .collect(); + let mut releases: Releases = + krate + .versions() + .iter() + .filter_map(|version| { + version.version().parse::().ok().map(|semversion| { + Release { + version: semversion, + yanked: Some(version.is_yanked()), + } + }) + }) + .collect(); releases.sort_by(|lhs, rhs| lhs.version.cmp(&rhs.version)); diff --git a/src/utils/consistency/mod.rs b/src/utils/consistency/mod.rs index 29ac32db0..d1c977448 100644 --- a/src/utils/consistency/mod.rs +++ b/src/utils/consistency/mod.rs @@ -156,7 +156,10 @@ where mod tests { use super::diff::Difference; use super::*; - use crate::test::{TestEnvironment, async_wrapper}; + use crate::{ + db::types::version::Version, + test::{TestEnvironment, V1, V2, async_wrapper}, + }; use sqlx::Row as _; async fn count(env: &TestEnvironment, sql: &str) -> Result { @@ -185,8 +188,8 @@ mod tests { env.fake_release() .await .name("krate") - .version("0.1.1") - .version("0.1.2") + .version(V1) + .version(V2) .create() .await?; @@ -218,20 +221,17 @@ mod tests { env.fake_release() .await .name("krate") - .version("0.1.1") + .version(V1) .create() .await?; env.fake_release() .await .name("krate") - .version("0.1.2") + .version(V2) .create() .await?; - let diff = [Difference::ReleaseNotInIndex( - "krate".into(), - "0.1.1".into(), - )]; + let diff = [Difference::ReleaseNotInIndex("krate".into(), V1)]; assert_eq!(count(&env, "SELECT count(*) FROM releases").await?, 2); @@ -242,8 +242,12 @@ mod tests { handle_diff(&env.context, diff.iter(), false).await?; assert_eq!( - single_row::(&env, "SELECT version FROM releases").await?, - vec!["0.1.2"] + single_row::( + &env, + r#"SELECT version as "version: Version" FROM releases"# + ) + .await?, + vec![V2] ); Ok(()) @@ -256,16 +260,12 @@ mod tests { env.fake_release() .await .name("krate") - .version("0.1.1") + .version(V1) .yanked(true) .create() .await?; - let diff = [Difference::ReleaseYank( - "krate".into(), - "0.1.1".into(), - false, - )]; + let diff = [Difference::ReleaseYank("krate".into(), V1, false)]; handle_diff(&env.context, diff.iter(), true).await?; @@ -288,7 +288,7 @@ mod tests { #[test] fn test_missing_release_in_db() { async_wrapper(|env| async move { - let diff = [Difference::ReleaseNotInDb("krate".into(), "0.1.1".into())]; + let diff = [Difference::ReleaseNotInDb("krate".into(), V1)]; handle_diff(&env.context, diff.iter(), true).await?; @@ -302,10 +302,10 @@ mod tests { build_queue .queued_crates() .await? - .iter() - .map(|c| (c.name.as_str(), c.version.as_str(), c.priority)) + .into_iter() + .map(|c| (c.name, V1, c.priority)) .collect::>(), - vec![("krate", "0.1.1", 15)] + vec![("krate".into(), V1, 15)] ); Ok(()) }) @@ -314,10 +314,7 @@ mod tests { #[test] fn test_missing_crate_in_db() { async_wrapper(|env| async move { - let diff = [Difference::CrateNotInDb( - "krate".into(), - vec!["0.1.1".into(), "0.1.2".into()], - )]; + let diff = [Difference::CrateNotInDb("krate".into(), vec![V1, V2])]; handle_diff(&env.context, diff.iter(), true).await?; @@ -331,10 +328,10 @@ mod tests { build_queue .queued_crates() .await? - .iter() - .map(|c| (c.name.as_str(), c.version.as_str(), c.priority)) + .into_iter() + .map(|c| (c.name, c.version, c.priority)) .collect::>(), - vec![("krate", "0.1.1", 15), ("krate", "0.1.2", 15)] + vec![("krate".into(), V1, 15), ("krate".into(), V2, 15)] ); Ok(()) }) diff --git a/src/utils/html.rs b/src/utils/html.rs index e0005be8b..28af67d4f 100644 --- a/src/utils/html.rs +++ b/src/utils/html.rs @@ -237,14 +237,14 @@ where #[cfg(test)] mod test { - use crate::test::{AxumResponseTestExt, AxumRouterTestExt, async_wrapper}; + use crate::test::{AxumResponseTestExt, AxumRouterTestExt, V1, async_wrapper}; #[test] fn rewriting_only_injects_css_once() { async_wrapper(|env| async move { env.fake_release().await .name("testing") - .version("0.1.0") + .version(V1) // A somewhat representative rustdoc html file from 2016 .rustdoc_file_with("2016/index.html", br#" @@ -272,10 +272,18 @@ mod test { .create().await?; let web = env.web_app().await; - let output = web.get("/testing/0.1.0/2016/").await?.text().await?; + let output = web + .get(&format!("/testing/{V1}/2016/")) + .await? + .text() + .await?; assert_eq!(output.matches(r#"href="/-/static/vendored.css"#).count(), 1); - let output = web.get("/testing/0.1.0/2022/").await?.text().await?; + let output = web + .get(&format!("/testing/{V1}/2022/")) + .await? + .text() + .await?; assert_eq!(output.matches(r#"href="/-/static/vendored.css"#).count(), 1); Ok(()) diff --git a/src/utils/version.rs b/src/utils/version.rs new file mode 100644 index 000000000..e69de29bb diff --git a/src/web/build_details.rs b/src/web/build_details.rs index fcef258bf..cc9a99b92 100644 --- a/src/web/build_details.rs +++ b/src/web/build_details.rs @@ -1,6 +1,9 @@ use crate::{ AsyncStorage, Config, - db::{BuildId, types::BuildStatus}, + db::{ + BuildId, + types::{BuildStatus, version::Version}, + }, impl_axum_webpage, web::{ MetaData, @@ -16,7 +19,6 @@ use askama::Template; use axum::{extract::Extension, response::IntoResponse}; use chrono::{DateTime, Utc}; use futures_util::TryStreamExt; -use semver::Version; use serde::Deserialize; use std::sync::Arc; diff --git a/src/web/builds.rs b/src/web/builds.rs index 4aa96bd54..65660a2dc 100644 --- a/src/web/builds.rs +++ b/src/web/builds.rs @@ -1,6 +1,9 @@ use crate::{ AsyncBuildQueue, Config, - db::{BuildId, types::BuildStatus}, + db::{ + BuildId, + types::{BuildStatus, version::Version}, + }, docbuilder::Limits, impl_axum_webpage, web::{ @@ -24,7 +27,6 @@ use axum_extra::{ use chrono::{DateTime, Utc}; use constant_time_eq::constant_time_eq; use http::StatusCode; -use semver::Version; use std::sync::Arc; #[derive(Debug, Clone, PartialEq, Eq)] @@ -126,9 +128,7 @@ async fn build_trigger_check( return Err(AxumNope::VersionNotFound); } - let crate_version_is_in_queue = build_queue - .has_build_queued(name, &version.to_string()) - .await?; + let crate_version_is_in_queue = build_queue.has_build_queued(name, version).await?; if crate_version_is_in_queue { return Err(AxumNope::BadRequest(anyhow!( @@ -175,7 +175,7 @@ pub(crate) async fn build_trigger_rebuild_handler( build_queue .add_crate( &name, - &version.to_string(), + &version, TRIGGERED_REBUILD_PRIORITY, None, /* because crates.io is the only service that calls this endpoint */ ) @@ -219,8 +219,8 @@ mod tests { use crate::{ db::Overrides, test::{ - AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, async_wrapper, - fake_release_that_failed_before_build, + AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, V1, V2, + async_wrapper, fake_release_that_failed_before_build, }, web::cache::CachePolicy, }; @@ -365,7 +365,7 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .create() .await?; @@ -412,14 +412,14 @@ mod tests { let build_queue = env.async_build_queue(); assert_eq!(build_queue.pending_count().await?, 0); - assert!(!build_queue.has_build_queued("foo", "0.1.0").await?); + assert!(!build_queue.has_build_queued("foo", &V1).await?); { let app = env.web_app().await; let response = app .oneshot( Request::builder() - .uri("/crate/foo/0.1.0/rebuild") + .uri(format!("/crate/foo/{V1}/rebuild")) .method("POST") .header("Authorization", &format!("Bearer {correct_token}")) .body(Body::empty()) @@ -432,14 +432,14 @@ mod tests { } assert_eq!(build_queue.pending_count().await?, 1); - assert!(build_queue.has_build_queued("foo", "0.1.0").await?); + assert!(build_queue.has_build_queued("foo", &V1).await?); { let app = env.web_app().await; let response = app .oneshot( Request::builder() - .uri("/crate/foo/0.1.0/rebuild") + .uri(format!("/crate/foo/{V1}/rebuild")) .method("POST") .header("Authorization", &format!("Bearer {correct_token}")) .body(Body::empty()) @@ -452,13 +452,13 @@ mod tests { json, serde_json::json!({ "title": "Bad request", - "message": "crate foo 0.1.0 already queued for rebuild" + "message": format!("crate foo {V1} already queued for rebuild") }) ); } assert_eq!(build_queue.pending_count().await?, 1); - assert!(build_queue.has_build_queued("foo", "0.1.0").await?); + assert!(build_queue.has_build_queued("foo", &V1).await?); Ok(()) } @@ -469,12 +469,16 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .no_builds() .create() .await?; - let response = env.web_app().await.get("/crate/foo/0.1.0/builds").await?; + let response = env + .web_app() + .await + .get(&format!("/crate/foo/{V1}/builds")) + .await?; response.assert_cache_control(CachePolicy::NoCaching, env.config()); let page = kuchikiki::parse_html().one(response.text().await?); @@ -506,7 +510,7 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .create() .await?; @@ -521,7 +525,7 @@ mod tests { let page = kuchikiki::parse_html().one( env.web_app() .await - .get("/crate/foo/0.1.0/builds") + .get(&format!("/crate/foo/{V1}/builds")) .await? .text() .await?, @@ -553,7 +557,7 @@ mod tests { env.fake_release() .await .name("aquarelle") - .version("0.1.0") + .version(V1) .builds(vec![ FakeBuild::default() .rustc_version("rustc (blabla 2019-01-01)") @@ -565,7 +569,7 @@ mod tests { env.fake_release() .await .name("aquarelle") - .version("0.2.0") + .version(V2) .builds(vec![ FakeBuild::default() .rustc_version("rustc (blabla 2019-01-01)") @@ -600,7 +604,7 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .builds(vec![ FakeBuild::default() .rustc_version("rustc (blabla 2019-01-01)") @@ -609,7 +613,11 @@ mod tests { .create() .await?; - let resp = env.web_app().await.get("/crate/foo/0.2.0/builds").await?; + let resp = env + .web_app() + .await + .get(&format!("/crate/foo/{V2}/builds")) + .await?; assert_eq!(resp.status(), StatusCode::NOT_FOUND); Ok(()) }); diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 253596817..df3fa4155 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -1,10 +1,13 @@ use crate::{ AsyncStorage, - db::{BuildId, CrateId, ReleaseDependency, ReleaseId, types::BuildStatus}, + db::{ + BuildId, CrateId, ReleaseDependency, ReleaseId, + types::{BuildStatus, version::Version}, + }, impl_axum_webpage, registry_api::OwnerKind, storage::PathNotFoundError, - utils::{Dependency, get_correct_docsrs_style_file, report_error}, + utils::{Dependency, get_correct_docsrs_style_file}, web::{ MatchedRelease, MetaData, ReqVersion, cache::CachePolicy, @@ -27,7 +30,6 @@ use axum::{ use chrono::{DateTime, Utc}; use futures_util::stream::TryStreamExt; use log::warn; -use semver::Version; use serde_json::Value; use std::sync::Arc; @@ -82,7 +84,7 @@ struct RepositoryMetadata { #[derive(Debug, Clone, Eq, PartialEq)] pub(crate) struct Release { pub id: ReleaseId, - pub version: semver::Version, + pub version: Version, #[allow(clippy::doc_overindented_list_items)] /// Aggregated build status of the release. /// * no builds -> build In progress @@ -308,7 +310,7 @@ impl CrateDetails { let manifest = match storage .fetch_source_file( &self.name, - &self.version.to_string(), + &self.version, self.latest_build_id, "Cargo.toml", self.archive_storage, @@ -337,7 +339,7 @@ impl CrateDetails { match storage .fetch_source_file( &self.name, - &self.version.to_string(), + &self.version, self.latest_build_id, path, self.archive_storage, @@ -389,7 +391,7 @@ pub(crate) async fn releases_for_crate( let mut releases: Vec = sqlx::query!( r#"SELECT releases.id as "id: ReleaseId", - releases.version, + releases.version as "version: Version", release_build_status.build_status as "build_status!: BuildStatus", releases.yanked, releases.is_library, @@ -406,22 +408,9 @@ pub(crate) async fn releases_for_crate( ) .fetch(&mut *conn) .try_filter_map(|row| async move { - let semversion = match semver::Version::parse(&row.version).with_context(|| { - format!( - "invalid semver in database for crate {crate_id}: {}", - row.version - ) - }) { - Ok(semver) => semver, - Err(err) => { - report_error(&err); - return Ok(None); - } - }; - Ok(Some(Release { id: row.id, - version: semversion, + version: row.version, build_status: row.build_status, yanked: row.yanked, is_library: row.is_library, @@ -733,6 +722,8 @@ mod tests { name: &str, version: &str, ) -> BuildStatus { + let version: Version = version.parse().expect("invalid version"); + let status = sqlx::query_scalar!( r#" SELECT build_status as "build_status!: BuildStatus" @@ -741,7 +732,7 @@ mod tests { INNER JOIN release_build_status ON releases.id = release_build_status.rid WHERE crates.name = $1 AND releases.version = $2"#, name, - version + version as _ ) .fetch_one(&mut *conn) .await @@ -757,12 +748,18 @@ mod tests { status } - async fn crate_details( + async fn crate_details( conn: &mut sqlx::PgConnection, name: &str, - version: &str, + version: V, req_version: Option, - ) -> CrateDetails { + ) -> CrateDetails + where + V: TryInto, + V::Error: std::error::Error + Send + Sync + 'static, + { + let version = version.try_into().expect("invalid version"); + let crate_id = sqlx::query_scalar!( r#"SELECT id as "id: CrateId" FROM crates WHERE name = $1"#, name @@ -773,16 +770,10 @@ mod tests { let releases = releases_for_crate(&mut *conn, crate_id).await.unwrap(); - CrateDetails::new( - &mut *conn, - name, - &Version::parse(version).unwrap(), - req_version, - releases, - ) - .await - .unwrap() - .unwrap() + CrateDetails::new(&mut *conn, name, &version, req_version, releases) + .await + .unwrap() + .unwrap() } #[fn_error_context::context( @@ -794,6 +785,7 @@ mod tests { version: &str, expected_last_successful_build: Option, ) -> Result<(), Error> { + let version = version.parse::()?; let mut conn = db.async_conn().await; let details = crate_details(&mut conn, package, version, None).await; @@ -1044,7 +1036,7 @@ mod tests { details.releases, vec![ Release { - version: semver::Version::parse("1.0.0")?, + version: Version::parse("1.0.0")?, build_status: BuildStatus::Success, yanked: Some(false), is_library: Some(true), @@ -1056,7 +1048,7 @@ mod tests { doc_targets: Some(vec!["x86_64-unknown-linux-gnu".into()]), }, Release { - version: semver::Version::parse("0.12.0")?, + version: Version::parse("0.12.0")?, build_status: BuildStatus::Success, yanked: Some(false), is_library: Some(true), @@ -1068,7 +1060,7 @@ mod tests { doc_targets: Some(vec!["x86_64-unknown-linux-gnu".into()]), }, Release { - version: semver::Version::parse("0.3.0")?, + version: Version::parse("0.3.0")?, build_status: BuildStatus::Failure, yanked: Some(false), is_library: Some(true), @@ -1080,7 +1072,7 @@ mod tests { doc_targets: Some(vec!["x86_64-unknown-linux-gnu".into()]), }, Release { - version: semver::Version::parse("0.2.0")?, + version: Version::parse("0.2.0")?, build_status: BuildStatus::Success, yanked: Some(true), is_library: Some(true), @@ -1092,7 +1084,7 @@ mod tests { doc_targets: Some(vec!["x86_64-unknown-linux-gnu".into()]), }, Release { - version: semver::Version::parse("0.2.0-alpha")?, + version: Version::parse("0.2.0-alpha")?, build_status: BuildStatus::Success, yanked: Some(false), is_library: Some(true), @@ -1104,7 +1096,7 @@ mod tests { doc_targets: Some(vec!["x86_64-unknown-linux-gnu".into()]), }, Release { - version: semver::Version::parse("0.1.1")?, + version: Version::parse("0.1.1")?, build_status: BuildStatus::Success, yanked: Some(false), is_library: Some(true), @@ -1116,7 +1108,7 @@ mod tests { doc_targets: Some(vec!["x86_64-unknown-linux-gnu".into()]), }, Release { - version: semver::Version::parse("0.1.0")?, + version: Version::parse("0.1.0")?, build_status: BuildStatus::Success, yanked: Some(false), is_library: Some(true), @@ -1128,7 +1120,7 @@ mod tests { doc_targets: Some(vec!["x86_64-unknown-linux-gnu".into()]), }, Release { - version: semver::Version::parse("0.0.1")?, + version: Version::parse("0.0.1")?, build_status: BuildStatus::Failure, yanked: Some(false), is_library: Some(false), @@ -1202,10 +1194,10 @@ mod tests { let mut conn = db.async_conn().await; for version in &["0.0.1", "0.0.2", "0.0.3"] { - let details = crate_details(&mut conn, "foo", version, None).await; + let details = crate_details(&mut conn, "foo", *version, None).await; assert_eq!( details.latest_release().unwrap().version, - semver::Version::parse("0.0.3")? + Version::parse("0.0.3")? ); } @@ -1238,11 +1230,11 @@ mod tests { .await?; let mut conn = db.async_conn().await; - for version in &["0.0.1", "0.0.2", "0.0.3-pre.1"] { + for &version in &["0.0.1", "0.0.2", "0.0.3-pre.1"] { let details = crate_details(&mut conn, "foo", version, None).await; assert_eq!( details.latest_release().unwrap().version, - semver::Version::parse("0.0.2")? + Version::parse("0.0.2")? ); } @@ -1276,11 +1268,11 @@ mod tests { .await?; let mut conn = db.async_conn().await; - for version in &["0.0.1", "0.0.2", "0.0.3"] { + for &version in &["0.0.1", "0.0.2", "0.0.3"] { let details = crate_details(&mut conn, "foo", version, None).await; assert_eq!( details.latest_release().unwrap().version, - semver::Version::parse("0.0.2")? + Version::parse("0.0.2")? ); } @@ -1316,11 +1308,11 @@ mod tests { .await?; let mut conn = db.async_conn().await; - for version in &["0.0.1", "0.0.2", "0.0.3"] { + for &version in &["0.0.1", "0.0.2", "0.0.3"] { let details = crate_details(&mut conn, "foo", version, None).await; assert_eq!( details.latest_release().unwrap().version, - semver::Version::parse("0.0.3")? + Version::parse("0.0.3")? ); } @@ -1350,11 +1342,11 @@ mod tests { .await?; let mut conn = db.async_conn().await; - for version in &["0.0.1", "0.0.2"] { + for &version in &["0.0.1", "0.0.2"] { let details = crate_details(&mut conn, "foo", version, None).await; assert_eq!( details.latest_release().unwrap().version, - semver::Version::parse("0.0.1")? + Version::parse("0.0.1")? ); } diff --git a/src/web/extractors/rustdoc.rs b/src/web/extractors/rustdoc.rs index 7f5bf5421..c0b0a6f7a 100644 --- a/src/web/extractors/rustdoc.rs +++ b/src/web/extractors/rustdoc.rs @@ -816,9 +816,11 @@ fn find_static_route_suffix<'a, 'b>(route: &'a str, path: &'b str) -> Option ReleaseId { + let version = Version::parse(version).unwrap(); env.fake_release() .await .name("foo") diff --git a/src/web/releases.rs b/src/web/releases.rs index 80bb78232..cda9ee494 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -3,7 +3,9 @@ use crate::{ AsyncBuildQueue, Config, InstanceMetrics, RegistryApi, build_queue::{QueuedCrate, REBUILD_PRIORITY}, - cdn, impl_axum_webpage, + cdn, + db::types::version::Version, + impl_axum_webpage, utils::report_error, web::{ ReqVersion, axum_redirect, encode_url_path, @@ -24,7 +26,6 @@ use base64::{Engine, engine::general_purpose::STANDARD as b64}; use chrono::{DateTime, Utc}; use futures_util::stream::TryStreamExt; use itertools::Itertools; -use semver::Version; use serde::{Deserialize, Serialize}; use sqlx::Row; use std::{ @@ -47,7 +48,7 @@ const RELEASES_IN_FEED: i64 = 150; #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct Release { pub(crate) name: String, - pub(crate) version: semver::Version, + pub(crate) version: Version, pub(crate) description: Option, pub(crate) target_name: Option, pub(crate) rustdoc_status: bool, @@ -91,8 +92,8 @@ pub(crate) async fn get_releases( }; let query = format!( - "SELECT crates.name, - releases.version, + r#"SELECT crates.name, + releases.version as "version: Version", releases.description, releases.target_name, releases.rustdoc_status, @@ -108,7 +109,7 @@ pub(crate) async fn get_releases( release_build_status.build_status != 'in_progress' ORDER BY {0} DESC - LIMIT $1 OFFSET $2", + LIMIT $1 OFFSET $2"#, ordering, if latest_only { "INNER JOIN releases ON crates.latest_version_id = releases.id" @@ -117,28 +118,23 @@ pub(crate) async fn get_releases( } ); - sqlx::query(query.as_str()) + Ok(sqlx::query(query.as_str()) .bind(limit) .bind(offset) .bind(filter_failed) .fetch(conn) - .err_into::() - .and_then(|row| async move { - let version: semver::Version = row.get::(1).parse()?; - - Ok(Release { - name: row.get(0), - version, - description: row.get(2), - target_name: row.get(3), - rustdoc_status: row.get::, _>(4).unwrap_or(false), - build_time: row.get(5), - stars: row.get::, _>(6).unwrap_or(0), - has_unyanked_releases: None, - }) + .map_ok(|row| Release { + name: row.get(0), + version: row.get(1), + description: row.get(2), + target_name: row.get(3), + rustdoc_status: row.get::, _>(4).unwrap_or(false), + build_time: row.get(5), + stars: row.get::, _>(6).unwrap_or(0), + has_unyanked_releases: None, }) .try_collect() - .await + .await?) } #[derive(Debug, Clone, PartialEq, Eq)] @@ -183,7 +179,7 @@ async fn get_search_results( let mut crates: HashMap = sqlx::query!( r#"SELECT crates.name, - releases.version, + releases.version as "version: Version", releases.description, release_build_status.last_build_time, releases.target_name, @@ -208,14 +204,12 @@ async fn get_search_results( &names[..], ) .fetch(&mut *conn) - .err_into::() - .and_then(|row| async move { - let version: semver::Version = row.version.parse()?; - Ok(( + .map_ok(|row| { + ( row.name.clone(), Release { name: row.name, - version, + version: row.version, description: row.description, build_time: row.last_build_time, target_name: row.target_name, @@ -223,7 +217,7 @@ async fn get_search_results( stars: row.stars.unwrap_or(0), has_unyanked_releases: row.has_unyanked_releases, }, - )) + ) }) .try_collect() .await?; @@ -751,7 +745,7 @@ pub(crate) async fn build_queue_handler( let in_progress_builds: Vec<(String, Version)> = sqlx::query!( r#"SELECT crates.name, - releases.version + releases.version as "version: Version" FROM builds INNER JOIN releases ON releases.id = builds.rid INNER JOIN crates ON releases.crate_id = crates.id @@ -762,14 +756,7 @@ pub(crate) async fn build_queue_handler( .fetch_all(&mut *conn) .await? .into_iter() - .map(|rec| { - ( - rec.name, - rec.version - .parse() - .expect("all versions in the db are valid"), - ) - }) + .map(|rec| (rec.name, rec.version)) .collect(); let mut rebuild_queue = Vec::new(); @@ -779,11 +766,8 @@ pub(crate) async fn build_queue_handler( .into_iter() .filter(|krate| { !in_progress_builds.iter().any(|(name, version)| { - // temporary, until I migrated the other version occurences to semver::Version - // We know that in the DB we only have semver - let krate_version: Version = krate.version.parse().unwrap(); // use `.any` instead of `.contains` to avoid cloning name& version for the match - *name == krate.name && *version == krate_version + *name == krate.name && *version == krate.version }) }) .collect_vec(); @@ -818,8 +802,8 @@ mod tests { use crate::db::{finish_build, initialize_build, initialize_crate, initialize_release}; use crate::registry_api::{CrateOwner, OwnerKind}; use crate::test::{ - AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, async_wrapper, - fake_release_that_failed_before_build, + AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, V0_1, V1, V2, V3, + async_wrapper, fake_release_that_failed_before_build, }; use anyhow::Error; use chrono::{Duration, TimeZone}; @@ -836,7 +820,7 @@ mod tests { let mut conn = db.async_conn().await; let crate_id = initialize_crate(&mut conn, "foo").await?; - let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; + let release_id = initialize_release(&mut conn, crate_id, &V1).await?; let build_id = initialize_build(&mut conn, release_id).await?; finish_build( @@ -872,21 +856,21 @@ mod tests { env.fake_release() .await .name("foo") - .version("1.0.0") + .version(V1) .github_stats("ghost/foo", 10, 10, 10) .create() .await?; env.fake_release() .await .name("bar") - .version("1.0.0") + .version(V1) .github_stats("ghost/bar", 20, 20, 20) .create() .await?; env.fake_release() .await .name("bar") - .version("1.0.0") + .version(V1) .github_stats("ghost/bar", 20, 20, 20) .create() .await?; @@ -894,7 +878,7 @@ mod tests { env.fake_release() .await .name("baz") - .version("1.0.0") + .version(V1) .create() .await?; @@ -902,7 +886,7 @@ mod tests { env.fake_release() .await .name("in_progress") - .version("0.1.0") + .version(V0_1) .builds(vec![ FakeBuild::default() .build_status(BuildStatus::InProgress) @@ -938,12 +922,14 @@ mod tests { env.fake_release() .await .name("some_random_crate") + .version(V1) .build_result_failed() .create() .await?; env.fake_release() .await .name("some_other_crate") + .version(V1) .create() .await?; @@ -963,11 +949,13 @@ mod tests { env.fake_release() .await .name("some_random_crate") + .version(V1) .create() .await?; env.fake_release() .await .name("some_other_crate") + .version(V1) .create() .await?; @@ -1000,11 +988,12 @@ mod tests { .await .github_stats("some/repo", 333, 22, 11) .name("some_random_crate") + .version(V1) .create() .await?; web.assert_redirect( "/releases/search?query=&i-am-feeling-lucky=1", - "/some_random_crate/1.0.0/some_random_crate/", + &format!("/some_random_crate/{V1}/some_random_crate/"), ) .await?; Ok(()) @@ -1872,9 +1861,9 @@ mod tests { ); let queue = env.async_build_queue(); - queue.add_crate("foo", "1.0.0", 0, None).await?; - queue.add_crate("bar", "0.1.0", -10, None).await?; - queue.add_crate("baz", "0.0.1", 10, None).await?; + queue.add_crate("foo", &V1, 0, None).await?; + queue.add_crate("bar", &V2, -10, None).await?; + queue.add_crate("baz", &V3, 10, None).await?; let full = kuchikiki::parse_html().one(web.get("/releases/queue").await?.text().await?); let items = full @@ -1884,14 +1873,14 @@ mod tests { assert_eq!(items.len(), 3); let expected = [ - ("bar", "0.1.0", Some(10)), - ("foo", "1.0.0", None), - ("baz", "0.0.1", Some(-10)), + ("bar", V2, Some(10)), + ("foo", V1, None), + ("baz", V3, Some(-10)), ]; for (li, expected) in items.iter().zip(&expected) { let a = li.as_node().select_first("a").expect("missing link"); assert!(a.text_contents().contains(expected.0)); - assert!(a.text_contents().contains(expected.1)); + assert!(a.text_contents().contains(&expected.1.to_string())); if let Some(priority) = expected.2 { assert!( @@ -1912,13 +1901,13 @@ mod tests { // we have two queued releases, where the build for one is already in progress let queue = env.async_build_queue(); - queue.add_crate("foo", "1.0.0", 0, None).await?; - queue.add_crate("bar", "0.1.0", 0, None).await?; + queue.add_crate("foo", &V1, 0, None).await?; + queue.add_crate("bar", &V2, 0, None).await?; env.fake_release() .await .name("foo") - .version("1.0.0") + .version(V1) .builds(vec![ FakeBuild::default() .build_status(BuildStatus::InProgress) @@ -1942,7 +1931,7 @@ mod tests { .expect("missing in progress list items") .map(|node| node.text_contents().trim().to_string()) .collect(); - assert_eq!(in_progress_items, vec!["foo 1.0.0"]); + assert_eq!(in_progress_items, vec![format!("foo {V1}")]); let queued_items: Vec<_> = lists[1] .as_node() @@ -1950,7 +1939,7 @@ mod tests { .expect("missing queued list items") .map(|node| node.text_contents().trim().to_string()) .collect(); - assert_eq!(queued_items, vec!["bar 0.1.0"]); + assert_eq!(queued_items, vec![format!("bar {V2}")]); Ok(()) }); @@ -1987,14 +1976,12 @@ mod tests { async_wrapper(|env| async move { let web = env.web_app().await; let queue = env.async_build_queue(); + queue.add_crate("foo", &V1, REBUILD_PRIORITY, None).await?; queue - .add_crate("foo", "1.0.0", REBUILD_PRIORITY, None) - .await?; - queue - .add_crate("bar", "0.1.0", REBUILD_PRIORITY + 1, None) + .add_crate("bar", &V2, REBUILD_PRIORITY + 1, None) .await?; queue - .add_crate("baz", "0.0.1", REBUILD_PRIORITY - 1, None) + .add_crate("baz", &V3, REBUILD_PRIORITY - 1, None) .await?; let full = kuchikiki::parse_html().one(web.get("/releases/queue").await?.text().await?); @@ -2229,7 +2216,7 @@ mod tests { env.fake_release() .await .name("failed") - .version(&format!("0.0.{i}")) + .version(format!("0.0.{i}")) .build_result_failed() .create() .await?; diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 528fc74f2..1d7044dec 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -296,7 +296,7 @@ pub(crate) async fn rustdoc_redirector_handler( match storage .stream_rustdoc_file( &crate_name, - &krate.version.to_string(), + &krate.version, krate.latest_build_id, inner_path, krate.archive_storage, @@ -493,7 +493,7 @@ pub(crate) async fn rustdoc_html_server_handler( let blob = match storage .stream_rustdoc_file( params.name(), - &krate.version.to_string(), + &krate.version, krate.latest_build_id, &storage_path, krate.archive_storage, @@ -521,7 +521,7 @@ pub(crate) async fn rustdoc_html_server_handler( if storage .rustdoc_file_exists( params.name(), - &krate.version.to_string(), + &krate.version, krate.latest_build_id, ¶ms.storage_path(), krate.archive_storage, @@ -559,8 +559,8 @@ pub(crate) async fn rustdoc_html_server_handler( { error!( krate = params.name(), - version = krate.version.to_string(), - original_path = params.inner_path(), + version = %krate.version, + original_path = params.original_path(), storage_path, "Couldn't find crate documentation root on storage. Something is wrong with the build." @@ -653,7 +653,7 @@ pub(crate) async fn target_redirect_handler( let redirect_uri = if storage .rustdoc_file_exists( params.name(), - &crate_details.version.to_string(), + &crate_details.version, crate_details.latest_build_id, &storage_path, crate_details.archive_storage, @@ -795,7 +795,7 @@ pub(crate) async fn json_download_handler( let storage_path = rustdoc_json_path( &krate.name, - &krate.version.to_string(), + &krate.version, &target, wanted_format_version, Some(wanted_compression), @@ -817,7 +817,7 @@ pub(crate) async fn json_download_handler( if wanted_compression == CompressionAlgorithm::Zstd { let storage_path = rustdoc_json_path( &krate.name, - &krate.version.to_string(), + &krate.version, &target, wanted_format_version, None, @@ -846,7 +846,7 @@ pub(crate) async fn download_handler( .assume_exact_name()? .into_version(); - let archive_path = rustdoc_archive_path(&name, &version.to_string()); + let archive_path = rustdoc_archive_path(&name, &version); // not all archives are set for public access yet, so we check if // the access is set and fix it if needed. @@ -893,6 +893,7 @@ mod test { use super::*; use crate::{ Config, + db::types::version::Version, docbuilder::RUSTDOC_JSON_COMPRESSION_ALGORITHMS, registry_api::{CrateOwner, OwnerKind}, storage::compression::file_extension_for, @@ -1493,7 +1494,7 @@ mod test { ("dummy_mixed-separators", "0.2.0"), ]; - for (name, version) in &rels { + for (name, version) in rels { env.fake_release() .await .name(name) @@ -3240,7 +3241,7 @@ mod test { .await?; const NAME: &str = "dummy"; - const VERSION: &str = "0.1.0"; + const VERSION: Version = Version::new(0, 1, 0); const TARGET: &str = "x86_64-unknown-linux-gnu"; const FORMAT_VERSION: RustdocJsonFormatVersion = RustdocJsonFormatVersion::Latest; @@ -3259,7 +3260,7 @@ mod test { .get( &rustdoc_json_path( NAME, - VERSION, + &VERSION, TARGET, FORMAT_VERSION, Some(CompressionAlgorithm::Zstd), @@ -3269,13 +3270,14 @@ mod test { .await?; for compression in RUSTDOC_JSON_COMPRESSION_ALGORITHMS { - let path = rustdoc_json_path(NAME, VERSION, TARGET, FORMAT_VERSION, Some(*compression)); + let path = + rustdoc_json_path(NAME, &VERSION, TARGET, FORMAT_VERSION, Some(*compression)); storage.delete_prefix(&path).await?; assert!(!storage.exists(&path).await?); } storage .store_one( - &rustdoc_json_path(NAME, VERSION, TARGET, FORMAT_VERSION, None), + &rustdoc_json_path(NAME, &VERSION, TARGET, FORMAT_VERSION, None), zstd_blob.content, ) .await?; diff --git a/src/web/source.rs b/src/web/source.rs index b07c0e8d4..d1b2bb9fa 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -1,6 +1,6 @@ use crate::{ AsyncStorage, - db::BuildId, + db::{BuildId, types::version::Version}, impl_axum_webpage, storage::PathNotFoundError, web::{ @@ -22,7 +22,6 @@ use askama::Template; use axum::{Extension, response::IntoResponse}; use axum_extra::headers::HeaderMapExt; use mime::Mime; -use semver::Version; use std::{cmp::Ordering, sync::Arc}; use tracing::instrument; @@ -85,7 +84,7 @@ impl FileList { INNER JOIN crates ON crates.id = releases.crate_id WHERE crates.name = $1 AND releases.version = $2", name, - version.to_string(), + version as _, ) .fetch_optional(&mut *conn) .await? @@ -231,7 +230,7 @@ pub(crate) async fn source_browser_handler( name = $1 AND version = $2"#, params.name(), - version.to_string() + version as _, ) .fetch_one(&mut *conn) .await?; @@ -244,7 +243,7 @@ pub(crate) async fn source_browser_handler( match storage .fetch_source_file( params.name(), - &version.to_string(), + &version, row.latest_build_id, inner_path, row.archive_storage, diff --git a/src/web/status.rs b/src/web/status.rs index ff9315864..1ee8dc248 100644 --- a/src/web/status.rs +++ b/src/web/status.rs @@ -46,8 +46,10 @@ pub(crate) async fn status_handler( #[cfg(test)] mod tests { - use crate::test::{AxumResponseTestExt, AxumRouterTestExt, async_wrapper}; - use crate::web::cache::CachePolicy; + use crate::{ + test::{AxumResponseTestExt, AxumRouterTestExt, async_wrapper}, + web::{ReqVersion, cache::CachePolicy}, + }; use reqwest::StatusCode; use test_case::test_case; @@ -55,8 +57,10 @@ mod tests { #[test_case("0.1")] #[test_case("0.1.0")] #[test_case("=0.1.0"; "exact_version")] - fn status(version: &str) { + fn status(req_version: &str) { async_wrapper(|env| async move { + let req_version: ReqVersion = req_version.parse()?; + env.fake_release() .await .name("foo") @@ -67,7 +71,7 @@ mod tests { let response = env .web_app() .await - .get_and_follow_redirects(&format!("/crate/foo/{version}/status.json")) + .get_and_follow_redirects(&format!("/crate/foo/{req_version}/status.json")) .await?; response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); assert_eq!(response.headers()["access-control-allow-origin"], "*"); @@ -109,8 +113,10 @@ mod tests { #[test_case("0.1")] #[test_case("~0.1"; "semver")] - fn redirect(version: &str) { + fn redirect(req_version: &str) { async_wrapper(|env| async move { + let req_version: ReqVersion = req_version.parse()?; + env.fake_release() .await .name("foo") @@ -121,7 +127,7 @@ mod tests { let web = env.web_app().await; let redirect = web .assert_redirect( - &format!("/crate/foo/{version}/status.json"), + &format!("/crate/foo/{req_version}/status.json"), "/crate/foo/0.1.0/status.json", ) .await?; @@ -136,8 +142,10 @@ mod tests { #[test_case("0.1")] #[test_case("0.1.0")] #[test_case("=0.1.0"; "exact_version")] - fn failure(version: &str) { + fn failure(req_version: &str) { async_wrapper(|env| async move { + let req_version: ReqVersion = req_version.parse()?; + env.fake_release() .await .name("foo") @@ -149,7 +157,7 @@ mod tests { let response = env .web_app() .await - .get_and_follow_redirects(&format!("/crate/foo/{version}/status.json")) + .get_and_follow_redirects(&format!("/crate/foo/{req_version}/status.json")) .await?; response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); assert_eq!(response.headers()["access-control-allow-origin"], "*"); @@ -178,7 +186,7 @@ mod tests { // invalid semver #[test_case("foo", "0,1")] #[test_case("foo", "0,1,0")] - fn not_found(krate: &str, version: &str) { + fn not_found(krate: &str, req_version: &str) { async_wrapper(|env| async move { env.fake_release() .await @@ -190,7 +198,7 @@ mod tests { let response = env .web_app() .await - .get_and_follow_redirects(&format!("/crate/{krate}/{version}/status.json")) + .get_and_follow_redirects(&format!("/crate/{krate}/{req_version}/status.json")) .await?; response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); assert_eq!(response.headers()["access-control-allow-origin"], "*"); diff --git a/templates/macros.html b/templates/macros.html index 8b55d9ea1..db8d0462c 100644 --- a/templates/macros.html +++ b/templates/macros.html @@ -159,7 +159,7 @@ {% macro dependencies_list(dependencies, use_crate_details, if_empty) %} {%- for dep in dependencies -%}
  • - {%- set dependency_params = RustdocParams::new(dep.name.to_owned()).with_req_version(ReqVersion::Semver(dep.req.parse::()?.clone())) -%} + {%- set dependency_params = RustdocParams::new(dep.name.to_owned()).with_req_version(ReqVersion::Semver(dep.req.clone())) -%} {%- set href -%} {%- if use_crate_details -%} {%- set href = dependency_params.crate_details_url() -%}