diff --git a/.sqlx/query-45dce653c51bb8a13c9b091b1aa5f997fccf6a563f654c7943b4593598acccbc.json b/.sqlx/query-45dce653c51bb8a13c9b091b1aa5f997fccf6a563f654c7943b4593598acccbc.json new file mode 100644 index 000000000..ca87c1cb2 --- /dev/null +++ b/.sqlx/query-45dce653c51bb8a13c9b091b1aa5f997fccf6a563f654c7943b4593598acccbc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n version as \"version: Version\"\n FROM releases\n WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version: Version", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "45dce653c51bb8a13c9b091b1aa5f997fccf6a563f654c7943b4593598acccbc" +} diff --git a/.sqlx/query-118390f408685404fa25f1de88df56c6f943b5530760163ff8a667fac627626f.json b/.sqlx/query-60accfe674de796dfc20077095e00bb854f208fd4ff7d954e1ed476667fd2eb4.json similarity index 69% rename from .sqlx/query-118390f408685404fa25f1de88df56c6f943b5530760163ff8a667fac627626f.json rename to .sqlx/query-60accfe674de796dfc20077095e00bb854f208fd4ff7d954e1ed476667fd2eb4.json index 496380a0c..97235c466 100644 --- a/.sqlx/query-118390f408685404fa25f1de88df56c6f943b5530760163ff8a667fac627626f.json +++ b/.sqlx/query-60accfe674de796dfc20077095e00bb854f208fd4ff7d954e1ed476667fd2eb4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n releases.id as \"id: ReleaseId\",\n releases.version,\n release_build_status.build_status as \"build_status!: BuildStatus\",\n releases.yanked,\n releases.is_library,\n releases.rustdoc_status,\n releases.release_time,\n releases.target_name\n FROM releases\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n WHERE\n releases.crate_id = $1", + "query": "SELECT\n releases.id as \"id: ReleaseId\",\n releases.version as \"version: Version\",\n release_build_status.build_status as \"build_status!: BuildStatus\",\n releases.yanked,\n releases.is_library,\n releases.rustdoc_status,\n releases.release_time,\n releases.target_name\n FROM releases\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n WHERE\n releases.crate_id = $1", "describe": { "columns": [ { @@ -10,7 +10,7 @@ }, { "ordinal": 1, - "name": "version", + "name": "version: Version", "type_info": "Text" }, { @@ -71,5 +71,5 @@ true ] }, - "hash": "118390f408685404fa25f1de88df56c6f943b5530760163ff8a667fac627626f" + "hash": "60accfe674de796dfc20077095e00bb854f208fd4ff7d954e1ed476667fd2eb4" } diff --git a/.sqlx/query-682dd9adacb3841d81633bb918bcfffb0d87d017a75e82b070ea493023a8d0f6.json b/.sqlx/query-682dd9adacb3841d81633bb918bcfffb0d87d017a75e82b070ea493023a8d0f6.json deleted file mode 100644 index 74ec96372..000000000 --- a/.sqlx/query-682dd9adacb3841d81633bb918bcfffb0d87d017a75e82b070ea493023a8d0f6.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT version FROM releases WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "version", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [ - false - ] - }, - "hash": "682dd9adacb3841d81633bb918bcfffb0d87d017a75e82b070ea493023a8d0f6" -} diff --git a/.sqlx/query-743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67.json b/.sqlx/query-743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67.json new file mode 100644 index 000000000..55b209a3f --- /dev/null +++ b/.sqlx/query-743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO queue (name, version, priority, registry)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (name, version) DO UPDATE\n SET priority = EXCLUDED.priority,\n registry = EXCLUDED.registry,\n attempt = 0,\n last_attempt = NULL\n ;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67" +} diff --git a/.sqlx/query-85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json b/.sqlx/query-85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json deleted file mode 100644 index 24c575ac8..000000000 --- a/.sqlx/query-85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n name as \"name!\",\n version as \"version!\",\n yanked\n FROM (\n SELECT\n crates.name,\n releases.version,\n releases.yanked\n FROM crates\n INNER JOIN releases ON releases.crate_id = crates.id\n UNION ALL\n -- crates & releases that are already queued\n -- don't have to be requeued.\n SELECT\n queue.name,\n queue.version,\n NULL as yanked\n FROM queue\n LEFT OUTER JOIN crates ON crates.name = queue.name\n LEFT OUTER JOIN releases ON (\n releases.crate_id = crates.id AND\n releases.version = queue.version\n )\n WHERE queue.attempt < $1 AND (\n crates.id IS NULL OR\n releases.id IS NULL\n )\n ) AS inp\n ORDER BY name, version", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name!", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "version!", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "yanked", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [ - null, - null, - null - ] - }, - "hash": "85c3232cd4a48116e3ab0be87b348d5c49e64c6dcebcf4fb0164fe14a8a5cb95" -} diff --git a/.sqlx/query-aebd890b300726ef82aad7137c2a6b69fd0f7baf3d82b0bc4807852315ec63cf.json b/.sqlx/query-aebd890b300726ef82aad7137c2a6b69fd0f7baf3d82b0bc4807852315ec63cf.json new file mode 100644 index 000000000..945a85e93 --- /dev/null +++ b/.sqlx/query-aebd890b300726ef82aad7137c2a6b69fd0f7baf3d82b0bc4807852315ec63cf.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT i.* FROM (\n SELECT\n c.name,\n r.version as \"version: Version\",\n (\n SELECT MAX(COALESCE(b.build_finished, b.build_started))\n FROM builds AS b\n WHERE b.rid = r.id\n ) AS last_build_attempt\n FROM crates AS c\n INNER JOIN releases AS r ON c.latest_version_id = r.id\n\n WHERE\n r.rustdoc_status = TRUE\n ) as i\n ORDER BY i.last_build_attempt ASC\n LIMIT $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "version: Version", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "last_build_attempt", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + null + ] + }, + "hash": "aebd890b300726ef82aad7137c2a6b69fd0f7baf3d82b0bc4807852315ec63cf" +} diff --git a/.sqlx/query-aee84ef8b568045ef4928498578911074a74b38084b93dbe62c23d9348b8e609.json b/.sqlx/query-aee84ef8b568045ef4928498578911074a74b38084b93dbe62c23d9348b8e609.json deleted file mode 100644 index 6f3b89e08..000000000 --- a/.sqlx/query-aee84ef8b568045ef4928498578911074a74b38084b93dbe62c23d9348b8e609.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT i.* FROM (\n SELECT\n c.name,\n r.version,\n (\n SELECT MAX(COALESCE(b.build_finished, b.build_started))\n FROM builds AS b\n WHERE b.rid = r.id\n ) AS last_build_attempt\n FROM crates AS c\n INNER JOIN releases AS r ON c.latest_version_id = r.id\n\n WHERE\n r.rustdoc_status = TRUE\n ) as i\n ORDER BY i.last_build_attempt ASC\n LIMIT $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "name", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "version", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "last_build_attempt", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false, - false, - null - ] - }, - "hash": "aee84ef8b568045ef4928498578911074a74b38084b93dbe62c23d9348b8e609" -} diff --git a/.sqlx/query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json b/.sqlx/query-af663241b6e7d77402847d57ef9a1d69412ea5af6d0b6562644fe913bac2e1df.json similarity index 51% rename from .sqlx/query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json rename to .sqlx/query-af663241b6e7d77402847d57ef9a1d69412ea5af6d0b6562644fe913bac2e1df.json index d9cbc56c1..ea4fc500a 100644 --- a/.sqlx/query-bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7.json +++ b/.sqlx/query-af663241b6e7d77402847d57ef9a1d69412ea5af6d0b6562644fe913bac2e1df.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n crates.name,\n releases.version,\n releases.description,\n release_build_status.last_build_time,\n releases.target_name,\n releases.rustdoc_status,\n repositories.stars as \"stars?\",\n EXISTS (\n SELECT 1\n FROM releases AS all_releases\n WHERE\n all_releases.crate_id = crates.id AND\n all_releases.yanked = false\n ) AS has_unyanked_releases\n\n FROM crates\n INNER JOIN releases ON crates.latest_version_id = releases.id\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n LEFT JOIN repositories ON releases.repository_id = repositories.id\n\n WHERE\n crates.name = ANY($1) AND\n release_build_status.build_status <> 'in_progress'", + "query": "SELECT\n crates.name,\n releases.version as \"version: Version\",\n releases.description,\n release_build_status.last_build_time,\n releases.target_name,\n releases.rustdoc_status,\n repositories.stars as \"stars?\",\n EXISTS (\n SELECT 1\n FROM releases AS all_releases\n WHERE\n all_releases.crate_id = crates.id AND\n all_releases.yanked = false\n ) AS has_unyanked_releases\n\n FROM crates\n INNER JOIN releases ON crates.latest_version_id = releases.id\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n LEFT JOIN repositories ON releases.repository_id = repositories.id\n\n WHERE\n crates.name = ANY($1) AND\n release_build_status.build_status <> 'in_progress'", "describe": { "columns": [ { @@ -10,7 +10,7 @@ }, { "ordinal": 1, - "name": "version", + "name": "version: Version", "type_info": "Text" }, { @@ -60,5 +60,5 @@ null ] }, - "hash": "bc0b3932dc2f8bd2b8a9f5a312262eafefd3b80b3322116448901aa55f2d89e7" + "hash": "af663241b6e7d77402847d57ef9a1d69412ea5af6d0b6562644fe913bac2e1df" } diff --git a/.sqlx/query-b2266a1d32ffedec1b35d94404d3747632ee3144ebe16cfa11d852f73a4ebbff.json b/.sqlx/query-b2266a1d32ffedec1b35d94404d3747632ee3144ebe16cfa11d852f73a4ebbff.json deleted file mode 100644 index d26a5fc1c..000000000 --- a/.sqlx/query-b2266a1d32ffedec1b35d94404d3747632ee3144ebe16cfa11d852f73a4ebbff.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO queue (name, version, priority, registry)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (name, version) DO UPDATE\n SET priority = EXCLUDED.priority,\n registry = EXCLUDED.registry,\n attempt = 0,\n last_attempt = NULL\n ;", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Int4", - "Text" - ] - }, - "nullable": [] - }, - "hash": "b2266a1d32ffedec1b35d94404d3747632ee3144ebe16cfa11d852f73a4ebbff" -} diff --git a/.sqlx/query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json b/.sqlx/query-bb6a174647b07fd68bb2e1010bab9fd12acd6a57e29bf2ecdef5a99ab2186ce0.json similarity index 55% rename from .sqlx/query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json rename to .sqlx/query-bb6a174647b07fd68bb2e1010bab9fd12acd6a57e29bf2ecdef5a99ab2186ce0.json index 00e60e300..189f1c888 100644 --- a/.sqlx/query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json +++ b/.sqlx/query-bb6a174647b07fd68bb2e1010bab9fd12acd6a57e29bf2ecdef5a99ab2186ce0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, name, version, priority, registry, attempt\n FROM queue\n WHERE\n attempt < $1 AND\n (last_attempt IS NULL OR last_attempt < NOW() - make_interval(secs => $2))\n ORDER BY priority ASC, attempt ASC, id ASC\n LIMIT 1\n FOR UPDATE SKIP LOCKED", + "query": "SELECT\n id,\n name,\n version as \"version: Version\",\n priority,\n registry,\n attempt\n FROM queue\n WHERE\n attempt < $1 AND\n (last_attempt IS NULL OR last_attempt < NOW() - make_interval(secs => $2))\n ORDER BY priority ASC, attempt ASC, id ASC\n LIMIT 1\n FOR UPDATE SKIP LOCKED", "describe": { "columns": [ { @@ -15,7 +15,7 @@ }, { "ordinal": 2, - "name": "version", + "name": "version: Version", "type_info": "Text" }, { @@ -49,5 +49,5 @@ false ] }, - "hash": "4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b" + "hash": "bb6a174647b07fd68bb2e1010bab9fd12acd6a57e29bf2ecdef5a99ab2186ce0" } diff --git a/.sqlx/query-3cb4dcb5778c77148aeb8dfc7f942ad45269ad7aab9e9c08fdaa9cb218dbc752.json b/.sqlx/query-db8654a53a9914ebfaa15a9bcc0af66fc2f46a8b5caa5473278123403ebbd613.json similarity index 50% rename from .sqlx/query-3cb4dcb5778c77148aeb8dfc7f942ad45269ad7aab9e9c08fdaa9cb218dbc752.json rename to .sqlx/query-db8654a53a9914ebfaa15a9bcc0af66fc2f46a8b5caa5473278123403ebbd613.json index 0d8bee347..b35981b73 100644 --- a/.sqlx/query-3cb4dcb5778c77148aeb8dfc7f942ad45269ad7aab9e9c08fdaa9cb218dbc752.json +++ b/.sqlx/query-db8654a53a9914ebfaa15a9bcc0af66fc2f46a8b5caa5473278123403ebbd613.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n crates.name,\n releases.version\n FROM builds\n INNER JOIN releases ON releases.id = builds.rid\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE\n builds.build_status = 'in_progress'\n ORDER BY builds.id ASC", + "query": "SELECT\n crates.name,\n releases.version as \"version: Version\"\n FROM builds\n INNER JOIN releases ON releases.id = builds.rid\n INNER JOIN crates ON releases.crate_id = crates.id\n WHERE\n builds.build_status = 'in_progress'\n ORDER BY builds.id ASC", "describe": { "columns": [ { @@ -10,7 +10,7 @@ }, { "ordinal": 1, - "name": "version", + "name": "version: Version", "type_info": "Text" } ], @@ -22,5 +22,5 @@ false ] }, - "hash": "3cb4dcb5778c77148aeb8dfc7f942ad45269ad7aab9e9c08fdaa9cb218dbc752" + "hash": "db8654a53a9914ebfaa15a9bcc0af66fc2f46a8b5caa5473278123403ebbd613" } diff --git a/.sqlx/query-4504bbfe4e4b21c82c4f56562d42e904426f9899fbf95f177e61b1f6be226ae4.json b/.sqlx/query-e95856a4991ea37e23ee3f28fbc171111accba3638f90abb39e20484d034dfe9.json similarity index 62% rename from .sqlx/query-4504bbfe4e4b21c82c4f56562d42e904426f9899fbf95f177e61b1f6be226ae4.json rename to .sqlx/query-e95856a4991ea37e23ee3f28fbc171111accba3638f90abb39e20484d034dfe9.json index 3a6e5b742..fdc235e64 100644 --- a/.sqlx/query-4504bbfe4e4b21c82c4f56562d42e904426f9899fbf95f177e61b1f6be226ae4.json +++ b/.sqlx/query-e95856a4991ea37e23ee3f28fbc171111accba3638f90abb39e20484d034dfe9.json @@ -1,12 +1,14 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO queue (name, version, priority, attempt, last_attempt )\n VALUES ('failed_crate', '0.1.1', 0, 99, NOW())", + "query": "\n INSERT INTO queue (name, version, priority, attempt, last_attempt )\n VALUES ('failed_crate', $1, 0, 99, NOW())", "describe": { "columns": [], "parameters": { - "Left": [] + "Left": [ + "Text" + ] }, "nullable": [] }, - "hash": "4504bbfe4e4b21c82c4f56562d42e904426f9899fbf95f177e61b1f6be226ae4" + "hash": "e95856a4991ea37e23ee3f28fbc171111accba3638f90abb39e20484d034dfe9" } diff --git a/.sqlx/query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json b/.sqlx/query-e9e7696f5b32df8981648f3c0ca71c9d032b39b373f2db2713365c2efb03d408.json similarity index 64% rename from .sqlx/query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json rename to .sqlx/query-e9e7696f5b32df8981648f3c0ca71c9d032b39b373f2db2713365c2efb03d408.json index cbd84ed72..baa946d77 100644 --- a/.sqlx/query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json +++ b/.sqlx/query-e9e7696f5b32df8981648f3c0ca71c9d032b39b373f2db2713365c2efb03d408.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, name, version, priority, registry, attempt\n FROM queue\n WHERE attempt < $1\n ORDER BY priority ASC, attempt ASC, id ASC", + "query": "SELECT\n id,\n name,\n version as \"version: Version\",\n priority,\n registry,\n attempt\n FROM queue\n WHERE attempt < $1\n ORDER BY priority ASC, attempt ASC, id ASC", "describe": { "columns": [ { @@ -15,7 +15,7 @@ }, { "ordinal": 2, - "name": "version", + "name": "version: Version", "type_info": "Text" }, { @@ -48,5 +48,5 @@ false ] }, - "hash": "9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132" + "hash": "e9e7696f5b32df8981648f3c0ca71c9d032b39b373f2db2713365c2efb03d408" } diff --git a/.sqlx/query-f0bdee589c06dea8b71c5dd09cf8d3a4ebef2018ccefb61af8ed2dde44148ccc.json b/.sqlx/query-f0bdee589c06dea8b71c5dd09cf8d3a4ebef2018ccefb61af8ed2dde44148ccc.json new file mode 100644 index 000000000..3951f398f --- /dev/null +++ b/.sqlx/query-f0bdee589c06dea8b71c5dd09cf8d3a4ebef2018ccefb61af8ed2dde44148ccc.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n name as \"name!\",\n version as \"version!: Version\",\n yanked\n FROM (\n SELECT\n crates.name,\n releases.version,\n releases.yanked\n FROM crates\n INNER JOIN releases ON releases.crate_id = crates.id\n UNION ALL\n -- crates & releases that are already queued\n -- don't have to be requeued.\n SELECT\n queue.name,\n queue.version,\n NULL as yanked\n FROM queue\n LEFT OUTER JOIN crates ON crates.name = queue.name\n LEFT OUTER JOIN releases ON (\n releases.crate_id = crates.id AND\n releases.version = queue.version\n )\n WHERE queue.attempt < $1 AND (\n crates.id IS NULL OR\n releases.id IS NULL\n )\n ) AS inp\n ORDER BY name, version", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "version!: Version", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "yanked", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "f0bdee589c06dea8b71c5dd09cf8d3a4ebef2018ccefb61af8ed2dde44148ccc" +} diff --git a/Cargo.lock b/Cargo.lock index 3328d6f47..99d5b9541 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1203,6 +1203,39 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "camino" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "276a59bf2b2c967788139340c9f0c5b12d7fd6630315c15c217e559de85d2609" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cargo-platform" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122ec45a44b270afd1402f351b782c676b173e3c3fb28d86ff7ebfb4d86a4ee4" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "981a6f317983eec002839b90fae7411a85621410ae591a9cab2ecf5cb5744873" +dependencies = [ + "camino", + "cargo-platform", + "derive_builder", + "semver", + "serde", + "serde_json", + "thiserror 2.0.17", +] + [[package]] name = "caseless" version = "0.2.2" @@ -2024,6 +2057,7 @@ dependencies = [ "backtrace", "base64 0.22.1", "bzip2", + "cargo_metadata", "chrono", "clap", "comrak", diff --git a/Cargo.toml b/Cargo.toml index d01b04a9c..16717266a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ bzip2 = "0.6.0" getrandom = "0.3.1" itertools = { version = "0.14.0" } hex = "0.4.3" -derive_more = { version = "2.0.0", features = ["display"] } +derive_more = { version = "2.0.0", features = ["display", "deref", "from", "into"] } sysinfo = { version = "0.37.2", default-features = false, features = ["system"] } derive_builder = "0.20.2" @@ -75,6 +75,7 @@ aws-sdk-cloudfront = "1.3.0" aws-smithy-types-convert = { version = "0.60.0", features = ["convert-chrono"] } http = "1.0.0" uuid = { version = "1.1.2", features = ["v4"]} +cargo_metadata = "0.23.0" # Data serialization and deserialization serde = { version = "1.0", features = ["derive"] } @@ -118,6 +119,7 @@ aws-smithy-runtime = {version = "1.0.1", features = ["client", "test-util"]} aws-smithy-http = "0.62.0" indoc = "2.0.0" pretty_assertions = "1.4.0" +cargo_metadata = { version = "0.23.0", features = ["builder"] } [profile.dev.package."*"] opt-level = 2 diff --git a/clippy.toml b/clippy.toml index df7b6040e..b1eb5d507 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,3 +1,7 @@ [[disallowed-types]] path = "axum::extract::Path" reason = "use our own custom web::extractors::Path for a nicer error response" + +[[disallowed-types]] +path = "semver::Version" +reason = "use our own custom db::types::version::Version so you can use it with sqlx" diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 61c4dc1dd..599f4f3a7 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -2,7 +2,7 @@ use anyhow::{Context as _, Result, anyhow}; use clap::{Parser, Subcommand, ValueEnum}; use docs_rs::{ Config, Context, PackageKind, RustwideBuilder, - db::{self, CrateId, Overrides, add_path_into_database}, + db::{self, CrateId, Overrides, add_path_into_database, types::version::Version}, start_background_metrics_webserver, start_web_server, utils::{ ConfigName, get_config, get_crate_pattern_and_priority, list_crate_priorities, @@ -237,7 +237,7 @@ enum QueueSubcommand { crate_name: String, /// Version of crate to build #[arg(name = "CRATE_VERSION")] - crate_version: String, + crate_version: Version, /// Priority of build (new crate builds get priority 0) #[arg( name = "BUILD_PRIORITY", @@ -396,7 +396,7 @@ enum BuildSubcommand { /// Version of crate #[arg(name = "CRATE_VERSION")] - crate_version: Option, + crate_version: Option, /// Build a crate at a specific path #[arg(short = 'l', long = "local", conflicts_with_all(&["CRATE_NAME", "CRATE_VERSION"]))] @@ -794,6 +794,6 @@ enum DeleteSubcommand { /// The version of the crate to delete #[arg(name = "VERSION")] - version: String, + version: Version, }, } diff --git a/src/build_queue.rs b/src/build_queue.rs index ec7855846..8a047648b 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -1,17 +1,19 @@ -use crate::Context; -use crate::db::{CrateId, Pool, delete_crate, delete_version, update_latest_version_id}; -use crate::docbuilder::PackageKind; -use crate::error::Result; -use crate::storage::AsyncStorage; -use crate::utils::{ConfigName, get_config, get_crate_priority, report_error, retry, set_config}; -use crate::{BuildPackageSummary, cdn}; -use crate::{Config, Index, InstanceMetrics, RustwideBuilder}; +use crate::{ + BuildPackageSummary, Config, Context, Index, InstanceMetrics, RustwideBuilder, cdn, + db::{ + CrateId, Pool, delete_crate, delete_version, types::version::Version, + update_latest_version_id, + }, + docbuilder::PackageKind, + error::Result, + storage::AsyncStorage, + utils::{ConfigName, get_config, get_crate_priority, report_error, retry, set_config}, +}; use anyhow::Context as _; use fn_error_context::context; use futures_util::{StreamExt, stream::TryStreamExt}; use sqlx::Connection as _; -use std::collections::HashMap; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use tokio::runtime; use tracing::{debug, error, info, instrument}; @@ -26,7 +28,7 @@ pub(crate) struct QueuedCrate { #[serde(skip)] id: i32, pub(crate) name: String, - pub(crate) version: String, + pub(crate) version: Version, pub(crate) priority: i32, pub(crate) registry: Option, pub(crate) attempt: i32, @@ -87,7 +89,7 @@ impl AsyncBuildQueue { pub async fn add_crate( &self, name: &str, - version: &str, + version: &Version, priority: i32, registry: Option<&str>, ) -> Result<()> { @@ -95,15 +97,15 @@ impl AsyncBuildQueue { sqlx::query!( "INSERT INTO queue (name, version, priority, registry) - VALUES ($1, $2, $3, $4) - ON CONFLICT (name, version) DO UPDATE - SET priority = EXCLUDED.priority, - registry = EXCLUDED.registry, - attempt = 0, - last_attempt = NULL - ;", + VALUES ($1, $2, $3, $4) + ON CONFLICT (name, version) DO UPDATE + SET priority = EXCLUDED.priority, + registry = EXCLUDED.registry, + attempt = 0, + last_attempt = NULL + ;", name, - version, + version as _, priority, registry, ) @@ -166,17 +168,23 @@ impl AsyncBuildQueue { Ok(sqlx::query_as!( QueuedCrate, - "SELECT id, name, version, priority, registry, attempt - FROM queue - WHERE attempt < $1 - ORDER BY priority ASC, attempt ASC, id ASC", + r#"SELECT + id, + name, + version as "version: Version", + priority, + registry, + attempt + FROM queue + WHERE attempt < $1 + ORDER BY priority ASC, attempt ASC, id ASC"#, self.max_attempts ) .fetch_all(&mut *conn) .await?) } - pub(crate) async fn has_build_queued(&self, name: &str, version: &str) -> Result { + pub(crate) async fn has_build_queued(&self, name: &str, version: &Version) -> Result { let mut conn = self.db.get_async().await?; Ok(sqlx::query_scalar!( "SELECT id @@ -188,7 +196,7 @@ impl AsyncBuildQueue { ", self.max_attempts, name, - version, + version as _, ) .fetch_optional(&mut *conn) .await? @@ -267,7 +275,10 @@ impl AsyncBuildQueue { &self.storage, &self.config, &release.name, - &release.version, + &release + .version + .parse() + .context("couldn't parse release version as semver")?, ) .await .with_context(|| { @@ -296,7 +307,10 @@ impl AsyncBuildQueue { match self .add_crate( &release.name, - &release.version, + &release + .version + .parse() + .context("couldn't parse release version as semver")?, priority, index.repository_url(), ) @@ -324,14 +338,15 @@ impl AsyncBuildQueue { if let Some(release) = yanked.or(unyanked) { // FIXME: delay yanks of crates that have not yet finished building // https://github.com/rust-lang/docs.rs/issues/1934 - if let Err(err) = self - .set_yanked_inner( - &mut conn, - release.name.as_str(), - release.version.as_str(), - yanked.is_some(), - ) - .await + if let Ok(release_version) = Version::parse(&release.version) + && let Err(err) = self + .set_yanked_inner( + &mut conn, + release.name.as_str(), + &release_version, + yanked.is_some(), + ) + .await { report_error(&err); } @@ -352,7 +367,7 @@ impl AsyncBuildQueue { Ok(crates_added) } - pub async fn set_yanked(&self, name: &str, version: &str, yanked: bool) -> Result<()> { + pub async fn set_yanked(&self, name: &str, version: &Version, yanked: bool) -> Result<()> { let mut conn = self.db.get_async().await?; self.set_yanked_inner(&mut conn, name, version, yanked) .await @@ -363,7 +378,7 @@ impl AsyncBuildQueue { &self, conn: &mut sqlx::PgConnection, name: &str, - version: &str, + version: &Version, yanked: bool, ) -> Result<()> { let activity = if yanked { "yanked" } else { "unyanked" }; @@ -378,7 +393,7 @@ impl AsyncBuildQueue { RETURNING crates.id as "id: CrateId" "#, name, - version, + version as _, yanked, ) .fetch_optional(&mut *conn) @@ -424,7 +439,7 @@ impl BuildQueue { pub fn add_crate( &self, name: &str, - version: &str, + version: &Version, priority: i32, registry: Option<&str>, ) -> Result<()> { @@ -432,7 +447,7 @@ impl BuildQueue { .block_on(self.inner.add_crate(name, version, priority, registry)) } - pub fn set_yanked(&self, name: &str, version: &str, yanked: bool) -> Result<()> { + pub fn set_yanked(&self, name: &str, version: &Version, yanked: bool) -> Result<()> { self.runtime .block_on(self.inner.set_yanked(name, version, yanked)) } @@ -496,14 +511,20 @@ impl BuildQueue { let to_process = match self.runtime.block_on( sqlx::query_as!( QueuedCrate, - "SELECT id, name, version, priority, registry, attempt + r#"SELECT + id, + name, + version as "version: Version", + priority, + registry, + attempt FROM queue WHERE attempt < $1 AND (last_attempt IS NULL OR last_attempt < NOW() - make_interval(secs => $2)) ORDER BY priority ASC, attempt ASC, id ASC LIMIT 1 - FOR UPDATE SKIP LOCKED", + FOR UPDATE SKIP LOCKED"#, self.inner.max_attempts, self.inner.config.delay_between_build_attempts.as_secs_f64(), ) @@ -690,10 +711,10 @@ pub async fn queue_rebuilds( } let mut results = sqlx::query!( - "SELECT i.* FROM ( + r#"SELECT i.* FROM ( SELECT c.name, - r.version, + r.version as "version: Version", ( SELECT MAX(COALESCE(b.build_finished, b.build_started)) FROM builds AS b @@ -706,7 +727,7 @@ pub async fn queue_rebuilds( r.rustdoc_status = TRUE ) as i ORDER BY i.last_build_attempt ASC - LIMIT $1", + LIMIT $1"#, rebuilds_to_queue, ) .fetch(&mut *conn); @@ -730,9 +751,8 @@ pub async fn queue_rebuilds( #[cfg(test)] mod tests { - use crate::test::{FakeBuild, TestEnvironment}; - use super::*; + use crate::test::{FakeBuild, TestEnvironment, V1, V2}; use chrono::Utc; use std::time::Duration; @@ -748,7 +768,7 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .builds(vec![ FakeBuild::default().rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), ]) @@ -764,7 +784,7 @@ mod tests { let queue = build_queue.queued_crates().await?; assert_eq!(queue.len(), 1); assert_eq!(queue[0].name, "foo"); - assert_eq!(queue[0].version, "0.1.0"); + assert_eq!(queue[0].version, V1); assert_eq!(queue[0].priority, REBUILD_PRIORITY); Ok(()) @@ -781,10 +801,10 @@ mod tests { let build_queue = env.async_build_queue(); build_queue - .add_crate("foo1", "0.1.0", REBUILD_PRIORITY, None) + .add_crate("foo1", &V1, REBUILD_PRIORITY, None) .await?; build_queue - .add_crate("foo2", "0.1.0", REBUILD_PRIORITY, None) + .add_crate("foo2", &V1, REBUILD_PRIORITY, None) .await?; let mut conn = env.async_db().async_conn().await; @@ -797,7 +817,7 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .builds(vec![ FakeBuild::default().rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), ]) @@ -823,16 +843,16 @@ mod tests { let build_queue = env.async_build_queue(); build_queue - .add_crate("foo1", "0.1.0", REBUILD_PRIORITY, None) + .add_crate("foo1", &V1, REBUILD_PRIORITY, None) .await?; build_queue - .add_crate("foo2", "0.1.0", REBUILD_PRIORITY, None) + .add_crate("foo2", &V1, REBUILD_PRIORITY, None) .await?; env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .builds(vec![ FakeBuild::default().rustc_version("rustc 1.84.0-nightly (e7c0d2750 2020-10-15)"), ]) @@ -856,8 +876,8 @@ mod tests { let queue = env.async_build_queue(); - queue.add_crate("some_crate", "0.1.1", 0, None).await?; - queue.add_crate("some_crate", "0.1.1", 9, None).await?; + queue.add_crate("some_crate", &V1, 0, None).await?; + queue.add_crate("some_crate", &V1, 9, None).await?; let queued_crates = queue.queued_crates().await?; assert_eq!(queued_crates.len(), 1); @@ -878,14 +898,15 @@ mod tests { sqlx::query!( " INSERT INTO queue (name, version, priority, attempt, last_attempt ) - VALUES ('failed_crate', '0.1.1', 0, 99, NOW())", + VALUES ('failed_crate', $1, 0, 99, NOW())", + V1 as _ ) .execute(&mut *conn) .await?; assert_eq!(queue.pending_count().await?, 0); - queue.add_crate("failed_crate", "0.1.1", 9, None).await?; + queue.add_crate("failed_crate", &V1, 9, None).await?; assert_eq!(queue.pending_count().await?, 1); @@ -894,7 +915,7 @@ mod tests { FROM queue WHERE name = $1 AND version = $2", "failed_crate", - "0.1.1", + V1 as _ ) .fetch_one(&mut *conn) .await?; @@ -911,17 +932,17 @@ mod tests { let queue = env.async_build_queue(); - queue.add_crate("dummy", "0.1.1", 0, None).await?; + queue.add_crate("dummy", &V1, 0, None).await?; let mut conn = env.async_db().async_conn().await; - assert!(queue.has_build_queued("dummy", "0.1.1").await.unwrap()); + assert!(queue.has_build_queued("dummy", &V1).await.unwrap()); sqlx::query!("UPDATE queue SET attempt = 6") .execute(&mut *conn) .await .unwrap(); - assert!(!queue.has_build_queued("dummy", "0.1.1").await.unwrap()); + assert!(!queue.has_build_queued("dummy", &V1).await.unwrap()); Ok(()) } @@ -939,7 +960,7 @@ mod tests { let queue = env.build_queue(); - queue.add_crate("krate", "1.0.0", 0, None)?; + queue.add_crate("krate", &V1, 0, None)?; // first let it fail queue.process_next_crate(|krate| { @@ -989,15 +1010,15 @@ mod tests { let queue = env.build_queue(); let test_crates = [ - ("low-priority", "1.0.0", 1000), - ("high-priority-foo", "1.0.0", -1000), - ("medium-priority", "1.0.0", -10), - ("high-priority-bar", "1.0.0", -1000), - ("standard-priority", "1.0.0", 0), - ("high-priority-baz", "1.0.0", -1000), + ("low-priority", 1000), + ("high-priority-foo", -1000), + ("medium-priority", -10), + ("high-priority-bar", -1000), + ("standard-priority", 0), + ("high-priority-baz", -1000), ]; for krate in &test_crates { - queue.add_crate(krate.0, krate.1, krate.2, None)?; + queue.add_crate(krate.0, &V1, krate.1, None)?; } let assert_next = |name| -> Result<()> { @@ -1079,8 +1100,8 @@ mod tests { let queue = env.build_queue(); - queue.add_crate("will_succeed", "1.0.0", -1, None)?; - queue.add_crate("will_fail", "1.0.0", 0, None)?; + queue.add_crate("will_succeed", &V1, -1, None)?; + queue.add_crate("will_fail", &V1, 0, None)?; let fetch_invalidations = || { env.runtime() @@ -1130,9 +1151,9 @@ mod tests { let queue = env.build_queue(); assert_eq!(queue.pending_count()?, 0); - queue.add_crate("foo", "1.0.0", 0, None)?; + queue.add_crate("foo", &V1, 0, None)?; assert_eq!(queue.pending_count()?, 1); - queue.add_crate("bar", "1.0.0", 0, None)?; + queue.add_crate("bar", &V1, 0, None)?; assert_eq!(queue.pending_count()?, 2); queue.process_next_crate(|krate| { @@ -1153,11 +1174,11 @@ mod tests { let queue = env.build_queue(); assert_eq!(queue.prioritized_count()?, 0); - queue.add_crate("foo", "1.0.0", 0, None)?; + queue.add_crate("foo", &V1, 0, None)?; assert_eq!(queue.prioritized_count()?, 1); - queue.add_crate("bar", "1.0.0", -100, None)?; + queue.add_crate("bar", &V1, -100, None)?; assert_eq!(queue.prioritized_count()?, 2); - queue.add_crate("baz", "1.0.0", 100, None)?; + queue.add_crate("baz", &V1, 100, None)?; assert_eq!(queue.prioritized_count()?, 2); queue.process_next_crate(|krate| { @@ -1177,9 +1198,9 @@ mod tests { assert!(queue.pending_count_by_priority()?.is_empty()); - queue.add_crate("one", "1.0.0", 1, None)?; - queue.add_crate("two", "2.0.0", 2, None)?; - queue.add_crate("two_more", "2.0.0", 2, None)?; + queue.add_crate("one", &V1, 1, None)?; + queue.add_crate("two", &V2, 2, None)?; + queue.add_crate("two_more", &V2, 2, None)?; assert_eq!( queue.pending_count_by_priority()?, @@ -1208,9 +1229,9 @@ mod tests { let queue = env.build_queue(); assert_eq!(queue.failed_count()?, 0); - queue.add_crate("foo", "1.0.0", -100, None)?; + queue.add_crate("foo", &V1, -100, None)?; assert_eq!(queue.failed_count()?, 0); - queue.add_crate("bar", "1.0.0", 0, None)?; + queue.add_crate("bar", &V1, 0, None)?; for _ in 0..MAX_ATTEMPTS { assert_eq!(queue.failed_count()?, 0); @@ -1247,9 +1268,9 @@ mod tests { let queue = env.build_queue(); assert_eq!(queue.failed_count()?, 0); - queue.add_crate("foo", "1.0.0", -100, None)?; + queue.add_crate("foo", &V1, -100, None)?; assert_eq!(queue.failed_count()?, 0); - queue.add_crate("bar", "1.0.0", 0, None)?; + queue.add_crate("bar", &V1, 0, None)?; for _ in 0..MAX_ATTEMPTS { assert_eq!(queue.failed_count()?, 0); @@ -1275,25 +1296,21 @@ mod tests { let queue = env.build_queue(); - let test_crates = [ - ("bar", "1.0.0", 0), - ("foo", "1.0.0", -10), - ("baz", "1.0.0", 10), - ]; + let test_crates = [("bar", 0), ("foo", -10), ("baz", 10)]; for krate in &test_crates { - queue.add_crate(krate.0, krate.1, krate.2, None)?; + queue.add_crate(krate.0, &V1, krate.1, None)?; } assert_eq!( vec![ - ("foo", "1.0.0", -10), - ("bar", "1.0.0", 0), - ("baz", "1.0.0", 10), + ("foo".into(), V1, -10), + ("bar".into(), V1, 0), + ("baz".into(), V1, 10), ], queue .queued_crates()? - .iter() - .map(|c| (c.name.as_str(), c.version.as_str(), c.priority)) + .into_iter() + .map(|c| (c.name.clone(), c.version, c.priority)) .collect::>() ); @@ -1364,7 +1381,7 @@ mod tests { let name: String = "krate".repeat(100); - queue.add_crate(&name, "0.0.1", 0, None)?; + queue.add_crate(&name, &V1, 0, None)?; queue.process_next_crate(|krate| { assert_eq!(name, krate.name); @@ -1380,12 +1397,16 @@ mod tests { let queue = env.build_queue(); - let version: String = "version".repeat(100); + let long_version = Version::parse(&format!( + "1.2.3-{}+{}", + "prerelease".repeat(100), + "build".repeat(100) + ))?; - queue.add_crate("krate", &version, 0, None)?; + queue.add_crate("krate", &long_version, 0, None)?; queue.process_next_crate(|krate| { - assert_eq!(version, krate.version); + assert_eq!(long_version, krate.version); Ok(BuildPackageSummary::default()) })?; diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 8b5c0b4c7..a3f21627e 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -1,15 +1,17 @@ use crate::{ - db::types::{BuildStatus, Feature}, + db::types::{BuildStatus, Feature, version::Version}, docbuilder::DocCoverage, error::Result, registry_api::{CrateData, CrateOwner, ReleaseData}, storage::CompressionAlgorithm, - utils::{MetadataPackage, rustc_version::parse_rustc_date}, + utils::{cargo_metadata::PackageExt as _, rustc_version::parse_rustc_date}, web::crate_details::{latest_release, releases_for_crate}, }; use anyhow::{Context, anyhow}; +use cargo_metadata::DependencyKind; use derive_more::Display; use futures_util::stream::TryStreamExt; +use semver::VersionReq; use serde::Serialize; use serde_json::Value; use slug::slugify; @@ -45,7 +47,7 @@ pub(crate) async fn finish_release( conn: &mut sqlx::PgConnection, crate_id: CrateId, release_id: ReleaseId, - metadata_pkg: &MetadataPackage, + metadata_pkg: &cargo_metadata::Package, source_dir: &Path, default_target: &str, source_files: Value, @@ -351,7 +353,7 @@ pub(crate) async fn initialize_crate(conn: &mut sqlx::PgConnection, name: &str) pub(crate) async fn initialize_release( conn: &mut sqlx::PgConnection, crate_id: CrateId, - version: &str, + version: &Version, ) -> Result { let release_id = sqlx::query_scalar!( r#"INSERT INTO releases (crate_id, version, archive_storage) @@ -361,7 +363,7 @@ pub(crate) async fn initialize_release( version = EXCLUDED.version RETURNING id as "id: ReleaseId" "#, crate_id.0, - version + version as _, ) .fetch_one(&mut *conn) .await?; @@ -393,24 +395,22 @@ pub(crate) async fn initialize_build( Ok(build_id) } -/// Convert dependencies into Vec<(String, String, String, bool)> -fn convert_dependencies(pkg: &MetadataPackage) -> Vec<(String, String, String, bool)> { +fn convert_dependencies( + pkg: &cargo_metadata::Package, +) -> Vec<(String, VersionReq, DependencyKind, bool)> { pkg.dependencies .iter() .map(|dependency| { let name = dependency.name.clone(); let version = dependency.req.clone(); - let kind = dependency - .kind - .clone() - .unwrap_or_else(|| "normal".to_string()); + let kind = dependency.kind; (name, version, kind, dependency.optional) }) .collect() } /// Reads features and converts them to Vec with default being first -fn get_features(pkg: &MetadataPackage) -> Vec { +fn get_features(pkg: &cargo_metadata::Package) -> Vec { let mut features = Vec::with_capacity(pkg.features.len()); if let Some(subfeatures) = pkg.features.get("default") { features.push(Feature::new("default".into(), subfeatures.clone())); @@ -425,8 +425,8 @@ fn get_features(pkg: &MetadataPackage) -> Vec { } /// Reads readme if there is any read defined in Cargo.toml of a Package -fn get_readme(pkg: &MetadataPackage, source_dir: &Path) -> Result> { - let readme_path = source_dir.join(pkg.readme.as_deref().unwrap_or("README.md")); +fn get_readme(pkg: &cargo_metadata::Package, source_dir: &Path) -> Result> { + let readme_path = source_dir.join(pkg.readme.as_deref().unwrap_or("README.md".into())); if !readme_path.exists() { return Ok(None); @@ -446,8 +446,8 @@ fn get_readme(pkg: &MetadataPackage, source_dir: &Path) -> Result } } -fn get_rustdoc(pkg: &MetadataPackage, source_dir: &Path) -> Result> { - if let Some(src_path) = &pkg.targets.first().and_then(|t| t.src_path.as_ref()) { +fn get_rustdoc(pkg: &cargo_metadata::Package, source_dir: &Path) -> Result> { + if let Some(src_path) = &pkg.targets.first().map(|t| &t.src_path) { let src_path = Path::new(src_path); if src_path.is_absolute() { read_rust_doc(src_path) @@ -495,7 +495,7 @@ fn read_rust_doc(file_path: &Path) -> Result> { /// Adds keywords into database async fn add_keywords_into_database( conn: &mut sqlx::PgConnection, - pkg: &MetadataPackage, + pkg: &cargo_metadata::Package, release_id: ReleaseId, ) -> Result<()> { let wanted_keywords: HashMap = pkg @@ -647,7 +647,7 @@ mod test { use super::*; use crate::registry_api::OwnerKind; use crate::test::*; - use crate::utils::CargoMetadata; + use crate::utils::cargo_metadata::{MetadataExt as _, load_cargo_metadata_from_host_path}; use chrono::NaiveDate; use std::slice; use test_case::test_case; @@ -657,7 +657,7 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; - let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; + let release_id = initialize_release(&mut conn, crate_id, &V0_1).await?; let build_id = initialize_build(&mut conn, release_id).await?; update_build_with_error(&mut conn, build_id, Some("error message")).await?; @@ -691,7 +691,7 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; - let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; + let release_id = initialize_release(&mut conn, crate_id, &V0_1).await?; let build_id = initialize_build(&mut conn, release_id).await?; finish_build( @@ -740,7 +740,7 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; - let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; + let release_id = initialize_release(&mut conn, crate_id, &V0_1).await?; let build_id = initialize_build(&mut conn, release_id).await?; finish_build( @@ -785,7 +785,7 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; - let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; + let release_id = initialize_release(&mut conn, crate_id, &V0_1).await?; let build_id = initialize_build(&mut conn, release_id).await?; finish_build( @@ -832,7 +832,7 @@ mod test { .fake_release() .await .name("dummy") - .version("0.13.0") + .version(V0_1) .keywords(vec!["kw 1".into(), "kw 2".into()]) .create() .await?; @@ -875,7 +875,7 @@ mod test { env.fake_release() .await .name("dummy") - .version("0.13.0") + .version(V0_1) .keywords(vec!["kw 3".into(), "kw 4".into()]) .create() .await?; @@ -884,7 +884,7 @@ mod test { env.fake_release() .await .name("dummy") - .version("0.13.0") + .version(V0_1) .keywords(vec!["kw 3".into(), "kw 4".into()]) .create() .await?; @@ -899,7 +899,7 @@ mod test { env.fake_release() .await .name("dummy") - .version("0.13.0") + .version(V1) .keywords(vec!["kw 3".into(), "kw 4".into()]) .create() .await?; @@ -908,7 +908,7 @@ mod test { .fake_release() .await .name("dummy") - .version("0.13.0") + .version(V1) .keywords(vec!["kw 1".into(), "kw 2".into()]) .create() .await?; @@ -1193,7 +1193,7 @@ mod test { "#; std::fs::write(dir.path().join("Cargo.toml"), [base, extra].concat())?; - let metadata = CargoMetadata::load_from_host_path(dir.path())?; + let metadata = load_cargo_metadata_from_host_path(dir.path())?; let features = super::get_features(metadata.root()); assert_eq!(features, expected.as_ref()); @@ -1229,22 +1229,21 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let name = "krate"; - let version = "0.1.0"; let crate_id = initialize_crate(&mut conn, name).await?; - let release_id = initialize_release(&mut conn, crate_id, version).await?; + let release_id = initialize_release(&mut conn, crate_id, &V1).await?; let id = sqlx::query_scalar!( r#"SELECT id as "id: ReleaseId" FROM releases WHERE crate_id = $1 and version = $2"#, crate_id.0, - version + V1 as _, ) .fetch_one(&mut *conn) .await?; assert_eq!(release_id, id); - let same_release_id = initialize_release(&mut conn, crate_id, version).await?; + let same_release_id = initialize_release(&mut conn, crate_id, &V1).await?; assert_eq!(release_id, same_release_id); Ok(()) @@ -1256,9 +1255,8 @@ mod test { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; let name = "krate"; - let version = "0.1.0"; let crate_id = initialize_crate(&mut conn, name).await?; - let release_id = initialize_release(&mut conn, crate_id, version).await?; + let release_id = initialize_release(&mut conn, crate_id, &V1).await?; let build_id = initialize_build(&mut conn, release_id).await?; @@ -1302,13 +1300,23 @@ mod test { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, "krate").await?; - let version: String = "version".repeat(100); + let version = Version::parse(&format!( + "1.2.3-{}+{}", + "prerelease".repeat(100), + "build".repeat(100) + ))?; let release_id = initialize_release(&mut conn, crate_id, &version).await?; - let db_version = - sqlx::query_scalar!("SELECT version FROM releases WHERE id = $1", release_id.0) - .fetch_one(&mut *conn) - .await?; + let db_version = sqlx::query_scalar!( + r#" + SELECT + version as "version: Version" + FROM releases + WHERE id = $1"#, + release_id.0 + ) + .fetch_one(&mut *conn) + .await?; assert_eq!(db_version, version); diff --git a/src/db/delete.rs b/src/db/delete.rs index 11ba12019..94c896e2d 100644 --- a/src/db/delete.rs +++ b/src/db/delete.rs @@ -1,5 +1,6 @@ use crate::{ Config, + db::types::version::Version, error::Result, storage::{AsyncStorage, rustdoc_archive_path, source_archive_path}, }; @@ -65,7 +66,7 @@ pub async fn delete_version( storage: &AsyncStorage, config: &Config, name: &str, - version: &str, + version: &Version, ) -> Result<()> { let is_library = delete_version_from_database(conn, name, version).await?; let paths = if is_library { @@ -131,7 +132,7 @@ const METADATA: &[(&str, &str)] = &[ async fn delete_version_from_database( conn: &mut sqlx::PgConnection, name: &str, - version: &str, + version: &Version, ) -> Result { let crate_id = get_id(conn, name).await?; let mut transaction = conn.begin().await?; @@ -143,7 +144,7 @@ async fn delete_version_from_database( let is_library: bool = sqlx::query_scalar!( "DELETE FROM releases WHERE crate_id = $1 AND version = $2 RETURNING is_library", crate_id.0, - version, + version as _, ) .fetch_one(&mut *transaction) .await? @@ -223,7 +224,7 @@ mod tests { use crate::db::ReleaseId; use crate::registry_api::{CrateOwner, OwnerKind}; use crate::storage::{CompressionAlgorithm, rustdoc_json_path}; - use crate::test::{async_wrapper, fake_release_that_failed_before_build}; + use crate::test::{V1, V2, async_wrapper, fake_release_that_failed_before_build}; use test_case::test_case; async fn crate_exists(conn: &mut sqlx::PgConnection, name: &str) -> Result { @@ -246,7 +247,7 @@ mod tests { env.fake_release() .await .name("Some_Package") - .version("1.0.0") + .version(V1) .create() .await?; @@ -268,7 +269,7 @@ mod tests { .fake_release() .await .name("package-1") - .version("1.0.0") + .version(V1) .archive_storage(archive_storage) .create() .await?; @@ -276,7 +277,7 @@ mod tests { .fake_release() .await .name("package-1") - .version("2.0.0") + .version(V2) .archive_storage(archive_storage) .create() .await?; @@ -284,6 +285,7 @@ mod tests { .fake_release() .await .name("package-2") + .version(V1) .archive_storage(archive_storage) .create() .await?; @@ -293,11 +295,7 @@ mod tests { assert!(release_exists(&mut conn, pkg1_v1_id).await?); assert!(release_exists(&mut conn, pkg1_v2_id).await?); assert!(release_exists(&mut conn, pkg2_id).await?); - for (pkg, version) in &[ - ("package-1", "1.0.0"), - ("package-1", "2.0.0"), - ("package-2", "1.0.0"), - ] { + for (pkg, version) in &[("package-1", V1), ("package-1", V2), ("package-2", V1)] { assert!( env.async_storage() .rustdoc_file_exists( @@ -324,7 +322,7 @@ mod tests { env.async_storage() .rustdoc_file_exists( "package-2", - "1.0.0", + &V1, None, "package-2/index.html", archive_storage @@ -336,12 +334,12 @@ mod tests { if archive_storage { assert!( !env.async_storage() - .exists(&rustdoc_archive_path("package-1", "1.0.0")) + .exists(&rustdoc_archive_path("package-1", &V1)) .await? ); assert!( !env.async_storage() - .exists(&rustdoc_archive_path("package-1", "2.0.0")) + .exists(&rustdoc_archive_path("package-1", &V2)) .await? ); } else { @@ -349,7 +347,7 @@ mod tests { !env.async_storage() .rustdoc_file_exists( "package-1", - "1.0.0", + &V1, None, "package-1/index.html", archive_storage @@ -360,7 +358,7 @@ mod tests { !env.async_storage() .rustdoc_file_exists( "package-1", - "2.0.0", + &V2, None, "package-1/index.html", archive_storage @@ -394,7 +392,7 @@ mod tests { .collect()) } - async fn json_exists(storage: &AsyncStorage, version: &str) -> Result { + async fn json_exists(storage: &AsyncStorage, version: &Version) -> Result { storage .exists(&rustdoc_json_path( "a", @@ -411,7 +409,7 @@ mod tests { .fake_release() .await .name("a") - .version("1.0.0") + .version(V1) .archive_storage(archive_storage) .add_owner(CrateOwner { login: "malicious actor".into(), @@ -423,10 +421,10 @@ mod tests { assert!(release_exists(&mut conn, v1).await?); assert!( env.async_storage() - .rustdoc_file_exists("a", "1.0.0", None, "a/index.html", archive_storage) + .rustdoc_file_exists("a", &V1, None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(env.async_storage(), "1.0.0").await?); + assert!(json_exists(env.async_storage(), &V1).await?); let crate_id = sqlx::query_scalar!( r#"SELECT crate_id as "crate_id: CrateId" FROM releases WHERE id = $1"#, v1.0 @@ -442,7 +440,7 @@ mod tests { .fake_release() .await .name("a") - .version("2.0.0") + .version(V2) .archive_storage(archive_storage) .add_owner(CrateOwner { login: "Peter Rabbit".into(), @@ -454,21 +452,21 @@ mod tests { assert!(release_exists(&mut conn, v2).await?); assert!( env.async_storage() - .rustdoc_file_exists("a", "2.0.0", None, "a/index.html", archive_storage) + .rustdoc_file_exists("a", &V2, None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(env.async_storage(), "2.0.0").await?); + assert!(json_exists(env.async_storage(), &V2).await?); assert_eq!( owners(&mut conn, crate_id).await?, vec!["Peter Rabbit".to_string()] ); - delete_version(&mut conn, env.async_storage(), env.config(), "a", "1.0.0").await?; + delete_version(&mut conn, env.async_storage(), env.config(), "a", &V1).await?; assert!(!release_exists(&mut conn, v1).await?); if archive_storage { // for archive storage the archive and index files // need to be cleaned up. - let rustdoc_archive = rustdoc_archive_path("a", "1.0.0"); + let rustdoc_archive = rustdoc_archive_path("a", &V1); assert!(!env.async_storage().exists(&rustdoc_archive).await?); // local and remote index are gone too @@ -483,19 +481,19 @@ mod tests { } else { assert!( !env.async_storage() - .rustdoc_file_exists("a", "1.0.0", None, "a/index.html", archive_storage) + .rustdoc_file_exists("a", &V1, None, "a/index.html", archive_storage) .await? ); } - assert!(!json_exists(env.async_storage(), "1.0.0").await?); + assert!(!json_exists(env.async_storage(), &V1,).await?); assert!(release_exists(&mut conn, v2).await?); assert!( env.async_storage() - .rustdoc_file_exists("a", "2.0.0", None, "a/index.html", archive_storage) + .rustdoc_file_exists("a", &V2, None, "a/index.html", archive_storage) .await? ); - assert!(json_exists(env.async_storage(), "2.0.0").await?); + assert!(json_exists(env.async_storage(), &V2).await?); assert_eq!( owners(&mut conn, crate_id).await?, vec!["Peter Rabbit".to_string()] @@ -517,10 +515,9 @@ mod tests { let mut conn = db.async_conn().await; let (release_id, _) = - fake_release_that_failed_before_build(&mut conn, "a", "1.0.0", "some-error") - .await?; + fake_release_that_failed_before_build(&mut conn, "a", V1, "some-error").await?; - delete_version(&mut conn, env.async_storage(), env.config(), "a", "1.0.0").await?; + delete_version(&mut conn, env.async_storage(), env.config(), "a", &V1).await?; assert!(!release_exists(&mut conn, release_id).await?); @@ -535,8 +532,7 @@ mod tests { let mut conn = db.async_conn().await; let (release_id, _) = - fake_release_that_failed_before_build(&mut conn, "a", "1.0.0", "some-error") - .await?; + fake_release_that_failed_before_build(&mut conn, "a", V1, "some-error").await?; delete_crate(&mut conn, env.async_storage(), env.config(), "a").await?; diff --git a/src/db/mod.rs b/src/db/mod.rs index 4c903952b..16c9bf2a5 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -24,7 +24,7 @@ pub(crate) mod file; pub(crate) mod mimes; mod overrides; mod pool; -pub(crate) mod types; +pub mod types; static MIGRATOR: Migrator = sqlx::migrate!(); diff --git a/src/db/types.rs b/src/db/types/mod.rs similarity index 98% rename from src/db/types.rs rename to src/db/types/mod.rs index b3728ef1e..8d6b10f98 100644 --- a/src/db/types.rs +++ b/src/db/types/mod.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Serialize}; +pub mod version; + #[derive(Debug, Clone, PartialEq, Eq, Serialize, sqlx::Type)] #[sqlx(type_name = "feature")] pub struct Feature { diff --git a/src/db/types/version.rs b/src/db/types/version.rs new file mode 100644 index 000000000..6824c0ac2 --- /dev/null +++ b/src/db/types/version.rs @@ -0,0 +1,100 @@ +#[allow(clippy::disallowed_types)] +mod version_impl { + use crate::error::Result; + use derive_more::{Deref, Display, From, Into}; + use serde_with::{DeserializeFromStr, SerializeDisplay}; + use sqlx::{ + Postgres, + encode::IsNull, + error::BoxDynError, + postgres::{PgArgumentBuffer, PgTypeInfo, PgValueRef}, + prelude::*, + }; + use std::{io::Write, str::FromStr}; + + /// NewType around semver::Version to be able to use it with sqlx. + /// + /// Represented as string in the database. + #[derive( + Clone, + Debug, + Deref, + DeserializeFromStr, + Display, + Eq, + From, + Hash, + Into, + PartialEq, + SerializeDisplay, + )] + pub struct Version(pub semver::Version); + + impl Version { + pub const fn new(major: u64, minor: u64, patch: u64) -> Self { + Self(semver::Version::new(major, minor, patch)) + } + + pub fn parse(text: &str) -> Result { + Version::from_str(text) + } + } + + impl Type for Version { + fn type_info() -> PgTypeInfo { + >::type_info() + } + + fn compatible(ty: &PgTypeInfo) -> bool { + >::compatible(ty) + } + } + + impl<'q> Encode<'q, Postgres> for Version { + fn encode_by_ref(&self, buf: &mut PgArgumentBuffer) -> Result { + write!(**buf, "{}", self.0)?; + Ok(IsNull::No) + } + } + + impl<'r> Decode<'r, Postgres> for Version { + fn decode(value: PgValueRef<'r>) -> Result { + let s: &str = Decode::::decode(value)?; + Ok(Self(s.parse()?)) + } + } + + impl FromStr for Version { + type Err = semver::Error; + + fn from_str(s: &str) -> Result { + Ok(Version(semver::Version::from_str(s)?)) + } + } + + impl TryFrom<&str> for Version { + type Error = semver::Error; + + fn try_from(value: &str) -> Result { + Ok(Version(semver::Version::from_str(value)?)) + } + } + + impl TryFrom<&String> for Version { + type Error = semver::Error; + + fn try_from(value: &String) -> Result { + Ok(Version(semver::Version::from_str(value)?)) + } + } + + impl TryFrom for Version { + type Error = semver::Error; + + fn try_from(value: String) -> Result { + Ok(Version(semver::Version::from_str(&value)?)) + } + } +} + +pub use version_impl::Version; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 5f0242e58..0b96bf477 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -1,49 +1,53 @@ -use crate::RUSTDOC_STATIC_STORAGE_PREFIX; -use crate::db::{ - BuildId, - file::{add_path_into_database, file_list_to_json}, +use crate::{ + AsyncStorage, Config, Context, InstanceMetrics, RUSTDOC_STATIC_STORAGE_PREFIX, RegistryApi, + Storage, + db::{ + BuildId, CrateId, Pool, ReleaseId, add_doc_coverage, add_path_into_remote_archive, + blacklist::is_blacklisted, + file::{add_path_into_database, file_list_to_json}, + finish_build, finish_release, initialize_build, initialize_crate, initialize_release, + types::{BuildStatus, version::Version}, + update_build_with_error, update_crate_data_in_database, + }, + docbuilder::Limits, + error::Result, + repositories::RepositoryStatsUpdater, + storage::{ + CompressionAlgorithm, RustdocJsonFormatVersion, compress, get_file_list, + rustdoc_archive_path, rustdoc_json_path, source_archive_path, + }, + utils::{ + ConfigName, + cargo_metadata::{MetadataExt as _, PackageExt as _, load_cargo_metadata_from_rustwide}, + copy_dir_all, get_config, parse_rustc_version, report_error, set_config, + }, }; -use crate::db::{CrateId, ReleaseId}; -use crate::db::{ - Pool, add_doc_coverage, add_path_into_remote_archive, finish_build, finish_release, - initialize_build, initialize_crate, initialize_release, types::BuildStatus, - update_build_with_error, update_crate_data_in_database, -}; -use crate::docbuilder::Limits; -use crate::error::Result; -use crate::repositories::RepositoryStatsUpdater; -use crate::storage::{ - CompressionAlgorithm, RustdocJsonFormatVersion, compress, get_file_list, rustdoc_archive_path, - rustdoc_json_path, source_archive_path, -}; -use crate::utils::{ - CargoMetadata, ConfigName, copy_dir_all, get_config, parse_rustc_version, report_error, - set_config, -}; -use crate::{AsyncStorage, Config, Context, InstanceMetrics, RegistryApi, Storage}; -use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage}; use anyhow::{Context as _, Error, anyhow, bail}; use docsrs_metadata::{BuildTargets, DEFAULT_TARGETS, HOST_TARGET, Metadata}; use itertools::Itertools as _; use regex::Regex; -use rustwide::cmd::{Command, CommandError, SandboxBuilder, SandboxImage}; -use rustwide::logging::{self, LogStorage}; -use rustwide::toolchain::ToolchainError; -use rustwide::{AlternativeRegistry, Build, Crate, Toolchain, Workspace, WorkspaceBuilder}; +use rustwide::{ + AlternativeRegistry, Build, Crate, Toolchain, Workspace, WorkspaceBuilder, + cmd::{Command, CommandError, SandboxBuilder, SandboxImage}, + logging::{self, LogStorage}, + toolchain::ToolchainError, +}; use serde::Deserialize; -use std::collections::{HashMap, HashSet}; -use std::fs::{self, File}; -use std::io::BufReader; -use std::path::Path; -use std::sync::Arc; -use std::time::Instant; +use std::{ + collections::{HashMap, HashSet}, + fs::{self, File}, + io::BufReader, + path::Path, + sync::Arc, + time::Instant, +}; use tokio::runtime; use tracing::{debug, error, info, info_span, instrument, warn}; const USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)"; const COMPONENTS: &[&str] = &["llvm-tools-preview", "rustc-dev", "rustfmt"]; const DUMMY_CRATE_NAME: &str = "empty-library"; -const DUMMY_CRATE_VERSION: &str = "1.0.0"; +const DUMMY_CRATE_VERSION: Version = Version::new(1, 0, 0); pub const RUSTDOC_JSON_COMPRESSION_ALGORITHMS: &[CompressionAlgorithm] = &[CompressionAlgorithm::Zstd, CompressionAlgorithm::Gzip]; @@ -314,7 +318,7 @@ impl RustwideBuilder { .build_dir(&format!("essential-files-{parsed_rustc_version}")); // This is an empty library crate that is supposed to always build. - let krate = Crate::crates_io(DUMMY_CRATE_NAME, DUMMY_CRATE_VERSION); + let krate = Crate::crates_io(DUMMY_CRATE_NAME, &DUMMY_CRATE_VERSION.to_string()); krate.fetch(&self.workspace)?; build_dir @@ -325,7 +329,7 @@ impl RustwideBuilder { let res = self.execute_build( BuildId(0), DUMMY_CRATE_NAME, - DUMMY_CRATE_VERSION, + &DUMMY_CRATE_VERSION, HOST_TARGET, true, build, @@ -376,14 +380,14 @@ impl RustwideBuilder { } pub fn build_local_package(&mut self, path: &Path) -> Result { - let metadata = CargoMetadata::load_from_rustwide(&self.workspace, &self.toolchain, path) + let metadata = load_cargo_metadata_from_rustwide(&self.workspace, &self.toolchain, path) .map_err(|err| { err.context(format!("failed to load local package {}", path.display())) })?; let package = metadata.root(); self.build_package( &package.name, - &package.version, + &package.version(), PackageKind::Local(path), false, ) @@ -393,7 +397,7 @@ impl RustwideBuilder { pub fn build_package( &mut self, name: &str, - version: &str, + version: &Version, kind: PackageKind<'_>, collect_metrics: bool, ) -> Result { @@ -439,7 +443,7 @@ impl RustwideBuilder { fn build_package_inner( &mut self, name: &str, - version: &str, + version: &Version, kind: PackageKind<'_>, crate_id: CrateId, release_id: ReleaseId, @@ -500,11 +504,12 @@ impl RustwideBuilder { let krate = { let _span = info_span!("krate.fetch").entered(); + let version = version.to_string(); let krate = match kind { PackageKind::Local(path) => Crate::local(path), - PackageKind::CratesIo => Crate::crates_io(name, version), + PackageKind::CratesIo => Crate::crates_io(name, &version), PackageKind::Registry(registry) => { - Crate::registry(AlternativeRegistry::new(registry), name, version) + Crate::registry(AlternativeRegistry::new(registry), name, &version) } }; krate.fetch(&self.workspace)?; @@ -784,7 +789,7 @@ impl RustwideBuilder { &self, build_id: BuildId, name: &str, - version: &str, + version: &Version, target: &str, build: &Build, limits: &Limits, @@ -829,7 +834,7 @@ impl RustwideBuilder { &self, build_id: BuildId, name: &str, - version: &str, + version: &Version, target: &str, is_default_target: bool, build: &Build, @@ -988,7 +993,7 @@ impl RustwideBuilder { &self, build_id: BuildId, name: &str, - version: &str, + version: &Version, target: &str, is_default_target: bool, build: &Build, @@ -997,7 +1002,7 @@ impl RustwideBuilder { create_essential_files: bool, collect_metrics: bool, ) -> Result { - let cargo_metadata = CargoMetadata::load_from_rustwide( + let cargo_metadata = load_cargo_metadata_from_rustwide( &self.workspace, &self.toolchain, &build.host_source_dir(), @@ -1228,7 +1233,7 @@ impl RustwideBuilder { copy_dir_all(source, dest).map_err(Into::into) } - fn get_repo(&self, metadata: &MetadataPackage) -> Result> { + fn get_repo(&self, metadata: &cargo_metadata::Package) -> Result> { self.runtime .block_on(self.repository_stats_updater.load_repository(metadata)) } @@ -1237,7 +1242,7 @@ impl RustwideBuilder { struct FullBuildResult { result: BuildResult, target: String, - cargo_metadata: CargoMetadata, + cargo_metadata: cargo_metadata::Metadata, doc_coverage: Option, build_log: String, } @@ -1282,10 +1287,12 @@ impl Default for BuildPackageSummary { #[cfg(test)] mod tests { use super::*; - use crate::db::types::Feature; - use crate::registry_api::ReleaseData; - use crate::storage::{CompressionAlgorithm, compression}; - use crate::test::{AxumRouterTestExt, TestEnvironment}; + use crate::{ + db::types::Feature, + registry_api::ReleaseData, + storage::{CompressionAlgorithm, compression}, + test::{AxumRouterTestExt, TestEnvironment, dummy_metadata_package}, + }; use pretty_assertions::assert_eq; use std::{io, iter}; use test_case::test_case; @@ -1293,7 +1300,7 @@ mod tests { fn get_features( env: &TestEnvironment, name: &str, - version: &str, + version: &Version, ) -> Result>, sqlx::Error> { env.runtime().block_on(async { let mut conn = env.async_db().async_conn().await; @@ -1304,14 +1311,14 @@ mod tests { INNER JOIN crates ON crates.id = releases.crate_id WHERE crates.name = $1 AND releases.version = $2"#, name, - version, + version as _, ) .fetch_one(&mut *conn) .await }) } - fn remove_cache_files(env: &TestEnvironment, crate_: &str, version: &str) -> Result<()> { + fn remove_cache_files(env: &TestEnvironment, crate_: &str, version: &Version) -> Result<()> { let paths = [ format!("cache/index.crates.io-6f17d22bba15001f/{crate_}-{version}.crate"), format!("src/index.crates.io-6f17d22bba15001f/{crate_}-{version}"), @@ -1361,7 +1368,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); @@ -1390,7 +1397,7 @@ mod tests { c.name = $1 AND r.version = $2"#, crate_, - version, + version as _, ) .fetch_one(&mut *conn) .await @@ -1424,11 +1431,11 @@ mod tests { assert!(!storage.exists(&old_source_file)?); // doc archive exists - let doc_archive = rustdoc_archive_path(crate_, version); + let doc_archive = rustdoc_archive_path(crate_, &version); assert!(storage.exists(&doc_archive)?, "{}", doc_archive); // source archive exists - let source_archive = source_archive_path(crate_, version); + let source_archive = source_archive_path(crate_, &version); assert!(storage.exists(&source_archive)?, "{}", source_archive); // default target was built and is accessible @@ -1478,7 +1485,7 @@ mod tests { // check if rustdoc json files exist for all targets let path = rustdoc_json_path( crate_, - version, + &version, target, RustdocJsonFormatVersion::Latest, Some(*alg), @@ -1548,7 +1555,7 @@ mod tests { builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, true)? + .build_package(crate_, &version, PackageKind::CratesIo, true)? .successful ); @@ -1571,7 +1578,7 @@ mod tests { // some binary crate let crate_ = "heater"; - let version = "0.2.3"; + let version = Version::new(0, 2, 3); let storage = env.storage(); let old_rustdoc_file = format!("rustdoc/{crate_}/{version}/some_doc_file"); @@ -1583,7 +1590,7 @@ mod tests { builder.update_toolchain()?; assert!( !builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); @@ -1602,7 +1609,7 @@ mod tests { c.name = $1 AND r.version = $2", crate_, - version + version as _ ) .fetch_one(&mut *conn) .await @@ -1612,11 +1619,11 @@ mod tests { assert_eq!(row.is_library, Some(false)); // doc archive exists - let doc_archive = rustdoc_archive_path(crate_, version); + let doc_archive = rustdoc_archive_path(crate_, &version); assert!(!storage.exists(&doc_archive)?); // source archive exists - let source_archive = source_archive_path(crate_, version); + let source_archive = source_archive_path(crate_, &version); assert!(storage.exists(&source_archive)?); // old rustdoc & source files still exist @@ -1634,13 +1641,13 @@ mod tests { // rand 0.8.5 fails to build with recent nightly versions // https://github.com/rust-lang/docs.rs/issues/26750 let crate_ = "rand"; - let version = "0.8.5"; + let version = Version::new(0, 8, 5); // create a successful release & build in the database let release_id = env.runtime().block_on(async { let mut conn = env.async_db().async_conn().await; let crate_id = initialize_crate(&mut conn, crate_).await?; - let release_id = initialize_release(&mut conn, crate_id, version).await?; + let release_id = initialize_release(&mut conn, crate_id, &version).await?; let build_id = initialize_build(&mut conn, release_id).await?; finish_build( &mut conn, @@ -1656,7 +1663,7 @@ mod tests { &mut conn, crate_id, release_id, - &MetadataPackage::default(), + &dummy_metadata_package().build().unwrap(), Path::new("/unknown/"), "x86_64-unknown-linux-gnu", serde_json::Value::Array(vec![]), @@ -1700,7 +1707,7 @@ mod tests { assert!( // not successful build !builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); @@ -1708,29 +1715,29 @@ mod tests { Ok(()) } - #[test_case("scsys-macros", "0.2.6")] - #[test_case("scsys-derive", "0.2.6")] - #[test_case("thiserror-impl", "1.0.26")] + #[test_case("scsys-macros", Version::new(0, 2, 6))] + #[test_case("scsys-derive", Version::new(0, 2, 6))] + #[test_case("thiserror-impl", Version::new(1, 0, 26))] #[ignore] - fn test_proc_macro(crate_: &str, version: &str) -> Result<()> { + fn test_proc_macro(crate_: &str, version: Version) -> Result<()> { let env = TestEnvironment::new_with_runtime()?; let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); let storage = env.storage(); // doc archive exists - let doc_archive = rustdoc_archive_path(crate_, version); + let doc_archive = rustdoc_archive_path(crate_, &version); assert!(storage.exists(&doc_archive)?); // source archive exists - let source_archive = source_archive_path(crate_, version); + let source_archive = source_archive_path(crate_, &version); assert!(storage.exists(&source_archive)?); Ok(()) @@ -1742,7 +1749,7 @@ mod tests { let env = TestEnvironment::new_with_runtime()?; let crate_ = "windows-win"; - let version = "2.4.1"; + let version = Version::new(2, 4, 1); let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; if builder.toolchain.as_ci().is_some() { @@ -1750,18 +1757,18 @@ mod tests { } assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); let storage = env.storage(); // doc archive exists - let doc_archive = rustdoc_archive_path(crate_, version); + let doc_archive = rustdoc_archive_path(crate_, &version); assert!(storage.exists(&doc_archive)?, "{}", doc_archive); // source archive exists - let source_archive = source_archive_path(crate_, version); + let source_archive = source_archive_path(crate_, &version); assert!(storage.exists(&source_archive)?, "{}", source_archive); let target = "x86_64-unknown-linux-gnu"; @@ -1793,7 +1800,7 @@ mod tests { )?; // if the corrected dependency of the crate was already downloaded we need to remove it - remove_cache_files(&env, "rand_core", "0.5.1")?; + remove_cache_files(&env, "rand_core", &Version::new(0, 5, 1))?; // Specific setup required: // * crate has a binary so that it is published with a lockfile @@ -1823,7 +1830,7 @@ mod tests { )?; // if the corrected dependency of the crate was already downloaded we need to remove it - remove_cache_files(&env, "value-bag-sval2", "1.4.1")?; + remove_cache_files(&env, "value-bag-sval2", &Version::new(1, 4, 1))?; // Similar to above, this crate fails to build with the published // lockfile, but generating a new working lockfile requires @@ -1846,12 +1853,12 @@ mod tests { let env = TestEnvironment::new_with_runtime()?; let crate_ = "proc-macro2"; - let version = "1.0.95"; + let version = Version::new(1, 0, 95); let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); Ok(()) @@ -1867,19 +1874,19 @@ mod tests { // Will succeed in the crate fetch step, so sources are // added. Will fail when we try to build. let crate_ = "simconnect-sys"; - let version = "0.23.1"; + let version = Version::new(0, 23, 1); let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; // `Result` is `Ok`, but the build-result is `false` assert!( !builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); // source archive exists - let source_archive = source_archive_path(crate_, version); + let source_archive = source_archive_path(crate_, &version); assert!( env.storage().exists(&source_archive)?, "archive doesnt exist: {source_archive}" @@ -1896,12 +1903,12 @@ mod tests { // https://github.com/rust-lang/docs.rs/issues/2491 // package without Cargo.toml, so fails directly in the fetch stage. let crate_ = "emheap"; - let version = "0.1.0"; + let version = Version::new(0, 1, 0); let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; // `Result` is `Ok`, but the build-result is `false` - let summary = builder.build_package(crate_, version, PackageKind::CratesIo, false)?; + let summary = builder.build_package(crate_, &version, PackageKind::CratesIo, false)?; assert!(!summary.successful); assert!(summary.should_reattempt); @@ -1920,7 +1927,7 @@ mod tests { INNER JOIN builds as b on b.rid = r.id WHERE c.name = $1 and r.version = $2"#, crate_, - version, + version as _, ) .fetch_one(&mut *conn) .await @@ -1940,17 +1947,17 @@ mod tests { let env = TestEnvironment::new_with_runtime()?; let crate_ = "serde"; - let version = "1.0.152"; + let version = Version::new(1, 0, 152); let mut builder = RustwideBuilder::init(&env.context).unwrap(); builder.update_toolchain()?; assert!( builder - .build_package(crate_, version, PackageKind::CratesIo, false)? + .build_package(crate_, &version, PackageKind::CratesIo, false)? .successful ); assert!( - get_features(&env, crate_, version)? + get_features(&env, crate_, &version)? .unwrap() .iter() .any(|f| f.name == "serde_derive") @@ -1973,7 +1980,7 @@ mod tests { ); assert_eq!( - get_features(&env, "optional-dep", "0.0.1")? + get_features(&env, "optional-dep", &Version::new(0, 0, 1))? .unwrap() .iter() .map(|f| f.name.to_owned()) @@ -2065,7 +2072,7 @@ mod tests { builder .build_package( DUMMY_CRATE_NAME, - DUMMY_CRATE_VERSION, + &DUMMY_CRATE_VERSION, PackageKind::CratesIo, false )? diff --git a/src/registry_api.rs b/src/registry_api.rs index 74fe7f6dc..0631d7809 100644 --- a/src/registry_api.rs +++ b/src/registry_api.rs @@ -1,8 +1,7 @@ -use crate::{error::Result, utils::retry_async}; +use crate::{db::types::version::Version, error::Result, utils::retry_async}; use anyhow::{Context, anyhow, bail}; use chrono::{DateTime, Utc}; use reqwest::header::{ACCEPT, HeaderValue, USER_AGENT}; -use semver::Version; use serde::{Deserialize, Serialize}; use std::fmt; use tracing::instrument; @@ -119,7 +118,11 @@ impl RegistryApi { } #[instrument(skip(self))] - pub(crate) async fn get_release_data(&self, name: &str, version: &str) -> Result { + pub(crate) async fn get_release_data( + &self, + name: &str, + version: &Version, + ) -> Result { let (release_time, yanked, downloads) = self .get_release_time_yanked_downloads(name, version) .await @@ -136,7 +139,7 @@ impl RegistryApi { async fn get_release_time_yanked_downloads( &self, name: &str, - version: &str, + version: &Version, ) -> Result<(DateTime, bool, i32)> { let url = { let mut url = self.api_base.clone(); @@ -177,11 +180,10 @@ impl RegistryApi { .json() .await?; - let version = Version::parse(version)?; let version = response .versions .into_iter() - .find(|data| data.num == version) + .find(|data| data.num == *version) .with_context(|| anyhow!("Could not find version in response"))?; Ok((version.created_at, version.yanked, version.downloads)) diff --git a/src/repositories/updater.rs b/src/repositories/updater.rs index 43d29acd0..2bc3c5db6 100644 --- a/src/repositories/updater.rs +++ b/src/repositories/updater.rs @@ -1,6 +1,5 @@ use crate::error::Result; use crate::repositories::{GitHub, GitLab, RateLimitReached}; -use crate::utils::MetadataPackage; use crate::{Config, db::Pool}; use async_trait::async_trait; use chrono::{DateTime, Utc}; @@ -81,7 +80,10 @@ impl RepositoryStatsUpdater { Self { updaters, pool } } - pub(crate) async fn load_repository(&self, metadata: &MetadataPackage) -> Result> { + pub(crate) async fn load_repository( + &self, + metadata: &cargo_metadata::Package, + ) -> Result> { let url = match &metadata.repository { Some(url) => url, None => { diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 6925dfc58..5e4178825 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -4,15 +4,18 @@ mod database; mod s3; pub use self::compression::{CompressionAlgorithm, CompressionAlgorithms, compress, decompress}; -use self::compression::{wrap_reader_for_decompression, wrap_writer_for_compression}; -use self::database::DatabaseBackend; -use self::s3::S3Backend; +use self::{ + compression::{wrap_reader_for_decompression, wrap_writer_for_compression}, + database::DatabaseBackend, + s3::S3Backend, +}; use crate::{ Config, InstanceMetrics, db::{ BuildId, Pool, file::{FileEntry, detect_mime}, mimes, + types::version::Version, }, error::Result, utils::spawn_blocking, @@ -244,7 +247,7 @@ impl AsyncStorage { pub(crate) async fn stream_rustdoc_file( &self, name: &str, - version: &str, + version: &Version, latest_build_id: Option, path: &str, archive_storage: bool, @@ -264,7 +267,7 @@ impl AsyncStorage { pub(crate) async fn fetch_source_file( &self, name: &str, - version: &str, + version: &Version, latest_build_id: Option, path: &str, archive_storage: bool, @@ -287,7 +290,7 @@ impl AsyncStorage { pub(crate) async fn rustdoc_file_exists( &self, name: &str, - version: &str, + version: &Version, latest_build_id: Option, path: &str, archive_storage: bool, @@ -781,7 +784,7 @@ impl Storage { pub(crate) fn fetch_source_file( &self, name: &str, - version: &str, + version: &Version, latest_build_id: Option, path: &str, archive_storage: bool, @@ -798,7 +801,7 @@ impl Storage { pub(crate) fn rustdoc_file_exists( &self, name: &str, - version: &str, + version: &Version, latest_build_id: Option, path: &str, archive_storage: bool, @@ -959,7 +962,7 @@ impl std::fmt::Debug for Storage { } } -pub(crate) fn rustdoc_archive_path(name: &str, version: &str) -> String { +pub(crate) fn rustdoc_archive_path(name: &str, version: &Version) -> String { format!("rustdoc/{name}/{version}.zip") } @@ -984,7 +987,7 @@ impl FromStr for RustdocJsonFormatVersion { pub(crate) fn rustdoc_json_path( name: &str, - version: &str, + version: &Version, target: &str, format_version: RustdocJsonFormatVersion, compression_algorithm: Option, @@ -1001,7 +1004,7 @@ pub(crate) fn rustdoc_json_path( path } -pub(crate) fn source_archive_path(name: &str, version: &str) -> String { +pub(crate) fn source_archive_path(name: &str, version: &Version) -> String { format!("sources/{name}/{version}.zip") } diff --git a/src/test/fakes.rs b/src/test/fakes.rs index 109407dfd..feb45f39b 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -1,37 +1,49 @@ use super::TestDatabase; - -use crate::db::file::{FileEntry, file_list_to_json}; -use crate::db::types::BuildStatus; -use crate::db::{ - BuildId, ReleaseId, initialize_build, initialize_crate, initialize_release, update_build_status, -}; -use crate::docbuilder::{DocCoverage, RUSTDOC_JSON_COMPRESSION_ALGORITHMS}; -use crate::error::Result; -use crate::registry_api::{CrateData, CrateOwner, ReleaseData}; -use crate::storage::{ - AsyncStorage, CompressionAlgorithm, RustdocJsonFormatVersion, compress, rustdoc_archive_path, - rustdoc_json_path, source_archive_path, +use crate::{ + db::{ + BuildId, ReleaseId, + file::{FileEntry, file_list_to_json}, + initialize_build, initialize_crate, initialize_release, + types::{BuildStatus, version::Version}, + update_build_status, + }, + docbuilder::{DocCoverage, RUSTDOC_JSON_COMPRESSION_ALGORITHMS}, + error::Result, + registry_api::{CrateData, CrateOwner, ReleaseData}, + storage::{ + AsyncStorage, CompressionAlgorithm, RustdocJsonFormatVersion, compress, + rustdoc_archive_path, rustdoc_json_path, source_archive_path, + }, + utils::cargo_metadata::PackageExt as _, }; -use crate::utils::{Dependency, MetadataPackage, Target}; use anyhow::{Context, bail}; use base64::{Engine, engine::general_purpose::STANDARD as b64}; +use cargo_metadata::{PackageId, PackageName}; use chrono::{DateTime, Utc}; -use std::collections::HashMap; -use std::iter; -use std::sync::Arc; +use semver::VersionReq; +use std::{ + collections::{BTreeMap, HashMap}, + fmt, iter, + sync::Arc, +}; use tracing::debug; /// Create a fake release in the database that failed before the build. /// This is a temporary small factory function only until we refactored the /// `FakeRelease` and `FakeBuild` factories to be more flexible. -pub(crate) async fn fake_release_that_failed_before_build( +pub(crate) async fn fake_release_that_failed_before_build( conn: &mut sqlx::PgConnection, name: &str, - version: &str, + version: V, errors: &str, -) -> Result<(ReleaseId, BuildId)> { +) -> Result<(ReleaseId, BuildId)> +where + V: TryInto, + V::Error: std::error::Error + Send + Sync + 'static, +{ + let version = version.try_into()?; let crate_id = initialize_crate(&mut *conn, name).await?; - let release_id = initialize_release(&mut *conn, crate_id, version).await?; + let release_id = initialize_release(&mut *conn, crate_id, &version).await?; let build_id = initialize_build(&mut *conn, release_id).await?; sqlx::query_scalar!( @@ -55,7 +67,7 @@ pub(crate) async fn fake_release_that_failed_before_build( pub(crate) struct FakeRelease<'a> { db: &'a TestDatabase, storage: Arc, - package: MetadataPackage, + package: cargo_metadata::Package, builds: Option>, /// name, content source_files: Vec<(&'a str, &'a [u8])>, @@ -92,35 +104,7 @@ impl<'a> FakeRelease<'a> { FakeRelease { db, storage, - package: MetadataPackage { - id: "fake-package-id".into(), - name: "fake-package".into(), - version: "1.0.0".into(), - license: Some("MIT".into()), - repository: Some("https://git.example.com".into()), - homepage: Some("https://www.example.com".into()), - description: Some("Fake package".into()), - documentation: Some("https://docs.example.com".into()), - dependencies: vec![Dependency { - name: "fake-dependency".into(), - req: "^1.0.0".into(), - kind: None, - rename: None, - optional: false, - }], - targets: vec![Target::dummy_lib("fake_package".into(), None)], - readme: None, - keywords: vec!["fake".into(), "package".into()], - features: [ - ("default".into(), vec!["feature1".into(), "feature3".into()]), - ("feature1".into(), Vec::new()), - ("feature2".into(), vec!["feature1".into()]), - ("feature3".into(), Vec::new()), - ] - .iter() - .cloned() - .collect::>>(), - }, + package: dummy_metadata_package().build().unwrap(), builds: None, source_files: Vec::new(), rustdoc_files: Vec::new(), @@ -147,7 +131,7 @@ impl<'a> FakeRelease<'a> { self } - pub(crate) fn add_dependency(mut self, dependency: Dependency) -> Self { + pub(crate) fn add_dependency(mut self, dependency: cargo_metadata::Dependency) -> Self { self.package.dependencies.push(dependency); self } @@ -158,14 +142,21 @@ impl<'a> FakeRelease<'a> { } pub(crate) fn name(mut self, new: &str) -> Self { - self.package.name = new.into(); - self.package.id = format!("{new}-id"); + self.package.name = PackageName::new(new.into()); + self.package.id = PackageId { + repr: format!("{new}-id"), + }; self.package.targets[0].name = new.into(); self } - pub(crate) fn version(mut self, new: &str) -> Self { - self.package.version = new.into(); + pub(crate) fn version(mut self, new: V) -> Self + where + V: TryInto, + V::Error: fmt::Debug, + { + let version: Version = new.try_into().expect("invalid version"); + self.package.version = version.into(); self } @@ -234,7 +225,7 @@ impl<'a> FakeRelease<'a> { pub(crate) fn target_source(mut self, path: &'a str) -> Self { if let Some(target) = self.package.targets.first_mut() { - target.src_path = Some(path.into()); + target.src_path = path.into(); } self } @@ -273,7 +264,11 @@ impl<'a> FakeRelease<'a> { pub(crate) fn add_platform>(mut self, platform: S) -> Self { let platform = platform.into(); let name = self.package.targets[0].name.clone(); - let target = Target::dummy_lib(name, Some(platform.clone())); + let target = dummy_metadata_target() + .name(name) + .src_path(platform.clone()) + .build() + .unwrap(); self.package.targets.push(target); self.doc_targets.push(platform); self @@ -303,7 +298,7 @@ impl<'a> FakeRelease<'a> { } } - pub(crate) fn features(mut self, features: HashMap>) -> Self { + pub(crate) fn features(mut self, features: BTreeMap>) -> Self { self.package.features = features; self } @@ -392,7 +387,7 @@ impl<'a> FakeRelease<'a> { kind: FileKind, source_directory: &Path, archive_storage: bool, - package: &MetadataPackage, + package: &cargo_metadata::Package, storage: &AsyncStorage, ) -> Result<(Vec, CompressionAlgorithm)> { debug!( @@ -402,12 +397,14 @@ impl<'a> FakeRelease<'a> { ); if archive_storage { let (archive, public) = match kind { - FileKind::Rustdoc => { - (rustdoc_archive_path(&package.name, &package.version), true) - } - FileKind::Sources => { - (source_archive_path(&package.name, &package.version), false) - } + FileKind::Rustdoc => ( + rustdoc_archive_path(&package.name, &package.version()), + true, + ), + FileKind::Sources => ( + source_archive_path(&package.name, &package.version()), + false, + ), }; debug!("store in archive: {:?}", archive); let (files_list, new_alg) = crate::db::add_path_into_remote_archive( @@ -442,7 +439,7 @@ impl<'a> FakeRelease<'a> { .iter() .any(|&(path, _)| path == "Cargo.toml") { - let MetadataPackage { name, version, .. } = &package; + let cargo_metadata::Package { name, version, .. } = &package; let content = format!( r#" [package] @@ -480,8 +477,8 @@ impl<'a> FakeRelease<'a> { debug!("added rustdoc files"); for target in &package.targets[1..] { - let platform = target.src_path.as_ref().unwrap(); - let platform_dir = rustdoc_path.join(platform); + let platform = target.src_path.to_path_buf(); + let platform_dir = rustdoc_path.join(&platform); fs::create_dir(&platform_dir)?; store_files_into(&rustdoc_files, &platform_dir)?; @@ -536,7 +533,7 @@ impl<'a> FakeRelease<'a> { .store_one_uncompressed( &rustdoc_json_path( &package.name, - &package.version, + &package.version(), target, format_version, Some(*alg), @@ -554,7 +551,7 @@ impl<'a> FakeRelease<'a> { // non-linux platforms. let mut async_conn = db.async_conn().await; let crate_id = initialize_crate(&mut async_conn, &package.name).await?; - let release_id = initialize_release(&mut async_conn, crate_id, &package.version).await?; + let release_id = initialize_release(&mut async_conn, crate_id, &package.version()).await?; crate::db::finish_release( &mut async_conn, crate_id, @@ -741,3 +738,76 @@ impl Default for FakeBuild { } } } + +pub(crate) fn dummy_metadata_dependency() -> cargo_metadata::DependencyBuilder { + cargo_metadata::DependencyBuilder::default() + .name("fake-dependency") + .source(None) + .req(VersionReq::parse("^1.0.0").unwrap()) + .kind(cargo_metadata::DependencyKind::Normal) + .optional(false) + .uses_default_features(true) + .features(vec![]) + .target(None) + .rename(None) + .registry(None) + .path(None) +} + +pub(crate) fn dummy_metadata_target() -> cargo_metadata::TargetBuilder { + cargo_metadata::TargetBuilder::default() + .name("fake_package") + .kind(vec![cargo_metadata::TargetKind::Lib]) + .crate_types(vec![cargo_metadata::CrateType::Lib]) + .required_features(vec![]) + .src_path("src/lib.rs") +} + +pub(crate) fn dummy_metadata_package() -> cargo_metadata::PackageBuilder { + // use ::new()? + cargo_metadata::PackageBuilder::default() + .id(cargo_metadata::PackageId { + repr: "fake-package-id".into(), + }) + .name( + "fake-package" + .parse::() + .unwrap(), + ) + .version(Version::new(1, 0, 0)) + .manifest_path("Cargo.toml") + .license(Some("MIT".to_string())) + .repository(Some("https://git.example.com".into())) + .homepage(Some("https://www.example.com".into())) + .description(Some("Fake package".into())) + .documentation(Some("https://docs.example.com".into())) + .dependencies(vec![dummy_metadata_dependency().build().unwrap()]) + .targets(vec![dummy_metadata_target().build().unwrap()]) + .keywords(vec!["fake".into(), "package".into()]) + .features( + [ + ("default".into(), vec!["feature1".into(), "feature3".into()]), + ("feature1".into(), Vec::new()), + ("feature2".into(), vec!["feature1".into()]), + ("feature3".into(), Vec::new()), + ] + .iter() + .cloned() + .collect::>>(), + ) +} + +#[test] +fn test_dummy_metadata_dependency_is_complete() { + dummy_metadata_dependency().build().unwrap(); +} + +#[test] +fn test_dummy_metadata_target_is_complete() { + dummy_metadata_target().build().unwrap(); +} + +#[test] +fn test_dummy_metadata_package_is_complete() { + dummy_metadata_package().build().unwrap(); +} diff --git a/src/test/mod.rs b/src/test/mod.rs index 7069c9eff..0b0754b25 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -1,13 +1,18 @@ mod fakes; -pub(crate) use self::fakes::{FakeBuild, fake_release_that_failed_before_build}; -use crate::cdn::CdnBackend; -use crate::config::ConfigBuilder; -use crate::db::{self, AsyncPoolClient, Pool}; -use crate::error::Result; -use crate::storage::{AsyncStorage, Storage, StorageKind}; -use crate::web::{build_axum_app, cache, page::TemplateData}; -use crate::{AsyncBuildQueue, BuildQueue, Config, Context, InstanceMetrics}; +pub(crate) use self::fakes::{ + FakeBuild, dummy_metadata_dependency, dummy_metadata_package, + fake_release_that_failed_before_build, +}; +use crate::{ + AsyncBuildQueue, BuildQueue, Config, Context, InstanceMetrics, + cdn::CdnBackend, + config::ConfigBuilder, + db::{self, AsyncPoolClient, Pool, types::version::Version}, + error::Result, + storage::{AsyncStorage, Storage, StorageKind}, + web::{build_axum_app, cache, page::TemplateData}, +}; use anyhow::Context as _; use axum::body::Bytes; use axum::{Router, body::Body, http::Request, response::Response as AxumResponse}; @@ -21,6 +26,12 @@ use tokio::{runtime, task::block_in_place}; use tower::ServiceExt; use tracing::error; +// some versions as constants for tests +pub(crate) const V0_1: Version = Version::new(0, 1, 0); +pub(crate) const V1: Version = Version::new(1, 0, 0); +pub(crate) const V2: Version = Version::new(2, 0, 0); +pub(crate) const V3: Version = Version::new(3, 0, 0); + pub(crate) fn async_wrapper(f: F) where F: FnOnce(Rc) -> Fut, diff --git a/src/utils/cargo_metadata.rs b/src/utils/cargo_metadata.rs index 66cd8e35b..58d98062f 100644 --- a/src/utils/cargo_metadata.rs +++ b/src/utils/cargo_metadata.rs @@ -1,87 +1,70 @@ -use crate::error::Result; -use anyhow::{Context, bail}; +use crate::{db::types::version::Version, error::Result}; +use anyhow::bail; +use cargo_metadata::{CrateType, Metadata, Target}; use rustwide::{Toolchain, Workspace, cmd::Command}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use std::path::Path; -pub(crate) struct CargoMetadata { - root: Package, +pub(crate) fn load_cargo_metadata_from_rustwide( + workspace: &Workspace, + toolchain: &Toolchain, + source_dir: &Path, +) -> Result { + let res = Command::new(workspace, toolchain.cargo()) + .args(&["metadata", "--format-version", "1"]) + .cd(source_dir) + .log_output(false) + .run_capture()?; + let [metadata] = res.stdout_lines() else { + bail!("invalid output returned by `cargo metadata`") + }; + + Ok(serde_json::from_str(metadata)?) } -impl CargoMetadata { - pub(crate) fn load_from_rustwide( - workspace: &Workspace, - toolchain: &Toolchain, - source_dir: &Path, - ) -> Result { - let res = Command::new(workspace, toolchain.cargo()) - .args(&["metadata", "--format-version", "1"]) - .cd(source_dir) - .log_output(false) - .run_capture()?; - let [metadata] = res.stdout_lines() else { - bail!("invalid output returned by `cargo metadata`") - }; - Self::load_from_metadata(metadata) - } - - #[cfg(test)] - pub(crate) fn load_from_host_path(source_dir: &Path) -> Result { - let res = std::process::Command::new("cargo") - .args(["metadata", "--format-version", "1", "--offline"]) - .current_dir(source_dir) - .output()?; - let status = res.status; - if !status.success() { - let stderr = std::str::from_utf8(&res.stderr).unwrap_or(""); - bail!("error returned by `cargo metadata`: {status}\n{stderr}") - } - Self::load_from_metadata(std::str::from_utf8(&res.stdout)?) +#[cfg(test)] +pub(crate) fn load_cargo_metadata_from_host_path(source_dir: &Path) -> Result { + let res = std::process::Command::new("cargo") + .args(["metadata", "--format-version", "1", "--offline"]) + .current_dir(source_dir) + .output()?; + let status = res.status; + if !status.success() { + let stderr = std::str::from_utf8(&res.stderr).unwrap_or(""); + bail!("error returned by `cargo metadata`: {status}\n{stderr}") } + Ok(serde_json::from_slice(&res.stdout)?) +} - pub(crate) fn load_from_metadata(metadata: &str) -> Result { - let metadata = serde_json::from_str::(metadata)?; - let root = metadata.resolve.root; - Ok(CargoMetadata { - root: metadata - .packages - .into_iter() - .find(|pkg| pkg.id == root) - .context("metadata.packages missing root package")?, - }) - } +pub(crate) trait MetadataExt { + fn root(&self) -> &cargo_metadata::Package; +} - pub(crate) fn root(&self) -> &Package { - &self.root +impl MetadataExt for cargo_metadata::Metadata { + fn root(&self) -> &cargo_metadata::Package { + self.root_package().as_ref().expect("missing root package") } } -#[derive(Debug, Deserialize, Serialize, Default)] -pub(crate) struct Package { - pub(crate) id: String, - pub(crate) name: String, - pub(crate) version: String, - pub(crate) license: Option, - pub(crate) repository: Option, - pub(crate) homepage: Option, - pub(crate) description: Option, - pub(crate) documentation: Option, - pub(crate) dependencies: Vec, - pub(crate) targets: Vec, - pub(crate) readme: Option, - pub(crate) keywords: Vec, - pub(crate) features: HashMap>, +pub(crate) trait PackageExt { + fn library_target(&self) -> Option<&Target>; + fn is_library(&self) -> bool; + fn normalize_package_name(&self, name: &str) -> String; + fn package_name(&self) -> String; + fn library_name(&self) -> Option; + fn version(&self) -> Version; } -impl Package { +impl PackageExt for cargo_metadata::Package { fn library_target(&self) -> Option<&Target> { - self.targets - .iter() - .find(|target| target.crate_types.iter().any(|kind| kind != "bin")) + self.targets.iter().find(|target| { + target + .crate_types + .iter() + .any(|kind| kind != &CrateType::Bin) + }) } - pub(crate) fn is_library(&self) -> bool { + fn is_library(&self) -> bool { self.library_target().is_some() } @@ -89,7 +72,7 @@ impl Package { name.replace('-', "_") } - pub(crate) fn package_name(&self) -> String { + fn package_name(&self) -> String { self.library_name().unwrap_or_else(|| { self.targets .first() @@ -98,80 +81,12 @@ impl Package { }) } - pub(crate) fn library_name(&self) -> Option { + fn library_name(&self) -> Option { self.library_target() .map(|target| self.normalize_package_name(&target.name)) } -} - -#[derive(Debug, Deserialize, Serialize)] -pub(crate) struct Target { - pub(crate) name: String, - #[cfg(not(test))] - crate_types: Vec, - #[cfg(test)] - pub(crate) crate_types: Vec, - pub(crate) src_path: Option, -} -impl Target { - #[cfg(test)] - pub(crate) fn dummy_lib(name: String, src_path: Option) -> Self { - Target { - name, - crate_types: vec!["lib".into()], - src_path, - } + fn version(&self) -> Version { + self.version.clone().into() } } - -#[derive(Debug, Deserialize, Serialize)] -pub(crate) struct Dependency { - pub(crate) name: String, - pub(crate) req: String, - pub(crate) kind: Option, - pub(crate) rename: Option, - pub(crate) optional: bool, -} - -impl Dependency { - #[cfg(test)] - pub fn new(name: String, req: String) -> Dependency { - Dependency { - name, - req, - kind: None, - rename: None, - optional: false, - } - } - - #[cfg(test)] - pub fn set_optional(mut self, optional: bool) -> Self { - self.optional = optional; - self - } -} - -#[derive(Deserialize, Serialize)] -struct DeserializedMetadata { - packages: Vec, - resolve: DeserializedResolve, -} - -#[derive(Deserialize, Serialize)] -struct DeserializedResolve { - root: String, - nodes: Vec, -} - -#[derive(Deserialize, Serialize)] -struct DeserializedResolveNode { - id: String, - deps: Vec, -} - -#[derive(Deserialize, Serialize)] -struct DeserializedResolveDep { - pkg: String, -} diff --git a/src/utils/consistency/data.rs b/src/utils/consistency/data.rs index 2c1e08d06..c38333f50 100644 --- a/src/utils/consistency/data.rs +++ b/src/utils/consistency/data.rs @@ -1,3 +1,5 @@ +use crate::db::types::version::Version; + #[derive(Clone, PartialEq, Debug)] pub(super) struct Crate { pub(super) name: String, @@ -10,6 +12,6 @@ pub(super) type Releases = Vec; #[derive(Clone, Debug, PartialEq)] pub(super) struct Release { - pub(super) version: String, + pub(super) version: Version, pub(super) yanked: Option, } diff --git a/src/utils/consistency/db.rs b/src/utils/consistency/db.rs index 42be527a8..c05c781e2 100644 --- a/src/utils/consistency/db.rs +++ b/src/utils/consistency/db.rs @@ -1,5 +1,5 @@ use super::data::{Crate, Crates, Release, Releases}; -use crate::Config; +use crate::{Config, db::types::version::Version}; use anyhow::Result; use itertools::Itertools; @@ -7,7 +7,7 @@ pub(super) async fn load(conn: &mut sqlx::PgConnection, config: &Config) -> Resu let rows = sqlx::query!( r#"SELECT name as "name!", - version as "version!", + version as "version!: Version", yanked FROM ( SELECT @@ -62,24 +62,24 @@ pub(super) async fn load(conn: &mut sqlx::PgConnection, config: &Config) -> Resu #[cfg(test)] mod tests { use super::*; - use crate::test::async_wrapper; + use crate::test::{V1, V2, V3, async_wrapper}; #[test] fn test_load() { async_wrapper(|env| async move { env.async_build_queue() - .add_crate("queued", "0.0.1", 0, None) + .add_crate("queued", &V1, 0, None) .await?; env.fake_release() .await .name("krate") - .version("0.0.2") + .version(V2) .create() .await?; env.fake_release() .await .name("krate") - .version("0.0.3") + .version(V3) .yanked(true) .create() .await?; @@ -94,11 +94,11 @@ mod tests { name: "krate".into(), releases: vec![ Release { - version: "0.0.2".into(), + version: V2, yanked: Some(false), }, Release { - version: "0.0.3".into(), + version: V3, yanked: Some(true), } ] @@ -106,7 +106,7 @@ mod tests { Crate { name: "queued".into(), releases: vec![Release { - version: "0.0.1".into(), + version: V1, yanked: None, }] }, diff --git a/src/utils/consistency/diff.rs b/src/utils/consistency/diff.rs index 78e929b60..efaa7fc9e 100644 --- a/src/utils/consistency/diff.rs +++ b/src/utils/consistency/diff.rs @@ -1,18 +1,18 @@ -use std::fmt::Display; - use super::data::Crate; +use crate::db::types::version::Version; use itertools::{ EitherOrBoth::{Both, Left, Right}, Itertools, }; +use std::fmt::Display; #[derive(Debug, PartialEq)] pub(super) enum Difference { CrateNotInIndex(String), - CrateNotInDb(String, Vec), - ReleaseNotInIndex(String, String), - ReleaseNotInDb(String, String), - ReleaseYank(String, String, bool), + CrateNotInDb(String, Vec), + ReleaseNotInIndex(String, Version), + ReleaseNotInDb(String, Version), + ReleaseYank(String, Version, bool), } impl Display for Difference { @@ -103,6 +103,8 @@ where #[cfg(test)] mod tests { + use crate::test::{V2, V3}; + use super::super::data::Release; use super::*; use std::iter; @@ -131,11 +133,11 @@ mod tests { name: "krate".into(), releases: vec![ Release { - version: "0.0.2".into(), + version: V2, yanked: Some(false), }, Release { - version: "0.0.3".into(), + version: V3, yanked: Some(true), }, ], @@ -143,10 +145,7 @@ mod tests { assert_eq!( calculate_diff([].iter(), index_releases.iter()), - vec![Difference::CrateNotInDb( - "krate".into(), - vec!["0.0.2".into(), "0.0.3".into()] - )] + vec![Difference::CrateNotInDb("krate".into(), vec![V2, V3])] ); } @@ -156,11 +155,11 @@ mod tests { name: "krate".into(), releases: vec![ Release { - version: "0.0.2".into(), + version: V2, yanked: Some(true), }, Release { - version: "0.0.3".into(), + version: V3, yanked: Some(true), }, ], @@ -169,11 +168,11 @@ mod tests { name: "krate".into(), releases: vec![ Release { - version: "0.0.2".into(), + version: V2, yanked: Some(false), }, Release { - version: "0.0.3".into(), + version: V3, yanked: Some(true), }, ], @@ -181,11 +180,7 @@ mod tests { assert_eq!( calculate_diff(db_releases.iter(), index_releases.iter()), - vec![Difference::ReleaseYank( - "krate".into(), - "0.0.2".into(), - false, - )] + vec![Difference::ReleaseYank("krate".into(), V2, false,)] ); } @@ -194,14 +189,14 @@ mod tests { let db_releases = [Crate { name: "krate".into(), releases: vec![Release { - version: "0.0.2".into(), + version: V2, yanked: None, }], }]; let index_releases = [Crate { name: "krate".into(), releases: vec![Release { - version: "0.0.2".into(), + version: V2, yanked: Some(false), }], }]; diff --git a/src/utils/consistency/index.rs b/src/utils/consistency/index.rs index a8a5ac9ff..5547e8d56 100644 --- a/src/utils/consistency/index.rs +++ b/src/utils/consistency/index.rs @@ -1,5 +1,5 @@ use super::data::{Crate, Crates, Release, Releases}; -use crate::Index; +use crate::{Index, db::types::version::Version}; use anyhow::Result; use rayon::iter::ParallelIterator; @@ -12,9 +12,15 @@ pub(super) fn load(index: &Index) -> Result { let mut releases: Releases = krate .versions() .iter() - .map(|version| Release { - version: version.version().into(), - yanked: Some(version.is_yanked()), + .filter_map(|version| { + version + .version() + .parse::() + .ok() + .map(|semversion| Release { + version: semversion, + yanked: Some(version.is_yanked()), + }) }) .collect(); releases.sort_by(|lhs, rhs| lhs.version.cmp(&rhs.version)); diff --git a/src/utils/consistency/mod.rs b/src/utils/consistency/mod.rs index acb90588d..4b54f3fca 100644 --- a/src/utils/consistency/mod.rs +++ b/src/utils/consistency/mod.rs @@ -159,7 +159,10 @@ where mod tests { use super::diff::Difference; use super::*; - use crate::test::{TestEnvironment, async_wrapper}; + use crate::{ + db::types::version::Version, + test::{TestEnvironment, V1, V2, async_wrapper}, + }; use sqlx::Row as _; async fn count(env: &TestEnvironment, sql: &str) -> Result { @@ -188,8 +191,8 @@ mod tests { env.fake_release() .await .name("krate") - .version("0.1.1") - .version("0.1.2") + .version(V1) + .version(V2) .create() .await?; @@ -221,20 +224,17 @@ mod tests { env.fake_release() .await .name("krate") - .version("0.1.1") + .version(V1) .create() .await?; env.fake_release() .await .name("krate") - .version("0.1.2") + .version(V2) .create() .await?; - let diff = [Difference::ReleaseNotInIndex( - "krate".into(), - "0.1.1".into(), - )]; + let diff = [Difference::ReleaseNotInIndex("krate".into(), V1)]; assert_eq!(count(&env, "SELECT count(*) FROM releases").await?, 2); @@ -245,8 +245,12 @@ mod tests { handle_diff(&env.context, diff.iter(), false).await?; assert_eq!( - single_row::(&env, "SELECT version FROM releases").await?, - vec!["0.1.2"] + single_row::( + &env, + r#"SELECT version as "version: Version" FROM releases"# + ) + .await?, + vec![V2] ); Ok(()) @@ -259,16 +263,12 @@ mod tests { env.fake_release() .await .name("krate") - .version("0.1.1") + .version(V1) .yanked(true) .create() .await?; - let diff = [Difference::ReleaseYank( - "krate".into(), - "0.1.1".into(), - false, - )]; + let diff = [Difference::ReleaseYank("krate".into(), V1, false)]; handle_diff(&env.context, diff.iter(), true).await?; @@ -291,7 +291,7 @@ mod tests { #[test] fn test_missing_release_in_db() { async_wrapper(|env| async move { - let diff = [Difference::ReleaseNotInDb("krate".into(), "0.1.1".into())]; + let diff = [Difference::ReleaseNotInDb("krate".into(), V1)]; handle_diff(&env.context, diff.iter(), true).await?; @@ -305,10 +305,10 @@ mod tests { build_queue .queued_crates() .await? - .iter() - .map(|c| (c.name.as_str(), c.version.as_str(), c.priority)) + .into_iter() + .map(|c| (c.name, V1, c.priority)) .collect::>(), - vec![("krate", "0.1.1", 15)] + vec![("krate".into(), V1, 15)] ); Ok(()) }) @@ -317,10 +317,7 @@ mod tests { #[test] fn test_missing_crate_in_db() { async_wrapper(|env| async move { - let diff = [Difference::CrateNotInDb( - "krate".into(), - vec!["0.1.1".into(), "0.1.2".into()], - )]; + let diff = [Difference::CrateNotInDb("krate".into(), vec![V1, V2])]; handle_diff(&env.context, diff.iter(), true).await?; @@ -334,10 +331,10 @@ mod tests { build_queue .queued_crates() .await? - .iter() - .map(|c| (c.name.as_str(), c.version.as_str(), c.priority)) + .into_iter() + .map(|c| (c.name, c.version, c.priority)) .collect::>(), - vec![("krate", "0.1.1", 15), ("krate", "0.1.2", 15)] + vec![("krate".into(), V1, 15), ("krate".into(), V2, 15)] ); Ok(()) }) diff --git a/src/utils/html.rs b/src/utils/html.rs index bae9d19c2..03e6d2b6a 100644 --- a/src/utils/html.rs +++ b/src/utils/html.rs @@ -212,14 +212,14 @@ where #[cfg(test)] mod test { - use crate::test::{AxumResponseTestExt, AxumRouterTestExt, async_wrapper}; + use crate::test::{AxumResponseTestExt, AxumRouterTestExt, V1, async_wrapper}; #[test] fn rewriting_only_injects_css_once() { async_wrapper(|env| async move { env.fake_release().await .name("testing") - .version("0.1.0") + .version(V1) // A somewhat representative rustdoc html file from 2016 .rustdoc_file_with("2016/index.html", br#" @@ -247,10 +247,18 @@ mod test { .create().await?; let web = env.web_app().await; - let output = web.get("/testing/0.1.0/2016/").await?.text().await?; + let output = web + .get(&format!("/testing/{V1}/2016/")) + .await? + .text() + .await?; assert_eq!(output.matches(r#"href="/-/static/vendored.css"#).count(), 1); - let output = web.get("/testing/0.1.0/2022/").await?.text().await?; + let output = web + .get(&format!("/testing/{V1}/2022/")) + .await? + .text() + .await?; assert_eq!(output.matches(r#"href="/-/static/vendored.css"#).count(), 1); Ok(()) diff --git a/src/utils/mod.rs b/src/utils/mod.rs index bbde45481..baa79d34d 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,6 +1,5 @@ //! Various utilities for docs.rs -pub(crate) use self::cargo_metadata::{CargoMetadata, Package as MetadataPackage}; pub(crate) use self::copy::copy_dir_all; pub use self::daemon::{start_daemon, watch_registry}; pub(crate) use self::html::rewrite_rustdoc_html_stream; @@ -11,10 +10,7 @@ pub use self::queue::{ pub use self::queue_builder::queue_builder; pub(crate) use self::rustc_version::{get_correct_docsrs_style_file, parse_rustc_version}; -#[cfg(test)] -pub(crate) use self::cargo_metadata::{Dependency, Target}; - -mod cargo_metadata; +pub(crate) mod cargo_metadata; pub mod consistency; mod copy; pub mod daemon; diff --git a/src/utils/version.rs b/src/utils/version.rs new file mode 100644 index 000000000..e69de29bb diff --git a/src/web/build_details.rs b/src/web/build_details.rs index 35c787156..37dbce46d 100644 --- a/src/web/build_details.rs +++ b/src/web/build_details.rs @@ -1,6 +1,9 @@ use crate::{ AsyncStorage, Config, - db::{BuildId, types::BuildStatus}, + db::{ + BuildId, + types::{BuildStatus, version::Version}, + }, impl_axum_webpage, web::{ MetaData, @@ -16,7 +19,6 @@ use askama::Template; use axum::{extract::Extension, response::IntoResponse}; use chrono::{DateTime, Utc}; use futures_util::TryStreamExt; -use semver::Version; use serde::Deserialize; use std::sync::Arc; @@ -161,7 +163,7 @@ pub(crate) async fn build_details_handler( #[cfg(test)] mod tests { use crate::test::{ - AxumResponseTestExt, AxumRouterTestExt, FakeBuild, async_wrapper, + AxumResponseTestExt, AxumRouterTestExt, FakeBuild, V1, async_wrapper, fake_release_that_failed_before_build, }; use kuchikiki::traits::TendrilSink; @@ -184,18 +186,14 @@ mod tests { fn test_partial_build_result() { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; - let (_, build_id) = fake_release_that_failed_before_build( - &mut conn, - "foo", - "0.1.0", - "some random error", - ) - .await?; + let (_, build_id) = + fake_release_that_failed_before_build(&mut conn, "foo", V1, "some random error") + .await?; let page = kuchikiki::parse_html().one( env.web_app() .await - .get(&format!("/crate/foo/0.1.0/builds/{build_id}")) + .get(&format!("/crate/foo/{V1}/builds/{build_id}")) .await? .error_for_status()? .text() @@ -215,13 +213,9 @@ mod tests { fn test_partial_build_result_plus_default_target_from_previous_build() { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; - let (release_id, build_id) = fake_release_that_failed_before_build( - &mut conn, - "foo", - "0.1.0", - "some random error", - ) - .await?; + let (release_id, build_id) = + fake_release_that_failed_before_build(&mut conn, "foo", V1, "some random error") + .await?; sqlx::query!( "UPDATE releases SET default_target = 'x86_64-unknown-linux-gnu' WHERE id = $1", @@ -233,7 +227,7 @@ mod tests { let page = kuchikiki::parse_html().one( env.web_app() .await - .get(&format!("/crate/foo/0.1.0/builds/{build_id}")) + .get(&format!("/crate/foo/{V1}/builds/{build_id}")) .await? .error_for_status()? .text() diff --git a/src/web/builds.rs b/src/web/builds.rs index 0980d332a..1877148a2 100644 --- a/src/web/builds.rs +++ b/src/web/builds.rs @@ -5,7 +5,10 @@ use super::{ }; use crate::{ AsyncBuildQueue, Config, - db::{BuildId, types::BuildStatus}, + db::{ + BuildId, + types::{BuildStatus, version::Version}, + }, docbuilder::Limits, impl_axum_webpage, web::{ @@ -26,7 +29,6 @@ use axum_extra::{ use chrono::{DateTime, Utc}; use constant_time_eq::constant_time_eq; use http::StatusCode; -use semver::Version; use std::sync::Arc; #[derive(Debug, Clone, PartialEq, Eq)] @@ -112,9 +114,7 @@ async fn build_trigger_check( return Err(AxumNope::VersionNotFound); } - let crate_version_is_in_queue = build_queue - .has_build_queued(name, &version.to_string()) - .await?; + let crate_version_is_in_queue = build_queue.has_build_queued(name, version).await?; if crate_version_is_in_queue { return Err(AxumNope::BadRequest(anyhow!( @@ -161,7 +161,7 @@ pub(crate) async fn build_trigger_rebuild_handler( build_queue .add_crate( &name, - &version.to_string(), + &version, TRIGGERED_REBUILD_PRIORITY, None, /* because crates.io is the only service that calls this endpoint */ ) @@ -205,8 +205,8 @@ mod tests { use crate::{ db::Overrides, test::{ - AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, async_wrapper, - fake_release_that_failed_before_build, + AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, V1, V2, + async_wrapper, fake_release_that_failed_before_build, }, web::cache::CachePolicy, }; @@ -220,12 +220,12 @@ mod tests { fn build_list_empty_build() { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; - fake_release_that_failed_before_build(&mut conn, "foo", "0.1.0", "some errors").await?; + fake_release_that_failed_before_build(&mut conn, "foo", V1, "some errors").await?; let response = env .web_app() .await - .get("/crate/foo/0.1.0/builds") + .get(&format!("/crate/foo/{V1}/builds")) .await? .error_for_status()?; response.assert_cache_control(CachePolicy::NoCaching, env.config()); @@ -351,7 +351,7 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .create() .await?; @@ -398,14 +398,14 @@ mod tests { let build_queue = env.async_build_queue(); assert_eq!(build_queue.pending_count().await?, 0); - assert!(!build_queue.has_build_queued("foo", "0.1.0").await?); + assert!(!build_queue.has_build_queued("foo", &V1).await?); { let app = env.web_app().await; let response = app .oneshot( Request::builder() - .uri("/crate/foo/0.1.0/rebuild") + .uri(format!("/crate/foo/{V1}/rebuild")) .method("POST") .header("Authorization", &format!("Bearer {correct_token}")) .body(Body::empty()) @@ -418,14 +418,14 @@ mod tests { } assert_eq!(build_queue.pending_count().await?, 1); - assert!(build_queue.has_build_queued("foo", "0.1.0").await?); + assert!(build_queue.has_build_queued("foo", &V1).await?); { let app = env.web_app().await; let response = app .oneshot( Request::builder() - .uri("/crate/foo/0.1.0/rebuild") + .uri(format!("/crate/foo/{V1}/rebuild")) .method("POST") .header("Authorization", &format!("Bearer {correct_token}")) .body(Body::empty()) @@ -438,13 +438,13 @@ mod tests { json, serde_json::json!({ "title": "Bad request", - "message": "crate foo 0.1.0 already queued for rebuild" + "message": format!("crate foo {V1} already queued for rebuild") }) ); } assert_eq!(build_queue.pending_count().await?, 1); - assert!(build_queue.has_build_queued("foo", "0.1.0").await?); + assert!(build_queue.has_build_queued("foo", &V1).await?); Ok(()) } @@ -455,12 +455,16 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .no_builds() .create() .await?; - let response = env.web_app().await.get("/crate/foo/0.1.0/builds").await?; + let response = env + .web_app() + .await + .get(&format!("/crate/foo/{V1}/builds")) + .await?; response.assert_cache_control(CachePolicy::NoCaching, env.config()); let page = kuchikiki::parse_html().one(response.text().await?); @@ -492,7 +496,7 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .create() .await?; @@ -507,7 +511,7 @@ mod tests { let page = kuchikiki::parse_html().one( env.web_app() .await - .get("/crate/foo/0.1.0/builds") + .get(&format!("/crate/foo/{V1}/builds")) .await? .text() .await?, @@ -540,7 +544,7 @@ mod tests { env.fake_release() .await .name("aquarelle") - .version("0.1.0") + .version(V1) .builds(vec![ FakeBuild::default() .rustc_version("rustc (blabla 2019-01-01)") @@ -552,7 +556,7 @@ mod tests { env.fake_release() .await .name("aquarelle") - .version("0.2.0") + .version(V2) .builds(vec![ FakeBuild::default() .rustc_version("rustc (blabla 2019-01-01)") @@ -587,7 +591,7 @@ mod tests { env.fake_release() .await .name("foo") - .version("0.1.0") + .version(V1) .builds(vec![ FakeBuild::default() .rustc_version("rustc (blabla 2019-01-01)") @@ -596,7 +600,11 @@ mod tests { .create() .await?; - let resp = env.web_app().await.get("/crate/foo/0.2.0/builds").await?; + let resp = env + .web_app() + .await + .get(&format!("/crate/foo/{V2}/builds")) + .await?; assert_eq!(resp.status(), StatusCode::NOT_FOUND); Ok(()) }); diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index d6e5692b9..419f5e92c 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -1,12 +1,14 @@ use super::{MetaData, match_version}; -use crate::db::{BuildId, ReleaseId}; -use crate::registry_api::OwnerKind; -use crate::utils::{get_correct_docsrs_style_file, report_error}; use crate::{ AsyncStorage, - db::{CrateId, types::BuildStatus}, + db::{ + BuildId, CrateId, ReleaseId, + types::{BuildStatus, version::Version}, + }, impl_axum_webpage, + registry_api::OwnerKind, storage::PathNotFoundError, + utils::get_correct_docsrs_style_file, web::{ MatchedRelease, ReqVersion, cache::CachePolicy, @@ -25,7 +27,6 @@ use axum::{ use chrono::{DateTime, Utc}; use futures_util::stream::TryStreamExt; use log::warn; -use semver::Version; use serde::Deserialize; use serde_json::Value; use std::sync::Arc; @@ -81,7 +82,7 @@ struct RepositoryMetadata { #[derive(Debug, Clone, Eq, PartialEq)] pub(crate) struct Release { pub id: ReleaseId, - pub version: semver::Version, + pub version: Version, #[allow(clippy::doc_overindented_list_items)] /// Aggregated build status of the release. /// * no builds -> build In progress @@ -297,7 +298,7 @@ impl CrateDetails { let manifest = match storage .fetch_source_file( &self.name, - &self.version.to_string(), + &self.version, self.latest_build_id, "Cargo.toml", self.archive_storage, @@ -326,7 +327,7 @@ impl CrateDetails { match storage .fetch_source_file( &self.name, - &self.version.to_string(), + &self.version, self.latest_build_id, path, self.archive_storage, @@ -378,7 +379,7 @@ pub(crate) async fn releases_for_crate( let mut releases: Vec = sqlx::query!( r#"SELECT releases.id as "id: ReleaseId", - releases.version, + releases.version as "version: Version", release_build_status.build_status as "build_status!: BuildStatus", releases.yanked, releases.is_library, @@ -393,22 +394,9 @@ pub(crate) async fn releases_for_crate( ) .fetch(&mut *conn) .try_filter_map(|row| async move { - let semversion = match semver::Version::parse(&row.version).with_context(|| { - format!( - "invalid semver in database for crate {crate_id}: {}", - row.version - ) - }) { - Ok(semver) => semver, - Err(err) => { - report_error(&err); - return Ok(None); - } - }; - Ok(Some(Release { id: row.id, - version: semversion, + version: row.version, build_status: row.build_status, yanked: row.yanked, is_library: row.is_library, @@ -798,7 +786,7 @@ pub(crate) async fn get_all_platforms( mod tests { use super::*; use crate::test::{ - AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestDatabase, TestEnvironment, + AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestDatabase, TestEnvironment, V1, async_wrapper, fake_release_that_failed_before_build, }; use crate::{db::update_build_status, registry_api::CrateOwner}; @@ -806,13 +794,15 @@ mod tests { use kuchikiki::traits::TendrilSink; use pretty_assertions::assert_eq; use reqwest::StatusCode; - use std::collections::HashMap; + use std::collections::BTreeMap; async fn release_build_status( conn: &mut sqlx::PgConnection, name: &str, version: &str, ) -> BuildStatus { + let version: Version = version.parse().expect("invalid version"); + let status = sqlx::query_scalar!( r#" SELECT build_status as "build_status!: BuildStatus" @@ -821,7 +811,7 @@ mod tests { INNER JOIN release_build_status ON releases.id = release_build_status.rid WHERE crates.name = $1 AND releases.version = $2"#, name, - version + version as _ ) .fetch_one(&mut *conn) .await @@ -837,12 +827,18 @@ mod tests { status } - async fn crate_details( + async fn crate_details( conn: &mut sqlx::PgConnection, name: &str, - version: &str, + version: V, req_version: Option, - ) -> CrateDetails { + ) -> CrateDetails + where + V: TryInto, + V::Error: std::error::Error + Send + Sync + 'static, + { + let version = version.try_into().expect("invalid version"); + let crate_id = sqlx::query_scalar!( r#"SELECT id as "id: CrateId" FROM crates WHERE name = $1"#, name @@ -853,16 +849,10 @@ mod tests { let releases = releases_for_crate(&mut *conn, crate_id).await.unwrap(); - CrateDetails::new( - &mut *conn, - name, - &Version::parse(version).unwrap(), - req_version, - releases, - ) - .await - .unwrap() - .unwrap() + CrateDetails::new(&mut *conn, name, &version, req_version, releases) + .await + .unwrap() + .unwrap() } #[fn_error_context::context( @@ -874,6 +864,7 @@ mod tests { version: &str, expected_last_successful_build: Option<&str>, ) -> Result<(), Error> { + let version = version.parse::()?; let mut conn = db.async_conn().await; let details = crate_details(&mut conn, package, version, None).await; @@ -1121,7 +1112,7 @@ mod tests { details.releases, vec![ Release { - version: semver::Version::parse("1.0.0")?, + version: Version::parse("1.0.0")?, build_status: BuildStatus::Success, yanked: Some(false), is_library: Some(true), @@ -1131,7 +1122,7 @@ mod tests { release_time: None, }, Release { - version: semver::Version::parse("0.12.0")?, + version: Version::parse("0.12.0")?, build_status: BuildStatus::Success, yanked: Some(false), is_library: Some(true), @@ -1141,7 +1132,7 @@ mod tests { release_time: None, }, Release { - version: semver::Version::parse("0.3.0")?, + version: Version::parse("0.3.0")?, build_status: BuildStatus::Failure, yanked: Some(false), is_library: Some(true), @@ -1151,7 +1142,7 @@ mod tests { release_time: None, }, Release { - version: semver::Version::parse("0.2.0")?, + version: Version::parse("0.2.0")?, build_status: BuildStatus::Success, yanked: Some(true), is_library: Some(true), @@ -1161,7 +1152,7 @@ mod tests { release_time: None, }, Release { - version: semver::Version::parse("0.2.0-alpha")?, + version: Version::parse("0.2.0-alpha")?, build_status: BuildStatus::Success, yanked: Some(false), is_library: Some(true), @@ -1171,7 +1162,7 @@ mod tests { release_time: None, }, Release { - version: semver::Version::parse("0.1.1")?, + version: Version::parse("0.1.1")?, build_status: BuildStatus::Success, yanked: Some(false), is_library: Some(true), @@ -1181,7 +1172,7 @@ mod tests { release_time: None, }, Release { - version: semver::Version::parse("0.1.0")?, + version: Version::parse("0.1.0")?, build_status: BuildStatus::Success, yanked: Some(false), is_library: Some(true), @@ -1191,7 +1182,7 @@ mod tests { release_time: None, }, Release { - version: semver::Version::parse("0.0.1")?, + version: Version::parse("0.0.1")?, build_status: BuildStatus::Failure, yanked: Some(false), is_library: Some(false), @@ -1263,10 +1254,10 @@ mod tests { let mut conn = db.async_conn().await; for version in &["0.0.1", "0.0.2", "0.0.3"] { - let details = crate_details(&mut conn, "foo", version, None).await; + let details = crate_details(&mut conn, "foo", *version, None).await; assert_eq!( details.latest_release().unwrap().version, - semver::Version::parse("0.0.3")? + Version::parse("0.0.3")? ); } @@ -1299,11 +1290,11 @@ mod tests { .await?; let mut conn = db.async_conn().await; - for version in &["0.0.1", "0.0.2", "0.0.3-pre.1"] { + for &version in &["0.0.1", "0.0.2", "0.0.3-pre.1"] { let details = crate_details(&mut conn, "foo", version, None).await; assert_eq!( details.latest_release().unwrap().version, - semver::Version::parse("0.0.2")? + Version::parse("0.0.2")? ); } @@ -1337,11 +1328,11 @@ mod tests { .await?; let mut conn = db.async_conn().await; - for version in &["0.0.1", "0.0.2", "0.0.3"] { + for &version in &["0.0.1", "0.0.2", "0.0.3"] { let details = crate_details(&mut conn, "foo", version, None).await; assert_eq!( details.latest_release().unwrap().version, - semver::Version::parse("0.0.2")? + Version::parse("0.0.2")? ); } @@ -1377,11 +1368,11 @@ mod tests { .await?; let mut conn = db.async_conn().await; - for version in &["0.0.1", "0.0.2", "0.0.3"] { + for &version in &["0.0.1", "0.0.2", "0.0.3"] { let details = crate_details(&mut conn, "foo", version, None).await; assert_eq!( details.latest_release().unwrap().version, - semver::Version::parse("0.0.3")? + Version::parse("0.0.3")? ); } @@ -1411,11 +1402,11 @@ mod tests { .await?; let mut conn = db.async_conn().await; - for version in &["0.0.1", "0.0.2"] { + for &version in &["0.0.1", "0.0.2"] { let details = crate_details(&mut conn, "foo", version, None).await; assert_eq!( details.latest_release().unwrap().version, - semver::Version::parse("0.0.1")? + Version::parse("0.0.1")? ); } @@ -1620,7 +1611,7 @@ mod tests { .await .name("library") .version("0.1.0") - .features(HashMap::new()) + .features(BTreeMap::new()) .create() .await?; @@ -1643,7 +1634,7 @@ mod tests { let features = [("_private".into(), Vec::new())] .iter() .cloned() - .collect::>>(); + .collect::>>(); env.fake_release() .await .name("library") @@ -1671,7 +1662,7 @@ mod tests { let features = [("feature1".into(), Vec::new())] .iter() .cloned() - .collect::>>(); + .collect::>>(); env.fake_release() .await .name("library") @@ -1707,7 +1698,7 @@ mod tests { ] .iter() .cloned() - .collect::>>(); + .collect::>>(); env.fake_release() .await .name("library") @@ -1807,12 +1798,12 @@ mod tests { fn test_minimal_failed_release_doesnt_error_features() { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; - fake_release_that_failed_before_build(&mut conn, "foo", "0.1.0", "some errors").await?; + fake_release_that_failed_before_build(&mut conn, "foo", V1, "some errors").await?; let text_content = env .web_app() .await - .get("/crate/foo/0.1.0/features") + .get(&format!("/crate/foo/{V1}/features")) .await? .error_for_status()? .text() @@ -1831,12 +1822,12 @@ mod tests { fn test_minimal_failed_release_doesnt_error() { async_wrapper(|env| async move { let mut conn = env.async_db().async_conn().await; - fake_release_that_failed_before_build(&mut conn, "foo", "0.1.0", "some errors").await?; + fake_release_that_failed_before_build(&mut conn, "foo", V1, "some errors").await?; let text_content = env .web_app() .await - .get("/crate/foo/0.1.0") + .get(&format!("/crate/foo/{V1}")) .await? .error_for_status()? .text() diff --git a/src/web/features.rs b/src/web/features.rs index 836ff31ba..238f2a8dd 100644 --- a/src/web/features.rs +++ b/src/web/features.rs @@ -379,7 +379,7 @@ mod tests { .await .name("foo") .version("0.2.1") - .features(HashMap::new()) + .features(BTreeMap::new()) .create() .await?; @@ -402,7 +402,7 @@ mod tests { .await .name("foo") .version("0.2.0") - .features(HashMap::new()) + .features(BTreeMap::new()) .create() .await?; @@ -421,7 +421,7 @@ mod tests { .await .name("foo") .version("0.1.0") - .features(HashMap::new()) + .features(BTreeMap::new()) .create() .await?; @@ -429,7 +429,7 @@ mod tests { .await .name("foo") .version("0.2.0") - .features(HashMap::new()) + .features(BTreeMap::new()) .create() .await?; @@ -452,7 +452,7 @@ mod tests { .await .name("foo") .version("0.1.0") - .features(HashMap::new()) + .features(BTreeMap::new()) .create() .await?; @@ -470,7 +470,7 @@ mod tests { .await .name("foo") .version("0.1.0") - .features(HashMap::new()) + .features(BTreeMap::new()) .create() .await?; @@ -502,7 +502,7 @@ mod tests { .await .name("foo") .version("0.1.0") - .features(features.into_iter().collect::>()) + .features(features.into_iter().collect::>()) .create() .await?; diff --git a/src/web/mod.rs b/src/web/mod.rs index 615cd32d8..b9e68bba9 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -5,7 +5,7 @@ pub mod page; use crate::db::CrateId; use crate::db::ReleaseId; -use crate::db::types::BuildStatus; +use crate::db::types::{BuildStatus, version::Version}; use crate::utils::get_correct_docsrs_style_file; use crate::utils::report_error; use crate::web::page::templates::{RenderBrands, RenderSolid, filters}; @@ -51,7 +51,7 @@ use chrono::{DateTime, Utc}; use error::AxumNope; use page::TemplateData; use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; -use semver::{Version, VersionReq}; +use semver::VersionReq; use sentry::integrations::tower as sentry_tower; use serde_with::{DeserializeFromStr, SerializeDisplay}; use std::{ @@ -771,6 +771,7 @@ mod test { use test_case::test_case; async fn release(version: &str, env: &TestEnvironment) -> ReleaseId { + let version = Version::parse(version).unwrap(); env.fake_release() .await .name("foo") diff --git a/src/web/releases.rs b/src/web/releases.rs index 92d7c4c2f..e4606b53f 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -3,7 +3,9 @@ use crate::{ AsyncBuildQueue, Config, InstanceMetrics, RegistryApi, build_queue::{QueuedCrate, REBUILD_PRIORITY}, - cdn, impl_axum_webpage, + cdn, + db::types::version::Version, + impl_axum_webpage, utils::report_error, web::{ ReqVersion, axum_parse_uri_with_params, axum_redirect, encode_url_path, @@ -11,6 +13,7 @@ use crate::{ extractors::{DbConnection, Path}, match_version, page::templates::{RenderBrands, RenderRegular, RenderSolid, filters}, + rustdoc::OfficialCrateDescription, }, }; use anyhow::{Context as _, Result, anyhow}; @@ -25,9 +28,11 @@ use futures_util::stream::TryStreamExt; use itertools::Itertools; use serde::{Deserialize, Serialize}; use sqlx::Row; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::str; -use std::sync::Arc; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + str, + sync::Arc, +}; use tracing::warn; use url::form_urlencoded; @@ -43,14 +48,13 @@ const RELEASES_IN_FEED: i64 = 150; #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct Release { pub(crate) name: String, - pub(crate) version: String, + pub(crate) version: Version, pub(crate) description: Option, pub(crate) target_name: Option, pub(crate) rustdoc_status: bool, pub(crate) build_time: Option>, pub(crate) stars: i32, pub(crate) has_unyanked_releases: Option, - pub(crate) href: Option<&'static str>, } #[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] @@ -80,8 +84,8 @@ pub(crate) async fn get_releases( }; let query = format!( - "SELECT crates.name, - releases.version, + r#"SELECT crates.name, + releases.version as "version: Version", releases.description, releases.target_name, releases.rustdoc_status, @@ -97,7 +101,7 @@ pub(crate) async fn get_releases( release_build_status.build_status != 'in_progress' ORDER BY {0} DESC - LIMIT $1 OFFSET $2", + LIMIT $1 OFFSET $2"#, ordering, if latest_only { "INNER JOIN releases ON crates.latest_version_id = releases.id" @@ -120,7 +124,6 @@ pub(crate) async fn get_releases( build_time: row.get(5), stars: row.get::, _>(6).unwrap_or(0), has_unyanked_releases: None, - href: None, }) .try_collect() .await?) @@ -129,6 +132,7 @@ pub(crate) async fn get_releases( #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum ReleaseStatus { Available(Release), + External(&'static OfficialCrateDescription), /// Only contains the crate name. NotAvailable(String), } @@ -139,20 +143,6 @@ struct SearchResult { pub next_page: Option, } -fn rust_lib_release(name: &str, description: &str, href: &'static str) -> ReleaseStatus { - ReleaseStatus::Available(Release { - name: name.to_string(), - version: String::new(), - description: Some(description.to_string()), - build_time: None, - target_name: None, - rustdoc_status: false, - stars: 0, - has_unyanked_releases: None, - href: Some(href), - }) -} - /// Get the search results for a crate search query /// /// This delegates to the crates.io search API. @@ -181,7 +171,7 @@ async fn get_search_results( let mut crates: HashMap = sqlx::query!( r#"SELECT crates.name, - releases.version, + releases.version as "version: Version", releases.description, release_build_status.last_build_time, releases.target_name, @@ -218,7 +208,6 @@ async fn get_search_results( rustdoc_status: row.rustdoc_status.unwrap_or(false), stars: row.stars.unwrap_or(0), has_unyanked_releases: row.has_unyanked_releases, - href: None, }, ) }) @@ -229,13 +218,8 @@ async fn get_search_results( // extend with the release/build information from docs.rs // Crates that are not on docs.rs yet will not be returned. let mut results = Vec::new(); - if let Some(super::rustdoc::OfficialCrateDescription { - name, - href, - description, - }) = super::rustdoc::DOC_RUST_LANG_ORG_REDIRECTS.get(query) - { - results.push(rust_lib_release(name, description, href)) + if let Some(desc) = super::rustdoc::DOC_RUST_LANG_ORG_REDIRECTS.get(query) { + results.push(ReleaseStatus::External(desc)); } let names: Vec = @@ -720,7 +704,7 @@ struct BuildQueuePage { queue: Vec, rebuild_queue: Vec, active_cdn_deployments: Vec, - in_progress_builds: Vec<(String, String)>, + in_progress_builds: Vec<(String, Version)>, expand_rebuild_queue: bool, } @@ -749,10 +733,10 @@ pub(crate) async fn build_queue_handler( // reverse the list, so the oldest comes first active_cdn_deployments.reverse(); - let in_progress_builds: Vec<(String, String)> = sqlx::query!( + let in_progress_builds: Vec<(String, Version)> = sqlx::query!( r#"SELECT crates.name, - releases.version + releases.version as "version: Version" FROM builds INNER JOIN releases ON releases.id = builds.rid INNER JOIN crates ON releases.crate_id = crates.id @@ -809,8 +793,8 @@ mod tests { use crate::db::{finish_build, initialize_build, initialize_crate, initialize_release}; use crate::registry_api::{CrateOwner, OwnerKind}; use crate::test::{ - AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, async_wrapper, - fake_release_that_failed_before_build, + AxumResponseTestExt, AxumRouterTestExt, FakeBuild, TestEnvironment, V0_1, V1, V2, V3, + async_wrapper, fake_release_that_failed_before_build, }; use anyhow::Error; use chrono::{Duration, TimeZone}; @@ -827,7 +811,7 @@ mod tests { let mut conn = db.async_conn().await; let crate_id = initialize_crate(&mut conn, "foo").await?; - let release_id = initialize_release(&mut conn, crate_id, "0.1.0").await?; + let release_id = initialize_release(&mut conn, crate_id, &V1).await?; let build_id = initialize_build(&mut conn, release_id).await?; finish_build( @@ -863,21 +847,21 @@ mod tests { env.fake_release() .await .name("foo") - .version("1.0.0") + .version(V1) .github_stats("ghost/foo", 10, 10, 10) .create() .await?; env.fake_release() .await .name("bar") - .version("1.0.0") + .version(V1) .github_stats("ghost/bar", 20, 20, 20) .create() .await?; env.fake_release() .await .name("bar") - .version("1.0.0") + .version(V1) .github_stats("ghost/bar", 20, 20, 20) .create() .await?; @@ -885,7 +869,7 @@ mod tests { env.fake_release() .await .name("baz") - .version("1.0.0") + .version(V1) .create() .await?; @@ -893,7 +877,7 @@ mod tests { env.fake_release() .await .name("in_progress") - .version("0.1.0") + .version(V0_1) .builds(vec![ FakeBuild::default() .build_status(BuildStatus::InProgress) @@ -929,18 +913,20 @@ mod tests { env.fake_release() .await .name("some_random_crate") + .version(V1) .build_result_failed() .create() .await?; env.fake_release() .await .name("some_other_crate") + .version(V1) .create() .await?; web.assert_redirect( "/releases/search?query=some_random_crate&i-am-feeling-lucky=1", - "/crate/some_random_crate/1.0.0", + &format!("/crate/some_random_crate/{V1}"), ) .await?; Ok(()) @@ -954,17 +940,19 @@ mod tests { env.fake_release() .await .name("some_random_crate") + .version(V1) .create() .await?; env.fake_release() .await .name("some_other_crate") + .version(V1) .create() .await?; web.assert_redirect( "/releases/search?query=some_random_crate&i-am-feeling-lucky=1", - "/some_random_crate/1.0.0/some_random_crate/", + &format!("/some_random_crate/{V1}/some_random_crate/"), ) .await?; Ok(()) @@ -991,11 +979,12 @@ mod tests { .await .github_stats("some/repo", 333, 22, 11) .name("some_random_crate") + .version(V1) .create() .await?; web.assert_redirect( "/releases/search?query=&i-am-feeling-lucky=1", - "/some_random_crate/1.0.0/some_random_crate/", + &format!("/some_random_crate/{V1}/some_random_crate/"), ) .await?; Ok(()) @@ -1437,7 +1426,7 @@ mod tests { env.fake_release() .await .name("yet_another_crate") - .version("0.1.0") + .version(V1) .yanked(true) .create() .await?; @@ -1458,13 +1447,8 @@ mod tests { // release that failed in the fetch-step, will miss some details let mut conn = env.async_db().async_conn().await; - fake_release_that_failed_before_build( - &mut conn, - "failed_hard", - "0.1.0", - "some random error", - ) - .await?; + fake_release_that_failed_before_build(&mut conn, "failed_hard", V2, "some random error") + .await?; let _m = crates_io .mock("GET", "/api/v1/crates") @@ -1504,8 +1488,11 @@ mod tests { // * version used is the highest semver following our own "latest version" logic assert_eq!(links[0], "/some_random_crate/latest/some_random_crate/"); assert_eq!(links[1], "/and_another_one/latest/and_another_one/"); - assert_eq!(links[2], "/yet_another_crate/0.1.0/yet_another_crate/"); - assert_eq!(links[3], "/crate/failed_hard/0.1.0"); + assert_eq!( + links[2], + format!("/yet_another_crate/{V1}/yet_another_crate/") + ); + assert_eq!(links[3], format!("/crate/failed_hard/{V2}")); Ok(()) } @@ -1863,9 +1850,9 @@ mod tests { ); let queue = env.async_build_queue(); - queue.add_crate("foo", "1.0.0", 0, None).await?; - queue.add_crate("bar", "0.1.0", -10, None).await?; - queue.add_crate("baz", "0.0.1", 10, None).await?; + queue.add_crate("foo", &V1, 0, None).await?; + queue.add_crate("bar", &V2, -10, None).await?; + queue.add_crate("baz", &V3, 10, None).await?; let full = kuchikiki::parse_html().one(web.get("/releases/queue").await?.text().await?); let items = full @@ -1875,14 +1862,14 @@ mod tests { assert_eq!(items.len(), 3); let expected = [ - ("bar", "0.1.0", Some(10)), - ("foo", "1.0.0", None), - ("baz", "0.0.1", Some(-10)), + ("bar", V2, Some(10)), + ("foo", V1, None), + ("baz", V3, Some(-10)), ]; for (li, expected) in items.iter().zip(&expected) { let a = li.as_node().select_first("a").expect("missing link"); assert!(a.text_contents().contains(expected.0)); - assert!(a.text_contents().contains(expected.1)); + assert!(a.text_contents().contains(&expected.1.to_string())); if let Some(priority) = expected.2 { assert!( @@ -1903,13 +1890,13 @@ mod tests { // we have two queued releases, where the build for one is already in progress let queue = env.async_build_queue(); - queue.add_crate("foo", "1.0.0", 0, None).await?; - queue.add_crate("bar", "0.1.0", 0, None).await?; + queue.add_crate("foo", &V1, 0, None).await?; + queue.add_crate("bar", &V2, 0, None).await?; env.fake_release() .await .name("foo") - .version("1.0.0") + .version(V1) .builds(vec![ FakeBuild::default() .build_status(BuildStatus::InProgress) @@ -1933,7 +1920,7 @@ mod tests { .expect("missing in progress list items") .map(|node| node.text_contents().trim().to_string()) .collect(); - assert_eq!(in_progress_items, vec!["foo 1.0.0"]); + assert_eq!(in_progress_items, vec![format!("foo {V1}")]); let queued_items: Vec<_> = lists[1] .as_node() @@ -1941,7 +1928,7 @@ mod tests { .expect("missing queued list items") .map(|node| node.text_contents().trim().to_string()) .collect(); - assert_eq!(queued_items, vec!["bar 0.1.0"]); + assert_eq!(queued_items, vec![format!("bar {V2}")]); Ok(()) }); @@ -1978,14 +1965,12 @@ mod tests { async_wrapper(|env| async move { let web = env.web_app().await; let queue = env.async_build_queue(); + queue.add_crate("foo", &V1, REBUILD_PRIORITY, None).await?; queue - .add_crate("foo", "1.0.0", REBUILD_PRIORITY, None) - .await?; - queue - .add_crate("bar", "0.1.0", REBUILD_PRIORITY + 1, None) + .add_crate("bar", &V2, REBUILD_PRIORITY + 1, None) .await?; queue - .add_crate("baz", "0.0.1", REBUILD_PRIORITY - 1, None) + .add_crate("baz", &V3, REBUILD_PRIORITY - 1, None) .await?; let full = kuchikiki::parse_html().one(web.get("/releases/queue").await?.text().await?); @@ -2216,7 +2201,7 @@ mod tests { env.fake_release() .await .name("failed") - .version(&format!("0.0.{i}")) + .version(format!("0.0.{i}")) .build_result_failed() .create() .await?; diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 3b38bfa68..c1db1537b 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -1,8 +1,9 @@ //! rustdoc handler +use super::extractors::PathFileExtension; use crate::{ AsyncStorage, Config, InstanceMetrics, RUSTDOC_STATIC_STORAGE_PREFIX, - db::Pool, + db::{Pool, types::version::Version}, storage::{ CompressionAlgorithm, RustdocJsonFormatVersion, StreamingBlob, compression::compression_from_file_extension, rustdoc_archive_path, rustdoc_json_path, @@ -33,7 +34,6 @@ use axum::{ response::{IntoResponse, Response as AxumResponse}, }; use http::{HeaderValue, header}; -use semver::Version; use serde::Deserialize; use std::{ collections::{BTreeMap, HashMap}, @@ -41,8 +41,7 @@ use std::{ }; use tracing::{Instrument, debug, error, info_span, instrument, trace}; -use super::extractors::PathFileExtension; - +#[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct OfficialCrateDescription { pub(crate) name: &'static str, pub(crate) href: &'static str, @@ -294,7 +293,7 @@ pub(crate) async fn rustdoc_redirector_handler( match storage .stream_rustdoc_file( &crate_name, - &krate.version.to_string(), + &krate.version, krate.latest_build_id, target, krate.archive_storage, @@ -565,7 +564,7 @@ pub(crate) async fn rustdoc_html_server_handler( let blob = match storage .stream_rustdoc_file( ¶ms.name, - &krate.version.to_string(), + &krate.version, krate.latest_build_id, &storage_path, krate.archive_storage, @@ -591,7 +590,7 @@ pub(crate) async fn rustdoc_html_server_handler( if storage .rustdoc_file_exists( ¶ms.name, - &krate.version.to_string(), + &krate.version, krate.latest_build_id, &storage_path, krate.archive_storage, @@ -633,7 +632,7 @@ pub(crate) async fn rustdoc_html_server_handler( { error!( krate = params.name, - version = krate.version.to_string(), + version = %krate.version, original_path = original_path.as_ref(), storage_path, "Couldn't find crate documentation root on storage. @@ -857,7 +856,7 @@ pub(crate) async fn target_redirect_handler( let (redirect_path, query_args) = if storage .rustdoc_file_exists( &name, - &crate_details.version.to_string(), + &crate_details.version, crate_details.latest_build_id, &storage_location_for_path, crate_details.archive_storage, @@ -997,7 +996,7 @@ pub(crate) async fn json_download_handler( let storage_path = rustdoc_json_path( &krate.name, - &krate.version.to_string(), + &krate.version, &target, wanted_format_version, Some(wanted_compression), @@ -1019,7 +1018,7 @@ pub(crate) async fn json_download_handler( if wanted_compression == CompressionAlgorithm::Zstd { let storage_path = rustdoc_json_path( &krate.name, - &krate.version.to_string(), + &krate.version, &target, wanted_format_version, None, @@ -1048,7 +1047,7 @@ pub(crate) async fn download_handler( .assume_exact_name()? .into_version(); - let archive_path = rustdoc_archive_path(&name, &version.to_string()); + let archive_path = rustdoc_archive_path(&name, &version); // not all archives are set for public access yet, so we check if // the access is set and fix it if needed. @@ -1099,7 +1098,6 @@ mod test { registry_api::{CrateOwner, OwnerKind}, storage::compression::file_extension_for, test::*, - utils::Dependency, web::{cache::CachePolicy, encode_url_path}, }; use anyhow::{Context, Result}; @@ -1107,6 +1105,7 @@ mod test { use kuchikiki::traits::TendrilSink; use pretty_assertions::assert_eq; use reqwest::StatusCode; + use semver::VersionReq; use std::collections::BTreeMap; use test_case::test_case; use tracing::info; @@ -1701,7 +1700,7 @@ mod test { ("dummy_mixed-separators", "0.2.0"), ]; - for (name, version) in &rels { + for (name, version) in rels { env.fake_release() .await .name(name) @@ -2865,8 +2864,12 @@ mod test { .version("0.1.0") .rustdoc_file("testing/index.html") .add_dependency( - Dependency::new("optional-dep".to_string(), "1.2.3".to_string()) - .set_optional(true), + dummy_metadata_dependency() + .name("optional-dep".to_string()) + .req(VersionReq::parse("1.2.3").unwrap()) + .optional(true) + .build() + .unwrap(), ) .create() .await?; @@ -2880,9 +2883,11 @@ mod test { .await?, ); assert!( - dom.select(r#"a[href="/optional-dep/1.2.3"] > i[class="dependencies normal"] + i"#) - .expect("should have optional dependency") - .any(|el| { el.text_contents().contains("optional") }) + dom.select( + r#"a[href="/optional-dep/^1.2.3"] > i[class="dependencies normal"] + i"# + ) + .expect("should have optional dependency") + .any(|el| { el.text_contents().contains("optional") }) ); let dom = kuchikiki::parse_html().one( env.web_app() @@ -2894,7 +2899,7 @@ mod test { ); assert!( dom.select( - r#"a[href="/crate/optional-dep/1.2.3"] > i[class="dependencies normal"] + i"# + r#"a[href="/crate/optional-dep/^1.2.3"] > i[class="dependencies normal"] + i"# ) .expect("should have optional dependency") .any(|el| { el.text_contents().contains("optional") }) @@ -3437,7 +3442,7 @@ mod test { .await?; const NAME: &str = "dummy"; - const VERSION: &str = "0.1.0"; + const VERSION: Version = Version::new(0, 1, 0); const TARGET: &str = "x86_64-unknown-linux-gnu"; const FORMAT_VERSION: RustdocJsonFormatVersion = RustdocJsonFormatVersion::Latest; @@ -3456,7 +3461,7 @@ mod test { .get( &rustdoc_json_path( NAME, - VERSION, + &VERSION, TARGET, FORMAT_VERSION, Some(CompressionAlgorithm::Zstd), @@ -3466,13 +3471,14 @@ mod test { .await?; for compression in RUSTDOC_JSON_COMPRESSION_ALGORITHMS { - let path = rustdoc_json_path(NAME, VERSION, TARGET, FORMAT_VERSION, Some(*compression)); + let path = + rustdoc_json_path(NAME, &VERSION, TARGET, FORMAT_VERSION, Some(*compression)); storage.delete_prefix(&path).await?; assert!(!storage.exists(&path).await?); } storage .store_one( - &rustdoc_json_path(NAME, VERSION, TARGET, FORMAT_VERSION, None), + &rustdoc_json_path(NAME, &VERSION, TARGET, FORMAT_VERSION, None), zstd_blob.content, ) .await?; diff --git a/src/web/source.rs b/src/web/source.rs index 58fcc074d..95a0b3a8c 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -1,7 +1,7 @@ use super::{error::AxumResult, match_version}; use crate::{ AsyncStorage, - db::{BuildId, Pool}, + db::{BuildId, Pool, types::version::Version}, impl_axum_webpage, storage::PathNotFoundError, web::{ @@ -19,7 +19,6 @@ use askama::Template; use axum::{Extension, response::IntoResponse}; use axum_extra::headers::HeaderMapExt; use mime::Mime; -use semver::Version; use serde::Deserialize; use std::{cmp::Ordering, sync::Arc}; use tracing::instrument; @@ -84,7 +83,7 @@ impl FileList { INNER JOIN crates ON crates.id = releases.crate_id WHERE crates.name = $1 AND releases.version = $2", name, - version.to_string(), + version as _, ) .fetch_optional(&mut *conn) .await? @@ -242,7 +241,7 @@ pub(crate) async fn source_browser_handler( name = $1 AND version = $2"#, params.name, - version.to_string() + version as _, ) .fetch_one(&mut *conn) .await?; @@ -253,7 +252,7 @@ pub(crate) async fn source_browser_handler( match storage .fetch_source_file( ¶ms.name, - &version.to_string(), + &version, row.latest_build_id, ¶ms.path, row.archive_storage, diff --git a/src/web/status.rs b/src/web/status.rs index dfc048ea6..665e95dbc 100644 --- a/src/web/status.rs +++ b/src/web/status.rs @@ -47,8 +47,10 @@ pub(crate) async fn status_handler( #[cfg(test)] mod tests { - use crate::test::{AxumResponseTestExt, AxumRouterTestExt, async_wrapper}; - use crate::web::cache::CachePolicy; + use crate::{ + test::{AxumResponseTestExt, AxumRouterTestExt, async_wrapper}, + web::{ReqVersion, cache::CachePolicy}, + }; use reqwest::StatusCode; use test_case::test_case; @@ -56,8 +58,10 @@ mod tests { #[test_case("0.1")] #[test_case("0.1.0")] #[test_case("=0.1.0"; "exact_version")] - fn status(version: &str) { + fn status(req_version: &str) { async_wrapper(|env| async move { + let req_version: ReqVersion = req_version.parse()?; + env.fake_release() .await .name("foo") @@ -68,7 +72,7 @@ mod tests { let response = env .web_app() .await - .get_and_follow_redirects(&format!("/crate/foo/{version}/status.json")) + .get_and_follow_redirects(&format!("/crate/foo/{req_version}/status.json")) .await?; response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); assert_eq!(response.headers()["access-control-allow-origin"], "*"); @@ -110,8 +114,10 @@ mod tests { #[test_case("0.1")] #[test_case("~0.1"; "semver")] - fn redirect(version: &str) { + fn redirect(req_version: &str) { async_wrapper(|env| async move { + let req_version: ReqVersion = req_version.parse()?; + env.fake_release() .await .name("foo") @@ -122,7 +128,7 @@ mod tests { let web = env.web_app().await; let redirect = web .assert_redirect( - &format!("/crate/foo/{version}/status.json"), + &format!("/crate/foo/{req_version}/status.json"), "/crate/foo/0.1.0/status.json", ) .await?; @@ -137,8 +143,10 @@ mod tests { #[test_case("0.1")] #[test_case("0.1.0")] #[test_case("=0.1.0"; "exact_version")] - fn failure(version: &str) { + fn failure(req_version: &str) { async_wrapper(|env| async move { + let req_version: ReqVersion = req_version.parse()?; + env.fake_release() .await .name("foo") @@ -150,7 +158,7 @@ mod tests { let response = env .web_app() .await - .get_and_follow_redirects(&format!("/crate/foo/{version}/status.json")) + .get_and_follow_redirects(&format!("/crate/foo/{req_version}/status.json")) .await?; response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); assert_eq!(response.headers()["access-control-allow-origin"], "*"); @@ -180,7 +188,7 @@ mod tests { // invalid semver #[test_case("foo", "0,1")] #[test_case("foo", "0,1,0")] - fn not_found(krate: &str, version: &str) { + fn not_found(krate: &str, req_version: &str) { async_wrapper(|env| async move { env.fake_release() .await @@ -192,7 +200,7 @@ mod tests { let response = env .web_app() .await - .get_and_follow_redirects(&format!("/crate/{krate}/{version}/status.json")) + .get_and_follow_redirects(&format!("/crate/{krate}/{req_version}/status.json")) .await?; response.assert_cache_control(CachePolicy::NoStoreMustRevalidate, env.config()); assert_eq!(response.headers()["access-control-allow-origin"], "*"); diff --git a/templates/releases/releases.html b/templates/releases/releases.html index 49d32a43e..0ada23250 100644 --- a/templates/releases/releases.html +++ b/templates/releases/releases.html @@ -39,18 +39,31 @@
Documentation not available on docs.rs
+ {%- when ReleaseStatus::External(release) -%} + {#- -#} +
{#- -#} +
+ {{- release.name -}} +
+ +
+ {{- release.description -}} +
+ +
+
+
{#- -#} + {%- when ReleaseStatus::Available(release) -%} {%- set release_version -%} {%- set has_unyanked_releases = release.has_unyanked_releases.unwrap_or(true) -%} {%- if release_type == "search" && has_unyanked_releases -%} - {%- set release_version = "latest" -%} + {%- set release_version = "latest".to_string() -%} {%- else -%} - {%- set release_version = release.version -%} + {%- set release_version = release.version.to_string() -%} {%- endif -%} {%- set link -%} - {%- if let Some(href) = release.href -%} - {% set link = href.to_string() -%} - {%- elif release.rustdoc_status -%} + {%- if release.rustdoc_status -%} {% set link = "/{}/{}/{}/"|format(release.name, release_version, release.target_name.as_deref().unwrap_or_default()) -%} {%- else -%} {% set link = "/crate/{}/{}"|format(release.name, release_version) -%} @@ -59,7 +72,7 @@
{#- -#}
{{- release.name -}} - {%- if !release.version.is_empty() -%}-{{ release.version }}{% endif -%} + -{{ release.version }} {%- if !has_unyanked_releases ~%} {{- crate::icons::IconTrash.render_solid(false, false, "") ~}} @@ -85,7 +98,7 @@
{%- else -%}
- {%- if release.href.is_none() %}—{% endif -%} + —
{%- endif -%}
{#- -#}