diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fecc21b83..f80c4e614 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,7 +32,7 @@ jobs: - name: Install Rust Stable env: - RUST_VERSION: "1.85.0" + RUST_VERSION: "1.87.0" run: | rustc -vV rustup toolchain install $RUST_VERSION @@ -103,6 +103,7 @@ jobs: EMAIL_ENCRYPTION_KEY: ${{ secrets.EMAIL_ENCRYPTION_KEY }} ZULIP_API_TOKEN: ${{ secrets.ZULIP_API_TOKEN }} ZULIP_USERNAME: ${{ secrets.ZULIP_USERNAME }} + CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} run: | cargo run sync apply --src build diff --git a/README.md b/README.md index e4901fc3e..80269c899 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ The repository is automatically synchronized with: | Zulip user group membership | *Shortly after merge* | [Integration source][sync-team-src] | | [Governance section on the website][www] | Once per day | [Integration source][www-src] | | crates.io admin access | 1 hour | [Integration source][crates-io-admin-src] | +| crates.io trusted publishing config | *Shortly after merge* | [Integration source][sync-team-src] | If you need to add or remove a person from a team, send a PR to this repository. After it's merged, their account will be added/removed diff --git a/docs/toml-schema.md b/docs/toml-schema.md index 51c785bc3..801d401f2 100644 --- a/docs/toml-schema.md +++ b/docs/toml-schema.md @@ -430,3 +430,16 @@ allowed-merge-teams = ["awesome-team"] # (optional) merge-bots = ["homu"] ``` + +### Crates.io trusted publishing +Configure crates.io Trusted Publishing for crates published from a given repository from GitHub Actions. + +```toml +[[crates-io-publishing]] +# Crates that will be published with the given workflow file from this repository (required) +crates = ["regex"] +# Name of a GitHub Actions workflow file that will publish the crate (required) +workflow-filename = "ci.yml" +# GitHub Actions environment that has to be used for the publishing (required) +environment = "deploy" +``` diff --git a/rust_team_data/src/v1.rs b/rust_team_data/src/v1.rs index ebd482be5..593f52cae 100644 --- a/rust_team_data/src/v1.rs +++ b/rust_team_data/src/v1.rs @@ -174,6 +174,7 @@ pub struct Repo { pub teams: Vec, pub members: Vec, pub branch_protections: Vec, + pub crates: Vec, pub archived: bool, // This attribute is not synced by sync-team. pub private: bool, @@ -182,6 +183,12 @@ pub struct Repo { pub auto_merge_enabled: bool, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Crate { + pub name: String, + pub crates_io_publishing: Option, +} + #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "kebab-case")] pub enum Bot { @@ -244,6 +251,12 @@ pub struct BranchProtection { pub merge_bots: Vec, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct CratesIoPublishing { + pub workflow_file: String, + pub environment: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct Person { pub name: String, diff --git a/src/main.rs b/src/main.rs index be879a166..b6bbef7cc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,7 +9,7 @@ mod schema; mod static_api; mod validate; -const AVAILABLE_SERVICES: &[&str] = &["github", "mailgun", "zulip"]; +const AVAILABLE_SERVICES: &[&str] = &["github", "mailgun", "zulip", "crates-io"]; const USER_AGENT: &str = "https://github.com/rust-lang/team (infra@rust-lang.org)"; diff --git a/src/schema.rs b/src/schema.rs index 9840fdb03..399034c00 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -807,6 +807,8 @@ pub(crate) struct Repo { pub access: RepoAccess, #[serde(default)] pub branch_protections: Vec, + #[serde(default)] + pub crates_io_publishing: Vec, } #[derive(serde_derive::Deserialize, Debug, Clone, PartialEq)] @@ -865,3 +867,11 @@ pub(crate) struct BranchProtection { #[serde(default)] pub merge_bots: Vec, } + +#[derive(serde_derive::Deserialize, Debug)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub(crate) struct CratesIoPublishing { + pub crates: Vec, + pub workflow_filename: String, + pub environment: String, +} diff --git a/src/static_api.rs b/src/static_api.rs index bcfa2c501..45cbef110 100644 --- a/src/static_api.rs +++ b/src/static_api.rs @@ -147,6 +147,19 @@ impl<'a> Generator<'a> { members }, branch_protections, + crates: r + .crates_io_publishing + .iter() + .flat_map(|p| { + p.crates.iter().map(|krate| v1::Crate { + name: krate.to_string(), + crates_io_publishing: Some(v1::CratesIoPublishing { + workflow_file: p.workflow_filename.clone(), + environment: p.environment.clone(), + }), + }) + }) + .collect(), archived, auto_merge_enabled: !managed_by_bors, }; diff --git a/src/validate.rs b/src/validate.rs index 258568fa0..f0242de82 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -54,6 +54,7 @@ static CHECKS: &[Check)>] = checks![ validate_repos, validate_archived_repos, validate_branch_protections, + validate_trusted_publishing, validate_member_roles, validate_admin_access, validate_website, @@ -1000,6 +1001,30 @@ Please remove the attributes when using bors"#, }) } +/// Validate that trusted publishing configuration has unique crates across all repositories. +fn validate_trusted_publishing(data: &Data, errors: &mut Vec) { + let mut crates = HashMap::new(); + wrapper(data.repos(), errors, |repo, _| { + let repo_name = format!("{}/{}", repo.org, repo.name); + for publishing in &repo.crates_io_publishing { + if publishing.crates.is_empty() { + return Err(anyhow::anyhow!( + "Repository `{repo_name}` has trusted publishing for an empty set of crates.", + )); + } + + for krate in &publishing.crates { + if let Some(prev_repo) = crates.insert(krate.clone(), repo_name.clone()) { + return Err(anyhow::anyhow!( + "Repository `{repo_name}` configures trusted publishing for crate `{krate}` that is also configured in `{prev_repo}`. Each crate can only be configured once.", + )); + } + } + } + Ok(()) + }) +} + /// Enforce that roles are only assigned to a valid team member, and that the /// same role id always has a consistent description across teams (because the /// role id becomes the Fluent id used for translation). diff --git a/sync-team/src/crates_io/api.rs b/sync-team/src/crates_io/api.rs new file mode 100644 index 000000000..f30eb0f95 --- /dev/null +++ b/sync-team/src/crates_io/api.rs @@ -0,0 +1,172 @@ +use crate::crates_io::CratesIoPublishingConfig; +use crate::utils::ResponseExt; +use anyhow::{Context, anyhow}; +use log::debug; +use reqwest::blocking::Client; +use reqwest::header; +use reqwest::header::{HeaderMap, HeaderValue}; +use secrecy::{ExposeSecret, SecretString}; +use serde::Serialize; +use std::fmt::{Display, Formatter}; + +// OpenAPI spec: https://crates.io/api/openapi.json +const CRATES_IO_BASE_URL: &str = "https://crates.io/api/v1"; + +/// Access to the Zulip API +#[derive(Clone)] +pub(crate) struct CratesIoApi { + client: Client, + token: SecretString, + dry_run: bool, +} + +impl CratesIoApi { + pub(crate) fn new(token: SecretString, dry_run: bool) -> Self { + let mut map = HeaderMap::default(); + map.insert( + header::USER_AGENT, + HeaderValue::from_static(crate::USER_AGENT), + ); + + Self { + client: reqwest::blocking::ClientBuilder::default() + .default_headers(map) + .build() + .unwrap(), + token, + dry_run, + } + } + + /// List existing trusted publishing configurations for a given crate. + pub(crate) fn list_trusted_publishing_github_configs( + &self, + krate: &str, + ) -> anyhow::Result> { + #[derive(serde::Deserialize)] + struct GetTrustedPublishing { + github_configs: Vec, + } + + let response: GetTrustedPublishing = self + .req::<()>( + reqwest::Method::GET, + &format!("/trusted_publishing/github_configs?crate={krate}"), + None, + )? + .error_for_status()? + .json_annotated()?; + + Ok(response.github_configs) + } + + /// Create a new trusted publishing configuration for a given crate. + pub(crate) fn create_trusted_publishing_github_config( + &self, + config: &CratesIoPublishingConfig, + ) -> anyhow::Result<()> { + debug!( + "Creating trusted publishing config for '{}' in repo '{}/{}', workflow file '{}' and environment '{}'", + config.krate.0, + config.repo_org, + config.repo_name, + config.workflow_file, + config.environment + ); + + if self.dry_run { + return Ok(()); + } + + #[derive(serde::Serialize)] + struct TrustedPublishingGitHubConfigCreate<'a> { + repository_owner: &'a str, + repository_name: &'a str, + #[serde(rename = "crate")] + krate: &'a str, + workflow_filename: &'a str, + environment: Option<&'a str>, + } + + #[derive(serde::Serialize)] + struct CreateTrustedPublishing<'a> { + github_config: TrustedPublishingGitHubConfigCreate<'a>, + } + + let request = CreateTrustedPublishing { + github_config: TrustedPublishingGitHubConfigCreate { + repository_owner: &config.repo_org, + repository_name: &config.repo_name, + krate: &config.krate.0, + workflow_filename: &config.workflow_file, + environment: Some(&config.environment), + }, + }; + + self.req( + reqwest::Method::POST, + "/trusted_publishing/github_configs", + Some(&request), + )? + .error_for_status() + .with_context(|| anyhow!("Cannot created trusted publishing config {config:?}"))?; + + Ok(()) + } + + /// Delete a trusted publishing configuration with the given ID. + pub(crate) fn delete_trusted_publishing_github_config( + &self, + id: TrustedPublishingId, + ) -> anyhow::Result<()> { + debug!("Deleting trusted publishing with ID {id}"); + + if !self.dry_run { + self.req::<()>( + reqwest::Method::DELETE, + &format!("/trusted_publishing/github_configs/{}", id.0), + None, + )? + .error_for_status() + .with_context(|| anyhow!("Cannot delete trusted publishing config with ID {id}"))?; + } + + Ok(()) + } + + /// Perform a request against the crates.io API + fn req( + &self, + method: reqwest::Method, + path: &str, + data: Option<&T>, + ) -> anyhow::Result { + let mut req = self + .client + .request(method, format!("{CRATES_IO_BASE_URL}{path}")) + .bearer_auth(self.token.expose_secret()); + if let Some(data) = data { + req = req.json(data); + } + + Ok(req.send()?) + } +} + +#[derive(serde::Deserialize, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct TrustedPublishingId(u64); + +impl Display for TrustedPublishingId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[derive(serde::Deserialize, Debug)] +pub(crate) struct TrustedPublishingGitHubConfig { + pub(crate) id: TrustedPublishingId, + pub(crate) repository_owner: String, + pub(crate) repository_name: String, + pub(crate) workflow_filename: String, + pub(crate) environment: Option, +} diff --git a/sync-team/src/crates_io/mod.rs b/sync-team/src/crates_io/mod.rs new file mode 100644 index 000000000..b740ee264 --- /dev/null +++ b/sync-team/src/crates_io/mod.rs @@ -0,0 +1,202 @@ +mod api; + +use crate::team_api::TeamApi; +use std::cmp::Ordering; + +use crate::crates_io::api::{CratesIoApi, TrustedPublishingGitHubConfig}; +use anyhow::Context; +use secrecy::SecretString; +use std::collections::HashMap; + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +struct CrateName(String); + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] +struct CratesIoPublishingConfig { + krate: CrateName, + repo_org: String, + repo_name: String, + workflow_file: String, + environment: String, +} + +pub(crate) struct SyncCratesIo { + crates_io_api: CratesIoApi, + crates: HashMap, +} + +impl SyncCratesIo { + pub(crate) fn new( + token: SecretString, + team_api: &TeamApi, + dry_run: bool, + ) -> anyhow::Result { + let crates_io_api = CratesIoApi::new(token, dry_run); + let crates: HashMap = team_api + .get_repos()? + .into_iter() + .flat_map(|repo| { + repo.crates + .iter() + .filter_map(|krate| { + let Some(publishing) = &krate.crates_io_publishing else { + return None; + }; + Some(( + CrateName(krate.name.clone()), + CratesIoPublishingConfig { + krate: CrateName(krate.name.clone()), + repo_org: repo.org.clone(), + repo_name: repo.name.clone(), + workflow_file: publishing.workflow_file.clone(), + environment: publishing.environment.clone(), + }, + )) + }) + .collect::>() + }) + .collect(); + + Ok(Self { + crates_io_api, + crates, + }) + } + + pub(crate) fn diff_all(&self) -> anyhow::Result { + let mut config_diffs: Vec = vec![]; + + // Note: we currently only support one trusted publishing configuration per crate + for (krate, desired) in &self.crates { + let mut configs = self + .crates_io_api + .list_trusted_publishing_github_configs(&krate.0) + .with_context(|| format!("Failed to list configs for crate '{}'", krate.0))?; + + // Find if there are config(s) that match what we need + let matching_configs = configs + .extract_if(.., |config| { + let TrustedPublishingGitHubConfig { + id: _, + repository_owner, + repository_name, + workflow_filename, + environment, + } = config; + *repository_owner.to_lowercase() == desired.repo_org.to_lowercase() + && *repository_name.to_lowercase() == desired.repo_name.to_lowercase() + && *workflow_filename == desired.workflow_file + && environment.as_deref() == Some(&desired.environment) + }) + .collect::>(); + + if !matching_configs.is_empty() { + // If we found a matching config, we don't need to do anything with it + // It shouldn't be possible to have multiple configs with the same repo, workflow + // and environment for a single crate. + assert_eq!(matching_configs.len(), 1); + } else { + // If no match was found, we want to create this config + config_diffs.push(ConfigDiff::Create(desired.clone())); + } + + // Non-matching configs should be deleted + config_diffs.extend(configs.into_iter().map(ConfigDiff::Delete)); + } + + // We want to apply deletions first, and only then create new configs, to ensure that we + // don't try to create a duplicate config where e.g. only the environment differs, which + // would be an error in crates.io. + config_diffs.sort_by(|a, b| match &(a, b) { + (ConfigDiff::Delete(_), ConfigDiff::Create(_)) => Ordering::Less, + (ConfigDiff::Create(_), ConfigDiff::Delete(_)) => Ordering::Greater, + (ConfigDiff::Delete(a), ConfigDiff::Delete(b)) => a.id.cmp(&b.id), + (ConfigDiff::Create(a), ConfigDiff::Create(b)) => a.cmp(b), + }); + + Ok(Diff { config_diffs }) + } +} + +pub(crate) struct Diff { + config_diffs: Vec, +} + +impl Diff { + pub(crate) fn apply(&self, sync: &SyncCratesIo) -> anyhow::Result<()> { + for diff in &self.config_diffs { + diff.apply(sync)?; + } + Ok(()) + } + + pub(crate) fn is_empty(&self) -> bool { + self.config_diffs.is_empty() + } +} + +impl std::fmt::Display for Diff { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if !&self.config_diffs.is_empty() { + writeln!(f, "💻 Trusted Publishing Config Diffs:")?; + for diff in &self.config_diffs { + write!(f, "{diff}")?; + } + } + + Ok(()) + } +} + +enum ConfigDiff { + Create(CratesIoPublishingConfig), + Delete(TrustedPublishingGitHubConfig), +} +impl ConfigDiff { + fn apply(&self, sync: &SyncCratesIo) -> anyhow::Result<()> { + match self { + ConfigDiff::Create(config) => sync + .crates_io_api + .create_trusted_publishing_github_config(config), + ConfigDiff::Delete(config) => sync + .crates_io_api + .delete_trusted_publishing_github_config(config.id), + } + } +} + +impl std::fmt::Display for ConfigDiff { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ConfigDiff::Create(config) => { + writeln!( + f, + " Creating trusted publishing config for krate `{}`", + config.krate.0 + )?; + writeln!(f, " Repo: {}/{}", config.repo_org, config.repo_name)?; + writeln!(f, " Workflow file: {}", config.workflow_file)?; + writeln!(f, " Environment: {}", config.environment)?; + } + ConfigDiff::Delete(config) => { + writeln!( + f, + " Deleting trusted publishing config with ID {}", + config.id + )?; + writeln!( + f, + " Repo: {}/{}", + config.repository_owner, config.repository_name + )?; + writeln!(f, " Workflow file: {}", config.workflow_filename)?; + writeln!( + f, + " Environment: {}", + config.environment.as_deref().unwrap_or("(none)") + )?; + } + } + Ok(()) + } +} diff --git a/sync-team/src/github/tests/test_utils.rs b/sync-team/src/github/tests/test_utils.rs index 8a83d4eb2..ba7fdee25 100644 --- a/sync-team/src/github/tests/test_utils.rs +++ b/sync-team/src/github/tests/test_utils.rs @@ -351,6 +351,7 @@ impl From for v1::Repo { teams: teams.clone(), members: members.clone(), branch_protections, + crates: vec![], archived, private: false, auto_merge_enabled: allow_auto_merge, diff --git a/sync-team/src/lib.rs b/sync-team/src/lib.rs index 8999597f2..a413982e6 100644 --- a/sync-team/src/lib.rs +++ b/sync-team/src/lib.rs @@ -1,3 +1,4 @@ +mod crates_io; mod github; mod mailgun; pub mod team_api; @@ -6,6 +7,7 @@ mod zulip; use std::collections::BTreeSet; +use crate::crates_io::SyncCratesIo; use crate::github::{GitHubApiRead, GitHubWrite, HttpClient, create_diff}; use crate::team_api::TeamApi; use crate::zulip::SyncZulip; @@ -66,6 +68,17 @@ pub fn run_sync_team( diff.apply(&sync)?; } } + "crates-io" => { + let token = SecretString::from(get_env("CRATES_IO_TOKEN")?); + let sync = SyncCratesIo::new(token, &team_api, dry_run)?; + let diff = sync.diff_all()?; + if !diff.is_empty() { + info!("{diff}"); + } + if !only_print_plan { + diff.apply(&sync)?; + } + } _ => panic!("unknown service: {service}"), } } diff --git a/tests/static-api/_expected/v1/repos.json b/tests/static-api/_expected/v1/repos.json index d6d33a9aa..e5ea0a55c 100644 --- a/tests/static-api/_expected/v1/repos.json +++ b/tests/static-api/_expected/v1/repos.json @@ -24,6 +24,7 @@ "merge_bots": [] } ], + "crates": [], "archived": true, "private": false, "auto_merge_enabled": true @@ -63,6 +64,22 @@ "merge_bots": [] } ], + "crates": [ + { + "name": "my-crate", + "crates_io_publishing": { + "workflow_file": "ci.yml", + "environment": "deploy" + } + }, + { + "name": "my-crate-2", + "crates_io_publishing": { + "workflow_file": "ci.yml", + "environment": "deploy" + } + } + ], "archived": false, "private": false, "auto_merge_enabled": true diff --git a/tests/static-api/_expected/v1/repos/archived_repo.json b/tests/static-api/_expected/v1/repos/archived_repo.json index d91866e71..a73095a4e 100644 --- a/tests/static-api/_expected/v1/repos/archived_repo.json +++ b/tests/static-api/_expected/v1/repos/archived_repo.json @@ -22,6 +22,7 @@ "merge_bots": [] } ], + "crates": [], "archived": true, "private": false, "auto_merge_enabled": true diff --git a/tests/static-api/_expected/v1/repos/some_repo.json b/tests/static-api/_expected/v1/repos/some_repo.json index 6acc4a93b..5889a05ab 100644 --- a/tests/static-api/_expected/v1/repos/some_repo.json +++ b/tests/static-api/_expected/v1/repos/some_repo.json @@ -33,6 +33,22 @@ "merge_bots": [] } ], + "crates": [ + { + "name": "my-crate", + "crates_io_publishing": { + "workflow_file": "ci.yml", + "environment": "deploy" + } + }, + { + "name": "my-crate-2", + "crates_io_publishing": { + "workflow_file": "ci.yml", + "environment": "deploy" + } + } + ], "archived": false, "private": false, "auto_merge_enabled": true diff --git a/tests/static-api/repos/test-org/some_repo.toml b/tests/static-api/repos/test-org/some_repo.toml index cd5e772a7..59b3c6da8 100644 --- a/tests/static-api/repos/test-org/some_repo.toml +++ b/tests/static-api/repos/test-org/some_repo.toml @@ -10,3 +10,8 @@ foo = "maintain" pattern = "master" ci-checks = ["CI"] allowed-merge-teams = ["foo"] + +[[crates-io-publishing]] +crates = ["my-crate", "my-crate-2"] +workflow-filename = "ci.yml" +environment = "deploy"