Auto merge of #704 - Mark-Simulacrum:bump-deps, r=Mark-Simulacrum

Bump a host of dependencies to newer versions

Cargo.toml

@@ -9,10 +9,10 @@ build = "build.rs"
 opt-level = 0
 
 [dependencies]
-base64 = "0.20.0"
+base64 = "0.21.5"
 bytes = "1"
 chrono = { version = "0.4", features = ["serde"] }
-crates-index = "0.18"
+crates-index = { version = "2.2.0", default-features = false, features = ["git-performance", "git-https"] }
 crossbeam-channel = "0.5"
 csv = "1.0.2"
 docsrs-metadata = { git = "https://github.com/rust-lang/docs.rs/" }
@@ -25,8 +25,8 @@ lazy_static = "1.0"
 mime = "0.3.1"
 minifier = { version = "0.2", features = ["html"] }
 r2d2 = "0.8.2"
-r2d2_sqlite = "0.21.0"
-rusqlite = { version = "0.28.0", features = ["chrono", "functions", "bundled"] }
+r2d2_sqlite = "0.22.0"
+rusqlite = { version = "0.29.0", features = ["chrono", "functions", "bundled"] }
 rand = "0.8"
 regex = "1.0"
 reqwest = { version = "0.11", features = ["blocking", "json"] }
@@ -38,7 +38,7 @@ clap = { version = "4", features = ["derive"] }
 tar = "0.4.36"
 tempfile = "3.0.0"
 tera = "0.11.7"
-toml = "0.4.6"
+toml = "0.8.6"
 url = "2"
 walkdir = "2"
 warp = "0.3"
@@ -48,21 +48,21 @@ hmac = "0.12"
 sha-1 = "0.10"
 rust_team_data = { git = "https://github.com/rust-lang/team" }
 systemstat = "0.1.11"
-rustwide = { version = "0.15.0", features = ["unstable", "unstable-toolchain-ci"] }
+rustwide = { version = "0.16.0", features = ["unstable", "unstable-toolchain-ci"] }
 percent-encoding = "2.1.0"
 remove_dir_all = "0.7"
 ctrlc = "3.1.3"
-prometheus = "0.7.0"
-cargo_metadata = "0.15"
-indexmap = "1.4.0"
+prometheus = "0.13.3"
+cargo_metadata = "0.18.1"
+indexmap = { version = "2.0.2", features = ["serde"] }
 tokio = "1.24"
-aws-types = { version = "0.52", features = ["hardcoded-credentials"] }
-aws-smithy-http = "0.52"
-aws-smithy-async = "0.52"
-aws-sdk-s3 = "0.22"
+aws-types = "0.56.1"
+aws-credential-types = "0.56.1"
+aws-smithy-async = "0.56.1"
+aws-sdk-s3 = "0.34"
 thiserror = "1.0.38"
 
 [dev-dependencies]
 assert_cmd = "2.0.4"
 difference = "2.0.0"
-predicates = "2.0.0"
+predicates = "3.0.4"

src/agent/api.rs

@@ -8,6 +8,7 @@ use crate::results::TestResult;
 use crate::server::api_types::{AgentConfig, ApiResponse, CraterToken};
 use crate::toolchain::Toolchain;
 use crate::utils;
+use base64::Engine;
 use rand::Rng;
 use reqwest::blocking::RequestBuilder;
 use reqwest::header::AUTHORIZATION;
@@ -186,7 +187,7 @@ impl AgentApi {
                             "crate": krate,
                             "toolchain": toolchain,
                             "result": result,
-                            "log": base64::encode(log),
+                            "log": base64::engine::general_purpose::STANDARD.encode(log),
                         },
                     ],
                     "version": version

src/crates/sources/registry.rs

@@ -1,7 +1,7 @@
 use crate::crates::{lists::List, Crate};
 use crate::dirs::WORK_DIR;
 use crate::prelude::*;
-use crates_index::Index;
+use crates_index::GitIndex;
 use std::collections::HashMap;
 use std::fs::{self};
 
@@ -15,7 +15,7 @@ impl List for RegistryList {
         let mut counts = HashMap::new();
 
         fs::create_dir_all(&*WORK_DIR)?;
-        let mut index = Index::with_path(
+        let mut index = GitIndex::with_path(
             WORK_DIR.join("crates.io-index"),
             "https://github.com/rust-lang/crates.io-index",
         )

src/report/mod.rs

@@ -7,7 +7,7 @@ use crate::report::analyzer::{analyze_report, ReportConfig, ToolchainSelect};
 use crate::results::{EncodedLog, EncodingType, FailureReason, ReadResults, TestResult};
 use crate::toolchain::Toolchain;
 use crate::utils;
-use crates_index::Index;
+use crates_index::GitIndex;
 use mime::{self, Mime};
 use percent_encoding::{utf8_percent_encode, AsciiSet};
 use std::borrow::Cow;
@@ -182,7 +182,10 @@ fn crate_to_path_fragment(
     path
 }
 
-fn get_crate_version_status(index: &Index, krate: &Crate) -> Fallible<Option<CrateVersionStatus>> {
+fn get_crate_version_status(
+    index: &GitIndex,
+    krate: &Crate,
+) -> Fallible<Option<CrateVersionStatus>> {
     if let Crate::Registry(krate) = krate {
         let index_krate = index
             .crate_(&krate.name)
@@ -217,7 +220,7 @@ pub fn generate_report<DB: ReadResults>(
     crates: &[Crate],
 ) -> Fallible<RawTestResults> {
     let mut crates = crates.to_vec();
-    let index = Index::with_path(
+    let index = GitIndex::with_path(
         WORK_DIR.join("crates.io-index"),
         "https://github.com/rust-lang/crates.io-index",
     )
@@ -616,7 +619,7 @@ mod tests {
     use crate::experiments::{CapLints, Experiment, Mode, Status};
     use crate::results::{BrokenReason, DummyDB, FailureReason, TestResult};
     use crate::toolchain::{MAIN_TOOLCHAIN, TEST_TOOLCHAIN};
-    use crates_index::Index;
+    use crates_index::GitIndex;
 
     #[test]
     fn test_crate_to_path_fragment() {
@@ -698,12 +701,11 @@ mod tests {
         };
         let gh = Crate::GitHub(repo);
 
-        let mut index = Index::with_path(
+        let index = GitIndex::with_path(
             WORK_DIR.join("crates.io-index"),
             "https://github.com/rust-lang/crates.io-index",
         )
         .unwrap();
-        index.update().unwrap();
 
         assert_eq!(
             get_crate_version_status(&index, &reg).unwrap().unwrap(),

src/report/s3.rs

@@ -86,7 +86,7 @@ impl ReportWriter for S3Writer {
             let mut request = self
                 .client
                 .create_multipart_upload()
-                .acl(aws_sdk_s3::model::ObjectCannedAcl::PublicRead)
+                .acl(aws_sdk_s3::types::ObjectCannedAcl::PublicRead)
                 .key(format!(
                     "{}/{}",
                     self.prefix,
@@ -111,7 +111,7 @@ impl ReportWriter for S3Writer {
             let bytes = bytes::Bytes::from(s);
             let mut part = 1;
             let mut start = 0;
-            let mut parts = aws_sdk_s3::model::CompletedMultipartUpload::builder();
+            let mut parts = aws_sdk_s3::types::CompletedMultipartUpload::builder();
             while start < bytes.len() {
                 let chunk = bytes.slice(start..std::cmp::min(start + chunk_size, bytes.len()));
 
@@ -126,7 +126,7 @@ impl ReportWriter for S3Writer {
                 match self.runtime.block_on(request.send()) {
                     Ok(p) => {
                         parts = parts.parts(
-                            aws_sdk_s3::model::CompletedPart::builder()
+                            aws_sdk_s3::types::CompletedPart::builder()
                                 .e_tag(p.e_tag.clone().unwrap())
                                 .part_number(part)
                                 .build(),
@@ -160,8 +160,8 @@ impl ReportWriter for S3Writer {
             let mut request = self
                 .client
                 .put_object()
-                .body(aws_smithy_http::byte_stream::ByteStream::from(s))
-                .acl(aws_sdk_s3::model::ObjectCannedAcl::PublicRead)
+                .body(aws_sdk_s3::primitives::ByteStream::from(s))
+                .acl(aws_sdk_s3::types::ObjectCannedAcl::PublicRead)
                 .key(format!(
                     "{}/{}",
                     self.prefix,

src/results/db.rs

@@ -7,6 +7,7 @@ use crate::results::{
     DeleteResults, EncodedLog, EncodingType, ReadResults, TestResult, WriteResults,
 };
 use crate::toolchain::Toolchain;
+use base64::Engine;
 use rustwide::logging::{self, LogStorage};
 
 #[derive(Deserialize)]
@@ -89,7 +90,9 @@ impl<'a> DatabaseDB<'a> {
                 &result.krate,
                 &result.toolchain,
                 &result.result,
-                &base64::decode(&result.log).with_context(|_| "invalid base64 log provided")?,
+                &base64::engine::general_purpose::STANDARD
+                    .decode(&result.log)
+                    .with_context(|_| "invalid base64 log provided")?,
                 encoding_type,
             )?;
 
@@ -262,6 +265,8 @@ impl<'a> DeleteResults for DatabaseDB<'a> {
 
 #[cfg(test)]
 mod tests {
+    use base64::Engine;
+
     use super::{DatabaseDB, ProgressData, TaskResult};
     use crate::actions::{Action, ActionsCtx, CreateExperiment};
     use crate::config::Config;
@@ -468,7 +473,7 @@ mod tests {
                         krate: updated.clone(),
                         toolchain: MAIN_TOOLCHAIN.clone(),
                         result: TestResult::TestPass,
-                        log: base64::encode("foo"),
+                        log: base64::engine::general_purpose::STANDARD.encode("foo"),
                     }],
                     version: Some((krate.clone(), updated.clone())),
                 },

src/server/metrics.rs

@@ -4,10 +4,7 @@ use crate::prelude::*;
 use crate::server::agents::Agent;
 use chrono::{DateTime, Utc};
 use prometheus::proto::{Metric, MetricFamily};
-use prometheus::{
-    HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, __register_counter_vec,
-    __register_gauge, __register_gauge_vec, opts, register_counter, register_int_counter,
-};
+use prometheus::{HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec};
 
 const JOBS_METRIC: &str = "crater_completed_jobs_total";
 const AGENT_WORK_METRIC: &str = "crater_agent_supposed_to_work";
@@ -66,7 +63,7 @@ impl Metrics {
             .inc_by(1);
     }
 
-    pub fn record_completed_jobs(&self, agent: &str, experiment: &str, amount: i64) {
+    pub fn record_completed_jobs(&self, agent: &str, experiment: &str, amount: u64) {
         self.crater_completed_jobs_total
             .with_label_values(&[agent, experiment])
             .inc_by(amount);

src/server/reports.rs

@@ -24,13 +24,13 @@ fn generate_report(data: &Data, ex: &Experiment, results: &DatabaseDB) -> Fallib
         }
         BucketRegion::Custom { url } => {
             config.set_region(Some(aws_types::region::Region::from_static("us-east-1")));
-            config.set_endpoint_resolver(Some(Arc::new(aws_sdk_s3::Endpoint::immutable(url)?)));
+            config.set_endpoint_url(Some(url.clone()));
         }
     }
     config.set_credentials_provider(Some(data.tokens.reports_bucket.to_aws_credentials()));
     // https://github.com/awslabs/aws-sdk-rust/issues/586 -- without this, the
     // SDK will just completely not retry requests.
-    config.set_sleep_impl(Some(Arc::new(
+    config.set_sleep_impl(Some(aws_sdk_s3::config::SharedAsyncSleep::new(
         aws_smithy_async::rt::sleep::TokioSleep::new(),
     )));
     config.set_retry_config(Some(RetryConfig::standard()));

src/server/routes/agent.rs

@@ -206,7 +206,7 @@ impl RecordProgressThread {
                     metrics.record_completed_jobs(
                         &worker_name,
                         &ex.name,
-                        result.data.results.len() as i64,
+                        result.data.results.len() as u64,
                     );
 
                     if let Err(e) = db.clear_stale_records() {

src/server/tokens.rs

@@ -30,12 +30,18 @@ pub struct ReportsBucket {
 }
 
 impl ReportsBucket {
-    pub(crate) fn to_aws_credentials(&self) -> aws_types::credentials::SharedCredentialsProvider {
-        aws_types::credentials::SharedCredentialsProvider::new(aws_types::Credentials::from_keys(
-            self.access_key.clone(),
-            self.secret_key.clone(),
-            None,
-        ))
+    pub(crate) fn to_aws_credentials(
+        &self,
+    ) -> aws_credential_types::provider::SharedCredentialsProvider {
+        aws_credential_types::provider::SharedCredentialsProvider::new(
+            aws_sdk_s3::config::Credentials::new(
+                self.access_key.clone(),
+                self.secret_key.clone(),
+                None,
+                None,
+                "crater-credentials",
+            ),
+        )
     }
 }
 

tests/check_config/mod.rs

@@ -21,7 +21,7 @@ fn test_bad_config_duplicate_crate() {
         .assert()
         .failure()
         .code(1)
-        .stderr(contains("duplicate key: `lazy_static` for key `crates`"));
+        .stderr(contains("duplicate key `lazy_static` in table `crates`"));
 }
 
 #[test]
@@ -32,7 +32,7 @@ fn test_bad_config_duplicate_repo() {
         .failure()
         .code(1)
         .stderr(contains(
-            "duplicate key: `brson/hello-rs` for key `github-repos`",
+            "duplicate key `brson/hello-rs` in table `github-repos`",
         ));
 }