From 613bee966a48646a5c526bcd5816b2cd5d99d364 Mon Sep 17 00:00:00 2001 From: cameron Date: Wed, 20 Jul 2022 17:32:19 +0100 Subject: [PATCH 01/10] better errors --- Cargo.lock | 9 +- catalyst-toolbox/Cargo.toml | 1 + .../ideascale/models/de/clean_string.txt | 8 ++ .../src/ideascale/models/de/ada_rewards.rs | 125 ++++++++++++++++++ .../src/ideascale/models/de/clean_string.rs | 107 +++++++++++++++ .../src/ideascale/models/{de.rs => de/mod.rs} | 83 ++---------- catalyst-toolbox/tests/tally/generator.rs | 2 +- 7 files changed, 259 insertions(+), 76 deletions(-) create mode 100644 catalyst-toolbox/proptest-regressions/ideascale/models/de/clean_string.txt create mode 100644 catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs create mode 100644 catalyst-toolbox/src/ideascale/models/de/clean_string.rs rename catalyst-toolbox/src/ideascale/models/{de.rs => de/mod.rs} (60%) diff --git a/Cargo.lock b/Cargo.lock index 1659480d..9bcb039e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -508,6 +508,7 @@ dependencies = [ "thor", "time 0.3.11", "tokio", + "unarray", "url", "versionisator", "vit-servicing-station-lib", @@ -717,7 +718,7 @@ name = "chain-vote" version = "0.1.0" source = "git+https://github.com/input-output-hk/chain-libs.git?branch=master#a8ac1f74eb596e27f9383f3a13e4da3fc16d8d43" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "chain-core", "chain-crypto", "const_format", @@ -5667,6 +5668,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unarray" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634233a4d474fb493db992470f77c312dd7cf9c967a4a3125ff76d511535e210" + [[package]] name = "unicase" version = "2.6.0" diff --git a/catalyst-toolbox/Cargo.toml b/catalyst-toolbox/Cargo.toml index 3faa7d63..0ee49cb8 100644 --- a/catalyst-toolbox/Cargo.toml +++ b/catalyst-toolbox/Cargo.toml @@ -64,6 +64,7 @@ vit-servicing-station-lib = { git = "https://github.com/input-output-hk/vit-serv env_logger = "0.9" voting-hir = { path = "../voting-hir", features = ["serde"] } fraction = "0.10" +unarray = "0.1" [dev-dependencies] rand_chacha = "0.3" diff --git a/catalyst-toolbox/proptest-regressions/ideascale/models/de/clean_string.txt b/catalyst-toolbox/proptest-regressions/ideascale/models/de/clean_string.txt new file mode 100644 index 00000000..1ec91551 --- /dev/null +++ b/catalyst-toolbox/proptest-regressions/ideascale/models/de/clean_string.txt @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +xx 35f62b501c8f4be5ccc1ca95c85692680d9c5bb51cf31931292962bf805e8142 # shrinks to input = _AnyStringDeserializesToCleanStringArgs { s: "" } +xx 6628679236d41598dac84340f0b668e82989e373457f91e51dbd077d870494a6 # shrinks to input = _AnyStringDeserializesToCleanStringArgs { s: "\"" } diff --git a/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs b/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs new file mode 100644 index 00000000..ef97d130 --- /dev/null +++ b/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs @@ -0,0 +1,125 @@ +use std::{ + fmt::{Display, Formatter}, + num::ParseIntError, +}; + +use once_cell::sync::Lazy; +use regex::Regex; +use serde::{ + de::{Error, Visitor}, + Deserialize, Deserializer, +}; + +#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)] +pub struct AdaRewards(pub u64); + +impl<'de> Deserialize<'de> for AdaRewards { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let num = deserializer.deserialize_str(V)?; + Ok(Self(num)) + } +} + +static REGEX: Lazy = Lazy::new(|| Regex::new(r#"\$([0-9]+) in ada"#).unwrap()); + +struct V; + +impl<'a> Visitor<'a> for V { + type Value = u64; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str( + "a string of the form: `$N in ada`, where `N` is a u64 (e.g. \"$123 in ada\")", + ) + } + + fn visit_str(self, v: &str) -> Result + where + E: Error, + { + // input is not standarized, hack an early return if it is just 0 ada + if v.starts_with("0 ada") { + return Ok(0); + } + + let bad_pattern = || E::custom("didn't match `$N in ada` pattern"); + let bad_u64 = |_: ParseIntError| E::custom("unvalid u64"); + + // ignore the first capture, since this is the whole string + let capture = REGEX.captures_iter(v).next().ok_or_else(bad_pattern)?; + let value = capture.get(1).ok_or_else(bad_pattern)?; + value.as_str().parse().map_err(bad_u64) + } +} + +impl From for AdaRewards { + fn from(v: u64) -> Self { + Self(v) + } +} + +impl From for u64 { + fn from(rewards: AdaRewards) -> Self { + rewards.0 + } +} + +impl Display for AdaRewards { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "${} in ada", self.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(s: &str) -> Result { + let s = format!(r#""{s}""#); + serde_json::from_str(&s) + } + + #[test] + fn can_parse_good_values() { + assert_eq!(parse("$123 in ada").unwrap(), AdaRewards(123)); + assert_eq!(parse("0 ada").unwrap(), AdaRewards(0)); + assert_eq!( + parse("0 ada with some stuff at the end").unwrap(), + AdaRewards(0) + ); + } + + #[test] + fn fails_to_parse_bad_values() { + // missing dollar sign + assert!(parse("123 in ada").is_err()); + // negative number + assert!(parse("$-123 in ada").is_err()); + // fraction + assert!(parse("$123.0 in ada").is_err()); + } +} + +// fn deserialize_rewards<'de, D: Deserializer<'de>>(deserializer: D) -> Result { +// let rewards_str = String::deserialize(deserializer)?; +// +// if rewards_str.starts_with("0 ada") { +// return Ok(0); +// } +// sscanf::scanf!(rewards_str.trim_end(), "${} in {}", String, String) +// // trim all . or , in between numbers +// .map(|(mut amount, _currency)| { +// amount.retain(|c: char| c.is_numeric() && !(matches!(c, '.') || matches!(c, ','))); +// amount +// }) +// .and_then(|s| s.parse().ok()) +// .ok_or_else(|| { +// D::Error::custom(&format!( +// "Unable to read malformed value: '{}'", +// rewards_str +// )) +// }) +// } diff --git a/catalyst-toolbox/src/ideascale/models/de/clean_string.rs b/catalyst-toolbox/src/ideascale/models/de/clean_string.rs new file mode 100644 index 00000000..fc33d232 --- /dev/null +++ b/catalyst-toolbox/src/ideascale/models/de/clean_string.rs @@ -0,0 +1,107 @@ +use std::fmt::Formatter; + +use once_cell::sync::Lazy; +use regex::Regex; +use serde::{ + de::{Error, Visitor}, + Deserialize, Deserializer, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CleanString(String); + +impl<'de> Deserialize<'de> for CleanString { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_str(V) + } +} + +struct V; + +impl<'a> Visitor<'a> for V { + type Value = CleanString; + + fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + formatter.write_str("a string") + } + + fn visit_str(self, v: &str) -> Result + where + E: Error, + { + let s = clean_str(v); + Ok(CleanString(s)) + } +} + +impl From<&str> for CleanString { + fn from(s: &str) -> Self { + CleanString(s.to_string()) + } +} + +impl From for CleanString { + fn from(s: String) -> Self { + CleanString(s) + } +} + +impl ToString for CleanString { + fn to_string(&self) -> String { + self.0.clone() + } +} + +impl AsRef for CleanString { + fn as_ref(&self) -> &str { + &self.0 + } +} + +static REGEX: Lazy = Lazy::new(|| Regex::new("[-*/]").unwrap()); + +pub fn clean_str(s: &str) -> String { + REGEX.replace_all(s, "").to_string() +} + +#[cfg(any(test, feature = "property-test-api"))] +mod tests { + use proptest::{ + arbitrary::{Arbitrary, StrategyFor}, + prelude::*, + strategy::Map, + }; + use serde_json::json; + use test_strategy::proptest; + + use super::*; + + impl Arbitrary for CleanString { + type Parameters = (); + type Strategy = Map, fn(String) -> Self>; + + fn arbitrary_with((): Self::Parameters) -> Self::Strategy { + any::().prop_map(|s| CleanString(clean_str(&s))) + } + } + + fn parse(s: &str) -> CleanString { + let s = format!(r#""{s}""#); + serde_json::from_str(&s).unwrap() + } + + #[test] + fn correctly_formats_strings() { + assert_eq!(parse("hello"), CleanString::from("hello")); + assert_eq!(parse("h*e-l/lo"), CleanString::from("hello")); + } + + #[proptest] + fn any_string_deserializes_to_clean_string(s: String) { + let json = json!(s); + let _: CleanString = serde_json::from_value(json).unwrap(); + } +} diff --git a/catalyst-toolbox/src/ideascale/models/de.rs b/catalyst-toolbox/src/ideascale/models/de/mod.rs similarity index 60% rename from catalyst-toolbox/src/ideascale/models/de.rs rename to catalyst-toolbox/src/ideascale/models/de/mod.rs index 521f818b..a35c4c6e 100644 --- a/catalyst-toolbox/src/ideascale/models/de.rs +++ b/catalyst-toolbox/src/ideascale/models/de/mod.rs @@ -1,9 +1,10 @@ -use serde::de::Error; use serde::{Deserialize, Deserializer}; -use std::fmt::{Display, Formatter}; -#[derive(Debug, Deserialize, Clone, Ord, PartialOrd, Eq, PartialEq)] -pub struct AdaRewards(#[serde(deserialize_with = "deserialize_rewards")] u64); +mod ada_rewards; +pub use ada_rewards::AdaRewards; + +mod clean_string; +pub use clean_string::{CleanString, clean_str}; #[derive(Debug, Deserialize, Clone)] pub struct Challenge { @@ -94,62 +95,17 @@ pub struct Stage { pub assessment_id: u32, } -#[derive(Debug, Deserialize, Clone)] -pub struct CleanString(#[serde(deserialize_with = "deserialize_clean_string")] String); - -impl Funnel { - pub fn is_community(&self) -> bool { - self.title.as_ref().contains("Community Setting") - } -} - -impl From for AdaRewards { - fn from(v: u64) -> Self { - Self(v) - } -} - -impl ToString for CleanString { - fn to_string(&self) -> String { - self.0.clone() - } -} - -impl AsRef for CleanString { - fn as_ref(&self) -> &str { - &self.0 - } -} - -impl From for u64 { - fn from(rewards: AdaRewards) -> Self { - rewards.0 - } -} - -impl Display for AdaRewards { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} fn deserialize_approved<'de, D: Deserializer<'de>>(deserializer: D) -> Result { let approved = String::deserialize(deserializer)?; Ok(matches!(approved.as_str(), "approved")) } - -pub fn clean_str(s: &str) -> String { - let mut result = s.to_string(); - result.retain(|c| !matches!(c, '*' | '-' | '/')); - result +impl Funnel { + pub fn is_community(&self) -> bool { + self.title.as_ref().contains("Community Setting") + } } -fn deserialize_clean_string<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result { - let rewards_str = String::deserialize(deserializer)?; - Ok(clean_str(&rewards_str)) -} fn deserialize_clean_challenge_title<'de, D: Deserializer<'de>>( deserializer: D, @@ -165,24 +121,3 @@ fn deserialize_clean_challenge_title<'de, D: Deserializer<'de>>( Ok(rewards_str) } -fn deserialize_rewards<'de, D: Deserializer<'de>>(deserializer: D) -> Result { - let rewards_str = String::deserialize(deserializer)?; - - // input is not standarized, hack an early return if it is just 0 ada - if rewards_str.starts_with("0 ada") { - return Ok(0); - } - sscanf::scanf!(rewards_str.trim_end(), "${} in {}", String, String) - // trim all . or , in between numbers - .map(|(mut amount, _currency)| { - amount.retain(|c: char| c.is_numeric() && !(matches!(c, '.') || matches!(c, ','))); - amount - }) - .and_then(|s| s.parse().ok()) - .ok_or_else(|| { - D::Error::custom(&format!( - "Unable to read malformed value: '{}'", - rewards_str - )) - }) -} diff --git a/catalyst-toolbox/tests/tally/generator.rs b/catalyst-toolbox/tests/tally/generator.rs index ea79bf13..4da7a70e 100644 --- a/catalyst-toolbox/tests/tally/generator.rs +++ b/catalyst-toolbox/tests/tally/generator.rs @@ -45,7 +45,7 @@ fn account_from_slice

( } } -impl<'a> VoteRoundGenerator { +impl VoteRoundGenerator { pub fn new(blockchain: TestBlockchain) -> Self { let TestBlockchain { config, From 567ee8c517daf1552731aed57aa990808734a0c6 Mon Sep 17 00:00:00 2001 From: cameron Date: Wed, 20 Jul 2022 21:02:50 +0100 Subject: [PATCH 02/10] more types --- catalyst-toolbox/src/ideascale/mod.rs | 2 +- .../src/ideascale/models/de/approval.rs | 60 +++++++++++++++++++ .../src/ideascale/models/de/clean_string.rs | 2 +- .../src/ideascale/models/de/mod.rs | 18 +++--- 4 files changed, 69 insertions(+), 13 deletions(-) create mode 100644 catalyst-toolbox/src/ideascale/models/de/approval.rs diff --git a/catalyst-toolbox/src/ideascale/mod.rs b/catalyst-toolbox/src/ideascale/mod.rs index b91edbe9..4c2cc14e 100644 --- a/catalyst-toolbox/src/ideascale/mod.rs +++ b/catalyst-toolbox/src/ideascale/mod.rs @@ -79,7 +79,7 @@ pub fn fetch_all( // TODO: Handle error better here .flat_map(Result::unwrap) // filter out non approved or staged proposals - .filter(|p| p.approved && filter_proposal_by_stage_type(&p.stage_type, &matches)) + .filter(|p| p.approved.as_bool() && filter_proposal_by_stage_type(&p.stage_type, &matches)) .filter(|p| !excluded_proposals.contains(&p.proposal_id)) .collect(); diff --git a/catalyst-toolbox/src/ideascale/models/de/approval.rs b/catalyst-toolbox/src/ideascale/models/de/approval.rs new file mode 100644 index 00000000..76fbfb02 --- /dev/null +++ b/catalyst-toolbox/src/ideascale/models/de/approval.rs @@ -0,0 +1,60 @@ +use serde::{de::Visitor, Deserialize, Deserializer}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Approval { + Approved, + NotApproved, +} + +impl Approval { + pub fn as_bool(&self) -> bool { + (*self).into() + } +} + +impl From for Approval { + fn from(b: bool) -> Self { + match b { + true => Approval::Approved, + false => Approval::NotApproved, + } + } +} + +impl From for bool { + fn from(b: Approval) -> Self { + match b { + Approval::Approved => true, + Approval::NotApproved => false, + } + } +} + +impl<'de> Deserialize<'de> for Approval { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_str(V) + } +} + +struct V; + +impl Visitor<'_> for V { + type Value = Approval; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a string representing an approval status (with ths string \"approved\" meaning `Approved`, and all other strings being `NotApproved`)") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + match v { + "approved" => Ok(Approval::Approved), + _ => Ok(Approval::NotApproved), + } + } +} diff --git a/catalyst-toolbox/src/ideascale/models/de/clean_string.rs b/catalyst-toolbox/src/ideascale/models/de/clean_string.rs index fc33d232..4d81ddb0 100644 --- a/catalyst-toolbox/src/ideascale/models/de/clean_string.rs +++ b/catalyst-toolbox/src/ideascale/models/de/clean_string.rs @@ -8,7 +8,7 @@ use serde::{ }; #[derive(Debug, Clone, PartialEq, Eq)] -pub struct CleanString(String); +pub struct CleanString(pub String); impl<'de> Deserialize<'de> for CleanString { fn deserialize(deserializer: D) -> Result diff --git a/catalyst-toolbox/src/ideascale/models/de/mod.rs b/catalyst-toolbox/src/ideascale/models/de/mod.rs index a35c4c6e..757f77bc 100644 --- a/catalyst-toolbox/src/ideascale/models/de/mod.rs +++ b/catalyst-toolbox/src/ideascale/models/de/mod.rs @@ -4,7 +4,10 @@ mod ada_rewards; pub use ada_rewards::AdaRewards; mod clean_string; -pub use clean_string::{CleanString, clean_str}; +pub use clean_string::{clean_str, CleanString}; + +mod approval; +pub use approval::Approval; #[derive(Debug, Deserialize, Clone)] pub struct Challenge { @@ -68,8 +71,8 @@ pub struct Proposal { #[serde(alias = "campaignId")] pub challenge_id: u32, - #[serde(alias = "flag", deserialize_with = "deserialize_approved")] - pub approved: bool, + #[serde(alias = "flag")] + pub approved: Approval, } #[derive(Debug, Deserialize, Clone)] @@ -95,18 +98,12 @@ pub struct Stage { pub assessment_id: u32, } - -fn deserialize_approved<'de, D: Deserializer<'de>>(deserializer: D) -> Result { - let approved = String::deserialize(deserializer)?; - Ok(matches!(approved.as_str(), "approved")) -} impl Funnel { pub fn is_community(&self) -> bool { - self.title.as_ref().contains("Community Setting") + self.title.0.contains("Community Setting") } } - fn deserialize_clean_challenge_title<'de, D: Deserializer<'de>>( deserializer: D, ) -> Result { @@ -120,4 +117,3 @@ fn deserialize_clean_challenge_title<'de, D: Deserializer<'de>>( } Ok(rewards_str) } - From 783c9f5bbfe315944cfb3c3e0d3a93a3a7419725 Mon Sep 17 00:00:00 2001 From: cameron Date: Thu, 21 Jul 2022 11:16:38 +0100 Subject: [PATCH 03/10] remove unneeded depdencency and improve parse int error --- Cargo.lock | 7 -- catalyst-toolbox/Cargo.toml | 1 - catalyst-toolbox/src/ideascale/mod.rs | 2 +- .../src/ideascale/models/de/ada_rewards.rs | 2 +- .../ideascale/models/de/challenge_title.rs | 68 +++++++++++++++++++ .../src/ideascale/models/de/mod.rs | 23 ++----- 6 files changed, 76 insertions(+), 27 deletions(-) create mode 100644 catalyst-toolbox/src/ideascale/models/de/challenge_title.rs diff --git a/Cargo.lock b/Cargo.lock index 9bcb039e..311fa6da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -508,7 +508,6 @@ dependencies = [ "thor", "time 0.3.11", "tokio", - "unarray", "url", "versionisator", "vit-servicing-station-lib", @@ -5668,12 +5667,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "unarray" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634233a4d474fb493db992470f77c312dd7cf9c967a4a3125ff76d511535e210" - [[package]] name = "unicase" version = "2.6.0" diff --git a/catalyst-toolbox/Cargo.toml b/catalyst-toolbox/Cargo.toml index 0ee49cb8..3faa7d63 100644 --- a/catalyst-toolbox/Cargo.toml +++ b/catalyst-toolbox/Cargo.toml @@ -64,7 +64,6 @@ vit-servicing-station-lib = { git = "https://github.com/input-output-hk/vit-serv env_logger = "0.9" voting-hir = { path = "../voting-hir", features = ["serde"] } fraction = "0.10" -unarray = "0.1" [dev-dependencies] rand_chacha = "0.3" diff --git a/catalyst-toolbox/src/ideascale/mod.rs b/catalyst-toolbox/src/ideascale/mod.rs index 4c2cc14e..4e282274 100644 --- a/catalyst-toolbox/src/ideascale/mod.rs +++ b/catalyst-toolbox/src/ideascale/mod.rs @@ -139,7 +139,7 @@ pub fn build_challenges( id: i.to_string(), rewards_total: c.rewards.to_string(), proposers_rewards: c.rewards.to_string(), - title: c.title.clone(), + title: c.title.as_str().to_string(), highlight: sponsors.get(&c.challenge_url).map(|sponsor| { models::se::Highlight { sponsor: sponsor.clone(), diff --git a/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs b/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs index ef97d130..f6ad94cc 100644 --- a/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs +++ b/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs @@ -46,7 +46,7 @@ impl<'a> Visitor<'a> for V { } let bad_pattern = || E::custom("didn't match `$N in ada` pattern"); - let bad_u64 = |_: ParseIntError| E::custom("unvalid u64"); + let bad_u64 = |e: ParseIntError| E::custom("unvalid u64: {e}"); // ignore the first capture, since this is the whole string let capture = REGEX.captures_iter(v).next().ok_or_else(bad_pattern)?; diff --git a/catalyst-toolbox/src/ideascale/models/de/challenge_title.rs b/catalyst-toolbox/src/ideascale/models/de/challenge_title.rs new file mode 100644 index 00000000..b30be218 --- /dev/null +++ b/catalyst-toolbox/src/ideascale/models/de/challenge_title.rs @@ -0,0 +1,68 @@ +use serde::{de::Visitor, Deserialize}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChallengeTitle(String); + +impl<'de> Deserialize<'de> for ChallengeTitle { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_str(V) + } +} + +struct V; + +impl Visitor<'_> for V { + type Value = ChallengeTitle; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str( + "a string representing the title of a challenge (ignoring any leading `FX: `)", + ) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Ok(ChallengeTitle::new(v)) + } +} + +impl ChallengeTitle { + pub fn new(s: &str) -> Self { + let s = s.trim_start_matches("FX: "); + Self(s.to_string()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl From for String { + fn from(ChallengeTitle(inner): ChallengeTitle) -> Self { + inner + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn parse(s: &str) -> ChallengeTitle { + let json = json!(s); + serde_json::from_value(json).unwrap() + } + + #[test] + fn strips_leading_fx() { + assert_eq!(parse("hello"), ChallengeTitle("hello".into())); + assert_eq!(parse("FX: hello"), ChallengeTitle("hello".into())); + assert_eq!(parse("FX:hello"), ChallengeTitle("FX:hello".into())); + } +} diff --git a/catalyst-toolbox/src/ideascale/models/de/mod.rs b/catalyst-toolbox/src/ideascale/models/de/mod.rs index 757f77bc..a2a241d4 100644 --- a/catalyst-toolbox/src/ideascale/models/de/mod.rs +++ b/catalyst-toolbox/src/ideascale/models/de/mod.rs @@ -1,4 +1,4 @@ -use serde::{Deserialize, Deserializer}; +use serde::Deserialize; mod ada_rewards; pub use ada_rewards::AdaRewards; @@ -9,11 +9,14 @@ pub use clean_string::{clean_str, CleanString}; mod approval; pub use approval::Approval; +mod challenge_title; +pub use challenge_title::ChallengeTitle; + #[derive(Debug, Deserialize, Clone)] pub struct Challenge { pub id: u32, - #[serde(alias = "name", deserialize_with = "deserialize_clean_challenge_title")] - pub title: String, + #[serde(alias = "name")] + pub title: ChallengeTitle, #[serde(alias = "tagline")] pub rewards: AdaRewards, pub description: CleanString, @@ -103,17 +106,3 @@ impl Funnel { self.title.0.contains("Community Setting") } } - -fn deserialize_clean_challenge_title<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result { - let mut rewards_str = String::deserialize(deserializer)?; - // Remove leading `FX: ` - if rewards_str.starts_with('F') { - if let Some(first_space) = rewards_str.find(' ') { - let (_, content) = rewards_str.split_at(first_space + 1); - rewards_str = content.to_string(); - } - } - Ok(rewards_str) -} From 3ccf4c8362d60361ab14ff6ddc3839272b0c083c Mon Sep 17 00:00:00 2001 From: cameron Date: Thu, 21 Jul 2022 11:21:55 +0100 Subject: [PATCH 04/10] actually use the error --- catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs b/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs index f6ad94cc..defa73c2 100644 --- a/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs +++ b/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs @@ -46,7 +46,7 @@ impl<'a> Visitor<'a> for V { } let bad_pattern = || E::custom("didn't match `$N in ada` pattern"); - let bad_u64 = |e: ParseIntError| E::custom("unvalid u64: {e}"); + let bad_u64 = |e: ParseIntError| E::custom(format!("unvalid u64: {e}")); // ignore the first capture, since this is the whole string let capture = REGEX.captures_iter(v).next().ok_or_else(bad_pattern)?; From efbe926d4c55366df19879c7cbc0987a85ade360 Mon Sep 17 00:00:00 2001 From: cameron Date: Fri, 22 Jul 2022 09:11:06 +0100 Subject: [PATCH 05/10] logging messages --- Cargo.lock | 19 +++---------------- catalyst-toolbox/Cargo.toml | 4 ++-- catalyst-toolbox/src/bin/catalyst-toolbox.rs | 2 +- .../src/bin/cli/recovery/votes.rs | 2 +- catalyst-toolbox/src/http/mod.rs | 2 +- catalyst-toolbox/src/http/rate_limit.rs | 3 ++- catalyst-toolbox/src/http/reqwest.rs | 1 + catalyst-toolbox/src/ideascale/fetch.rs | 3 +++ catalyst-toolbox/src/lib.rs | 3 +++ catalyst-toolbox/src/recovery/replay.rs | 2 +- catalyst-toolbox/src/recovery/tally.rs | 2 +- script.sh | 1 + 12 files changed, 20 insertions(+), 24 deletions(-) create mode 100755 script.sh diff --git a/Cargo.lock b/Cargo.lock index 311fa6da..54ce2276 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -469,7 +469,6 @@ dependencies = [ "chain-vote", "color-eyre", "csv", - "env_logger 0.9.0", "fraction", "futures", "gag", @@ -483,7 +482,6 @@ dependencies = [ "jormungandr-integration-tests", "jormungandr-lib", "jortestkit", - "log", "once_cell", "predicates 1.0.8", "proptest 1.0.0 (git+https://github.com/input-output-hk/proptest?branch=master)", @@ -508,6 +506,8 @@ dependencies = [ "thor", "time 0.3.11", "tokio", + "tracing", + "tracing-subscriber", "url", "versionisator", "vit-servicing-station-lib", @@ -1401,19 +1401,6 @@ dependencies = [ "regex", ] -[[package]] -name = "env_logger" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - [[package]] name = "environmental" version = "1.1.3" @@ -4019,7 +4006,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" dependencies = [ - "env_logger 0.7.1", + "env_logger", "log", "rand 0.7.3", "rand_core 0.5.1", diff --git a/catalyst-toolbox/Cargo.toml b/catalyst-toolbox/Cargo.toml index 3faa7d63..55b669a9 100644 --- a/catalyst-toolbox/Cargo.toml +++ b/catalyst-toolbox/Cargo.toml @@ -37,7 +37,6 @@ rayon = "1.5" rust_decimal = "1.16" rust_decimal_macros = "1" futures = "0.3" -log = "0.4" once_cell = "1.8" reqwest = { version = "0.11", features = ["blocking", "json"] } rand = "0.8.3" @@ -61,9 +60,10 @@ symmetric-cipher = { git = "https://github.com/input-output-hk/chain-wallet-libs graphql_client = { version = "0.10" } gag = "1" vit-servicing-station-lib = { git = "https://github.com/input-output-hk/vit-servicing-station.git", branch = "master" } -env_logger = "0.9" voting-hir = { path = "../voting-hir", features = ["serde"] } fraction = "0.10" +tracing = "0.1" +tracing-subscriber = "0.3" [dev-dependencies] rand_chacha = "0.3" diff --git a/catalyst-toolbox/src/bin/catalyst-toolbox.rs b/catalyst-toolbox/src/bin/catalyst-toolbox.rs index eb7a124e..3e6c4765 100644 --- a/catalyst-toolbox/src/bin/catalyst-toolbox.rs +++ b/catalyst-toolbox/src/bin/catalyst-toolbox.rs @@ -3,7 +3,7 @@ use structopt::StructOpt; pub mod cli; fn main() -> color_eyre::Result<()> { - env_logger::try_init()?; + tracing_subscriber::fmt().init(); color_eyre::install()?; cli::Cli::from_args().exec()?; Ok(()) diff --git a/catalyst-toolbox/src/bin/cli/recovery/votes.rs b/catalyst-toolbox/src/bin/cli/recovery/votes.rs index 5dcf6065..2e0482aa 100644 --- a/catalyst-toolbox/src/bin/cli/recovery/votes.rs +++ b/catalyst-toolbox/src/bin/cli/recovery/votes.rs @@ -81,7 +81,7 @@ fn group_by_voter)>>( .or_insert_with(Vec::new) .push(vote_cast); } - Err(e) => log::error!("Invalid transaction: {}", e), + Err(e) => tracing::error!("Invalid transaction: {}", e), } } } diff --git a/catalyst-toolbox/src/http/mod.rs b/catalyst-toolbox/src/http/mod.rs index f078fc6a..a549b054 100644 --- a/catalyst-toolbox/src/http/mod.rs +++ b/catalyst-toolbox/src/http/mod.rs @@ -2,8 +2,8 @@ use std::marker::PhantomData; use ::reqwest::{blocking::Response, StatusCode}; use color_eyre::eyre::Result; -use log::warn; use serde::Deserialize; +use tracing::warn; use self::{rate_limit::RateLimitClient, reqwest::ReqwestClient}; diff --git a/catalyst-toolbox/src/http/rate_limit.rs b/catalyst-toolbox/src/http/rate_limit.rs index 1be7e52d..97683a24 100644 --- a/catalyst-toolbox/src/http/rate_limit.rs +++ b/catalyst-toolbox/src/http/rate_limit.rs @@ -7,11 +7,12 @@ use governor::{ }; use color_eyre::Report; -use log::debug; +use tracing::debug; use serde::Deserialize; use super::{HttpClient, HttpResponse}; +#[derive(Debug)] pub struct RateLimitClient { inner: T, limiter: Option>, diff --git a/catalyst-toolbox/src/http/reqwest.rs b/catalyst-toolbox/src/http/reqwest.rs index f6a3c292..1fd883ed 100644 --- a/catalyst-toolbox/src/http/reqwest.rs +++ b/catalyst-toolbox/src/http/reqwest.rs @@ -10,6 +10,7 @@ use super::{HttpClient, HttpResponse}; const BASE_IDEASCALE_URL: &str = "https://cardano.ideascale.com/a/rest/v1/"; +#[derive(Debug)] pub struct ReqwestClient { client: Client, base_url: Url, diff --git a/catalyst-toolbox/src/ideascale/fetch.rs b/catalyst-toolbox/src/ideascale/fetch.rs index 783f6ebf..cf4bd2fb 100644 --- a/catalyst-toolbox/src/ideascale/fetch.rs +++ b/catalyst-toolbox/src/ideascale/fetch.rs @@ -9,6 +9,7 @@ pub type Scores = HashMap; pub type Sponsors = HashMap; pub fn get_funds_data(client: &impl HttpClient) -> Result, Report> { + info!("getting funds"); client.get("campaigns/groups")?.json() } @@ -25,10 +26,12 @@ pub fn get_proposals_data( client: &impl HttpClient, challenge_id: u32, ) -> Result, Report> { + info!("getting proposal data"); let path = &format!("campaigns/{}/ideas/0/100000", challenge_id); client.get(path)?.json() } pub fn get_funnels_data_for_fund(client: &impl HttpClient) -> Result, Report> { + info!("getting funnels"); client.get("funnels")?.json() } diff --git a/catalyst-toolbox/src/lib.rs b/catalyst-toolbox/src/lib.rs index ea625625..9633f34e 100644 --- a/catalyst-toolbox/src/lib.rs +++ b/catalyst-toolbox/src/lib.rs @@ -15,3 +15,6 @@ pub mod vote_check; pub mod http; #[cfg(feature = "test-api")] pub mod testing; + +#[macro_use] +extern crate tracing; diff --git a/catalyst-toolbox/src/recovery/replay.rs b/catalyst-toolbox/src/recovery/replay.rs index d5a52fb7..11611a2c 100644 --- a/catalyst-toolbox/src/recovery/replay.rs +++ b/catalyst-toolbox/src/recovery/replay.rs @@ -8,7 +8,7 @@ pub use jcli_lib::utils::{ use jormungandr_lib::interfaces::{ load_persistent_fragments_logs_from_folder_path, VotePlanStatus, }; -use log::warn; +use tracing::warn; use std::io::Write; use std::path::PathBuf; diff --git a/catalyst-toolbox/src/recovery/tally.rs b/catalyst-toolbox/src/recovery/tally.rs index d034d17b..ed863de6 100644 --- a/catalyst-toolbox/src/recovery/tally.rs +++ b/catalyst-toolbox/src/recovery/tally.rs @@ -26,7 +26,7 @@ use jormungandr_lib::{ }, time::SecondsSinceUnixEpoch, }; -use log::{debug, error, trace, warn}; +use tracing::{debug, error, trace, warn}; use std::collections::{HashMap, HashSet}; use std::ops::{Add, Range}; use std::time::{Duration, SystemTime}; diff --git a/script.sh b/script.sh new file mode 100755 index 00000000..4012db15 --- /dev/null +++ b/script.sh @@ -0,0 +1 @@ +cargo run --bin catalyst-toolbox ideascale import --api-token 3ff50be1-084c-47b1-9df5-53aa2e150099 --fund 8 --chain-vote-type private --output-dir . --threshold 450 --fund-goal "Create, fund and deliver the future of Cardano." --excluded-proposals ../catalyst-resources/ideascale/fund8/excluded_proposals.json --tags ../catalyst-resources/ideascale/fund8/tags.json --stages-filters "Assess QA" From e5c9a84cdb0ea10f56688aa1eb21aead0851bf3e Mon Sep 17 00:00:00 2001 From: Cameron Date: Thu, 28 Jul 2022 10:53:27 +0100 Subject: [PATCH 06/10] pr feedback --- .../src/ideascale/models/de/ada_rewards.rs | 21 ------------------- .../src/ideascale/models/de/approval.rs | 4 ++-- .../src/ideascale/models/de/clean_string.rs | 3 +++ script.sh | 1 - 4 files changed, 5 insertions(+), 24 deletions(-) delete mode 100755 script.sh diff --git a/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs b/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs index defa73c2..a7c6cf67 100644 --- a/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs +++ b/catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs @@ -102,24 +102,3 @@ mod tests { assert!(parse("$123.0 in ada").is_err()); } } - -// fn deserialize_rewards<'de, D: Deserializer<'de>>(deserializer: D) -> Result { -// let rewards_str = String::deserialize(deserializer)?; -// -// if rewards_str.starts_with("0 ada") { -// return Ok(0); -// } -// sscanf::scanf!(rewards_str.trim_end(), "${} in {}", String, String) -// // trim all . or , in between numbers -// .map(|(mut amount, _currency)| { -// amount.retain(|c: char| c.is_numeric() && !(matches!(c, '.') || matches!(c, ','))); -// amount -// }) -// .and_then(|s| s.parse().ok()) -// .ok_or_else(|| { -// D::Error::custom(&format!( -// "Unable to read malformed value: '{}'", -// rewards_str -// )) -// }) -// } diff --git a/catalyst-toolbox/src/ideascale/models/de/approval.rs b/catalyst-toolbox/src/ideascale/models/de/approval.rs index 76fbfb02..eacb805e 100644 --- a/catalyst-toolbox/src/ideascale/models/de/approval.rs +++ b/catalyst-toolbox/src/ideascale/models/de/approval.rs @@ -45,14 +45,14 @@ impl Visitor<'_> for V { type Value = Approval; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("a string representing an approval status (with ths string \"approved\" meaning `Approved`, and all other strings being `NotApproved`)") + formatter.write_str("a case-insensitive string representing an approval status (with this string \"approved\" meaning `Approved`, and all other strings being `NotApproved`)") } fn visit_str(self, v: &str) -> Result where E: serde::de::Error, { - match v { + match v.to_lowercase() { "approved" => Ok(Approval::Approved), _ => Ok(Approval::NotApproved), } diff --git a/catalyst-toolbox/src/ideascale/models/de/clean_string.rs b/catalyst-toolbox/src/ideascale/models/de/clean_string.rs index 4d81ddb0..07af2978 100644 --- a/catalyst-toolbox/src/ideascale/models/de/clean_string.rs +++ b/catalyst-toolbox/src/ideascale/models/de/clean_string.rs @@ -7,6 +7,9 @@ use serde::{ Deserialize, Deserializer, }; +/// A newtype wrapper around `String` +/// +/// When deserialized, the following characters are removed: `-`, `*`, `/` #[derive(Debug, Clone, PartialEq, Eq)] pub struct CleanString(pub String); diff --git a/script.sh b/script.sh deleted file mode 100755 index 4012db15..00000000 --- a/script.sh +++ /dev/null @@ -1 +0,0 @@ -cargo run --bin catalyst-toolbox ideascale import --api-token 3ff50be1-084c-47b1-9df5-53aa2e150099 --fund 8 --chain-vote-type private --output-dir . --threshold 450 --fund-goal "Create, fund and deliver the future of Cardano." --excluded-proposals ../catalyst-resources/ideascale/fund8/excluded_proposals.json --tags ../catalyst-resources/ideascale/fund8/tags.json --stages-filters "Assess QA" From 9e9bdce1217136eb3dd42a677ceec475b28c5e11 Mon Sep 17 00:00:00 2001 From: Cameron Date: Thu, 28 Jul 2022 12:37:13 +0100 Subject: [PATCH 07/10] compile error --- .pre-commit-config.yaml | 1 + catalyst-toolbox/src/http/rate_limit.rs | 2 +- catalyst-toolbox/src/ideascale/models/de/approval.rs | 2 +- catalyst-toolbox/src/recovery/replay.rs | 2 +- catalyst-toolbox/src/recovery/tally.rs | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) create mode 120000 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 120000 index 00000000..89d3ce1e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1 @@ +/nix/store/80r7admw5vnzl9zxhbznd1yqlfmajck1-pre-commit-config.json \ No newline at end of file diff --git a/catalyst-toolbox/src/http/rate_limit.rs b/catalyst-toolbox/src/http/rate_limit.rs index 97683a24..8772a5f6 100644 --- a/catalyst-toolbox/src/http/rate_limit.rs +++ b/catalyst-toolbox/src/http/rate_limit.rs @@ -7,8 +7,8 @@ use governor::{ }; use color_eyre::Report; -use tracing::debug; use serde::Deserialize; +use tracing::debug; use super::{HttpClient, HttpResponse}; diff --git a/catalyst-toolbox/src/ideascale/models/de/approval.rs b/catalyst-toolbox/src/ideascale/models/de/approval.rs index eacb805e..87c92291 100644 --- a/catalyst-toolbox/src/ideascale/models/de/approval.rs +++ b/catalyst-toolbox/src/ideascale/models/de/approval.rs @@ -52,7 +52,7 @@ impl Visitor<'_> for V { where E: serde::de::Error, { - match v.to_lowercase() { + match v.to_lowercase().as_ref() { "approved" => Ok(Approval::Approved), _ => Ok(Approval::NotApproved), } diff --git a/catalyst-toolbox/src/recovery/replay.rs b/catalyst-toolbox/src/recovery/replay.rs index 11611a2c..afbd5fd1 100644 --- a/catalyst-toolbox/src/recovery/replay.rs +++ b/catalyst-toolbox/src/recovery/replay.rs @@ -8,9 +8,9 @@ pub use jcli_lib::utils::{ use jormungandr_lib::interfaces::{ load_persistent_fragments_logs_from_folder_path, VotePlanStatus, }; -use tracing::warn; use std::io::Write; use std::path::PathBuf; +use tracing::warn; /// Recover the tally from fragment log files and the initial preloaded block0 binary file. pub struct Replay { diff --git a/catalyst-toolbox/src/recovery/tally.rs b/catalyst-toolbox/src/recovery/tally.rs index ed863de6..b50babd2 100644 --- a/catalyst-toolbox/src/recovery/tally.rs +++ b/catalyst-toolbox/src/recovery/tally.rs @@ -26,10 +26,10 @@ use jormungandr_lib::{ }, time::SecondsSinceUnixEpoch, }; -use tracing::{debug, error, trace, warn}; use std::collections::{HashMap, HashSet}; use std::ops::{Add, Range}; use std::time::{Duration, SystemTime}; +use tracing::{debug, error, trace, warn}; use wallet::{Settings, TransactionBuilder, Wallet}; #[allow(clippy::large_enum_variant)] From 687fd7fe4042d8e7928a6389132536afd352fe1f Mon Sep 17 00:00:00 2001 From: Cameron Date: Thu, 28 Jul 2022 12:51:22 +0100 Subject: [PATCH 08/10] remove old proptest regressions --- .../ideascale/models/de/clean_string.txt | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 catalyst-toolbox/proptest-regressions/ideascale/models/de/clean_string.txt diff --git a/catalyst-toolbox/proptest-regressions/ideascale/models/de/clean_string.txt b/catalyst-toolbox/proptest-regressions/ideascale/models/de/clean_string.txt deleted file mode 100644 index 1ec91551..00000000 --- a/catalyst-toolbox/proptest-regressions/ideascale/models/de/clean_string.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Seeds for failure cases proptest has generated in the past. It is -# automatically read and these particular cases re-run before any -# novel cases are generated. -# -# It is recommended to check this file in to source control so that -# everyone who runs the test benefits from these saved cases. -xx 35f62b501c8f4be5ccc1ca95c85692680d9c5bb51cf31931292962bf805e8142 # shrinks to input = _AnyStringDeserializesToCleanStringArgs { s: "" } -xx 6628679236d41598dac84340f0b668e82989e373457f91e51dbd077d870494a6 # shrinks to input = _AnyStringDeserializesToCleanStringArgs { s: "\"" } From b155c25980eafdcad8cbf90addaf3434daeba810 Mon Sep 17 00:00:00 2001 From: Cameron Date: Thu, 28 Jul 2022 13:55:52 +0100 Subject: [PATCH 09/10] update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 46221986..70938ee1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ qr-code.svg /target /venv +.pre-commit-config.yaml From 595cdd9ed8a7710a4555a0d7982cea3a9df54236 Mon Sep 17 00:00:00 2001 From: Cameron Date: Thu, 28 Jul 2022 13:58:20 +0100 Subject: [PATCH 10/10] update gitignore --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) delete mode 120000 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 120000 index 89d3ce1e..00000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1 +0,0 @@ -/nix/store/80r7admw5vnzl9zxhbznd1yqlfmajck1-pre-commit-config.json \ No newline at end of file