Merge branch 'feature/sink-as-action' into develop

config-format.md

@@ -10,6 +10,7 @@ read_filter_type: newer_than_read # XO. either:
                                   # * keep only the last read entry and filter out all "older" than it
                                   # * notify when the entry is updated
 read_filter_type: not_present_in_read_list # XO. keep a list of all items read and filter out all that are present in it
+template: <name> # copy-paste the contents of $XDG_CONFIG_PATH/fetcher/templates/<name>.yml. Field re-definition overrides the old value. 
 tasks:
   foo:
     tag: <string> # mark the message with a tag. That is usually a hashtag on top of the message or some kind of subscript in it. If a job has multiple tasks, it is automatically set to the task's name
@@ -55,6 +56,17 @@ tasks:
                                                     # * mark_as_read: mark read emails as read
                                                     # * delete: move the emails to the trash bin. Exact behavior depends on the email provider in question. Gmail archives the emails by default instead
     process:  # all actions are optional, so don't need to be marked with O
+      - import: <name> # import a list of actions from $XDG_CONFIG_PATH/fetcher/actions/<name>.yml
+      - sink:
+          discord: # X. Send as a discord message
+            user: <user_id> # X. The user to DM to. This is not a handle (i.e. not User#1234) but rather the ID (see below). 
+            channel: <channel_id> # X. The channel to send messages to
+            # The ID of a user or a channel can be gotten after enabling developer settings in Discord (under Settings -> Advanced) and rightclicking on a user/channel and selecting "Copy ID"
+          telegram: # X
+            chat_id: <chat_id>  # Either the private chat (group/channel) ID that can be gotten using bots or the public handle of a chat. DM aren't supported yet.
+            link_location: <prefer_title|bottom>  # O. Where to put the link. Either as try to put it in the title if it's present, or a separate "Link" button under the message
+          exec: <cmd> # X. Start a process and write the body of the message to its stdin
+          stdout # X. Just print to stdout. Isn't really useful but it is the default when run with --dry-run
       - read_filter # filter out already read entries using `read_filter_type` stradegy
       - take: # take `num` entries from either the newest or the oldest and ignore the rest
           <from_newest|from_oldest>: <int>
@@ -162,15 +174,8 @@ tasks:
       # debug related actions:
       - caps # make the message title uppercase
       - debug_print # debug print the entire contents of the entry
+
 sink:
-  discord: # X. Send as a discord message
-    user: <user_id> # X. The user to DM to. This is not a handle (i.e. not User#1234) but rather the ID (see below). 
-    channel: <channel_id> # X. The channel to send messages to
-    # The ID of a user or a channel can be gotten after enabling developer settings in Discord (under Settings -> Advanced) and rightclicking on a user/channel and selecting "Copy ID"
-  telegram: # X
-    chat_id: <chat_id>  # Either the private chat (group/channel) ID that can be gotten using bots or the public handle of a chat. DM aren't supported yet.
-    link_location: <prefer_title|bottom>  # O. Where to put the link. Either as try to put it in the title if it's present, or a separate "Link" button under the message
-  exec: <cmd> # X. Start a process and write the body of the message to its stdin
-  stdout # X. Just print to stdout. Isn't really useful but it is the default when run with --dry-run
+  ... # same as process: sink. Just appends itself to the process list. This is useful when the process list is set in a template and thus can't be overriden
 ```
 

fetcher-config/src/error.rs

@@ -4,6 +4,8 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
+pub type Result<T, E = Error> = std::result::Result<T, E>;
+
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
 	#[error(transparent)]
@@ -27,6 +29,9 @@ pub enum Error {
 	#[error("Discord bot token isn't set up")]
 	DiscordBotTokenMissing,
 
+	#[error("Importing is unavailable")]
+	ImportingUnavailable,
+
 	#[error("Wrong Google OAuth2 token")]
 	GoogleOAuth2WrongToken(#[from] fetcher_core::auth::google::GoogleOAuth2Error),
 

fetcher-config/src/jobs/action.rs

@@ -7,6 +7,7 @@
 pub mod contains;
 pub mod extract;
 pub mod html;
+pub mod import;
 pub mod json;
 pub mod remove_html;
 pub mod replace;
@@ -17,9 +18,11 @@ pub mod trim;
 pub mod use_as;
 
 use self::{
-	contains::Contains, extract::Extract, html::Html, json::Json, remove_html::RemoveHtml,
-	replace::Replace, set::Set, shorten::Shorten, take::Take, trim::Trim, use_as::Use,
+	contains::Contains, extract::Extract, html::Html, import::Import, json::Json,
+	remove_html::RemoveHtml, replace::Replace, set::Set, shorten::Shorten, take::Take, trim::Trim,
+	use_as::Use,
 };
+use super::{external_data::ProvideExternalData, sink::Sink};
 use crate::Error;
 use fetcher_core::{
 	action::{
@@ -33,6 +36,8 @@ use fetcher_core::{
 };
 
 use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+use tokio::sync::RwLock;
 
 #[derive(Deserialize, Serialize, Clone, Debug)]
 #[serde(rename_all = "snake_case", deny_unknown_fields)]
@@ -58,6 +63,10 @@ pub enum Action {
 	Replace(Replace),
 	Extract(Extract),
 	RemoveHtml(RemoveHtml),
+
+	// other
+	Sink(Sink),
+	Import(Import),
 }
 
 // TODO: add media
@@ -73,9 +82,14 @@ pub enum Field {
 }
 
 impl Action {
-	pub fn parse<RF>(self, rf: Option<RF>) -> Result<Option<Vec<CAction>>, Error>
+	pub fn parse<RF, D>(
+		self,
+		rf: Option<Arc<RwLock<RF>>>,
+		external: &D,
+	) -> Result<Option<Vec<CAction>>, Error>
 	where
 		RF: CReadFilter + 'static,
+		D: ProvideExternalData + ?Sized,
 	{
 		macro_rules! transform {
 			($tr:expr) => {
@@ -121,6 +135,14 @@ impl Action {
 			Action::Replace(x) => transform!(x.parse()?),
 			Action::Extract(x) => transform!(x.parse()?),
 			Action::RemoveHtml(x) => x.parse()?,
+
+			// other
+			Action::Sink(x) => vec![CAction::Sink(x.parse(external)?)],
+			Action::Import(x) => match x.parse(rf, external) {
+				Ok(Some(v)) => v,
+				// FIXME
+				other => return other,
+			},
 		};
 
 		Ok(Some(act))

fetcher-config/src/jobs/action/html/query.rs

@@ -4,7 +4,7 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/.
  */
 
-use crate::Error;
+use crate::error::Error;
 use fetcher_core::{
 	action::{transform::entry::html::query as c_query, transform::field::Replace as CReplace},
 	utils::OptionExt,

fetcher-config/src/jobs/action/import.rs

@@ -0,0 +1,49 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+use crate::{
+	error::{Error, Result},
+	jobs::external_data::{ExternalDataResult, ProvideExternalData},
+};
+use fetcher_core::{action::Action as CAction, read_filter::ReadFilter as CReadFilter};
+
+use itertools::process_results;
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+#[derive(Deserialize, Serialize, Clone, Debug)]
+pub struct Import(pub String);
+
+impl Import {
+	#[allow(clippy::needless_pass_by_value)]
+	pub fn parse<RF, D>(
+		self,
+		rf: Option<Arc<RwLock<RF>>>,
+		external: &D,
+	) -> Result<Option<Vec<CAction>>>
+	where
+		RF: CReadFilter + 'static,
+		D: ProvideExternalData + ?Sized,
+	{
+		match external.import(&self.0) {
+			ExternalDataResult::Ok(x) => {
+				let v =
+					process_results(x.into_iter().map(|x| x.parse(rf.clone(), external)), |i| {
+						i.flatten(/* option */).flatten(/* inner vec */).collect::<Vec<_>>()
+					})?;
+
+				if v.is_empty() {
+					Ok(None)
+				} else {
+					Ok(Some(v))
+				}
+			}
+			ExternalDataResult::Unavailable => Err(Error::ImportingUnavailable),
+			ExternalDataResult::Err(e) => Err(e.into()),
+		}
+	}
+}

fetcher-config/src/jobs/external_data.rs

@@ -5,15 +5,17 @@
  */
 
 use super::{
+	action::Action,
 	named::{JobName, TaskName},
 	read_filter::Kind as ReadFilterKind,
 };
 use fetcher_core::{
-	self as fcore, read_filter::ReadFilter as CReadFilter, task::entry_to_msg_map::EntryToMsgMap,
+	auth as c_auth, read_filter::ReadFilter as CReadFilter, task::entry_to_msg_map::EntryToMsgMap,
 	utils::DisplayDebug,
 };
 
 use std::{
+	error::Error as StdError,
 	fmt::{Debug, Display},
 	io,
 	path::Path,
@@ -32,7 +34,7 @@ pub trait ProvideExternalData {
 		ExternalDataResult::Unavailable
 	}
 
-	fn google_oauth2(&self) -> ExternalDataResult<fcore::auth::Google> {
+	fn google_oauth2(&self) -> ExternalDataResult<c_auth::Google> {
 		ExternalDataResult::Unavailable
 	}
 	fn email_password(&self) -> ExternalDataResult<String> {
@@ -61,6 +63,11 @@ pub trait ProvideExternalData {
 	) -> ExternalDataResult<EntryToMsgMap> {
 		ExternalDataResult::Unavailable
 	}
+
+	/// import action `name`
+	fn import(&self, _name: &str) -> ExternalDataResult<Vec<Action>> {
+		ExternalDataResult::Unavailable
+	}
 }
 
 #[derive(thiserror::Error, Debug)]
@@ -70,12 +77,22 @@ pub enum ExternalDataError {
 		source: io::Error,
 		payload: Option<Box<dyn DisplayDebug + Send + Sync>>,
 	},
+
 	#[error("Incompatible read filter types: in config: \"{expected}\" and found: \"{found}\"{}{}", .payload.is_some().then_some(": ").unwrap_or_default(), if let Some(p) = payload.as_ref() { p as &dyn Display } else { &"" })]
 	ReadFilterIncompatibleTypes {
 		expected: ReadFilterKind,
 		found: ReadFilterKind,
 		payload: Option<Box<dyn DisplayDebug + Send + Sync>>,
 	},
+
+	#[error("Action \"{}\" not found", .0)]
+	ActionNotFound(String),
+
+	#[error("Can't parse action \"{name}\": {err}")]
+	ActionParsingError {
+		name: String,
+		err: Box<dyn StdError + Send + Sync>,
+	},
 }
 
 impl<T, E> From<Result<T, E>> for ExternalDataResult<T, E> {

fetcher-config/src/jobs/job.rs

@@ -35,8 +35,8 @@ pub struct Job {
 	pub source: Option<Source>,
 	#[serde(rename = "process")]
 	pub actions: Option<Vec<Action>>,
-	pub sink: Option<Sink>,
 	pub entry_to_msg_map_enabled: Option<bool>,
+	pub sink: Option<Sink>,
 
 	pub tasks: Option<HashMap<TaskName, Task>>,
 	pub refresh: Option<TimePoint>,
@@ -65,8 +65,8 @@ impl Job {
 					tag: self.tag,
 					source: self.source,
 					actions: self.actions,
-					sink: self.sink,
 					entry_to_msg_map_enabled: self.entry_to_msg_map_enabled,
+					sink: self.sink,
 				};
 
 				let job = CJob {
@@ -113,13 +113,13 @@ impl Job {
 				task.actions = self.actions.clone();
 			}
 
-			if task.sink.is_none() {
-				task.sink = self.sink.clone();
-			}
-
 			if task.entry_to_msg_map_enabled.is_none() {
 				task.entry_to_msg_map_enabled = self.entry_to_msg_map_enabled;
 			}
+
+			if task.sink.is_none() {
+				task.sink = self.sink.clone();
+			}
 		}
 
 		// FIXME: broken. Filtering can remove tasks from the tasks map. Then, when checking if we should pass the task name as a tag, we ignore the fact that we could've had more tasks in the job and skip the tag which we shouldn't do

fetcher-config/src/jobs/sink.rs

@@ -35,11 +35,4 @@ impl Sink {
 			Self::Stdout => Box::new(CStdout {}),
 		})
 	}
-
-	pub fn has_message_id_support(&self) -> bool {
-		match self {
-			Self::Telegram(_) => true,
-			Self::Discord(_) | Self::Exec(_) | Self::Stdout => false, // TODO: implement message id support for Discord
-		}
-	}
 }

fetcher-config/src/jobs/source.rs

@@ -74,4 +74,13 @@ impl Source {
 			Self::AlwaysErrors => Box::new(CAlwaysErrors),
 		})
 	}
+
+	pub fn supports_replies(&self) -> bool {
+		// Source::Email will support replies in the future
+		#[allow(clippy::match_like_matches_macro)]
+		match self {
+			Self::Twitter(_) => true,
+			_ => false,
+		}
+	}
 }

fetcher-config/src/jobs/source/http.rs

@@ -10,7 +10,6 @@ use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, OneOrMany};
 use url::Url;
 
-// TODO: use a map
 #[serde_as]
 #[derive(Deserialize, Serialize, Clone, Debug)]
 #[serde(transparent)]