Skip to content

Commit

Permalink
Add all urls for a site (#458)
Browse files Browse the repository at this point in the history 
- Add button to add all urls for a site
- Add validation check, local url check and processing icon

---------

Co-authored-by: Joel Bredeson <joel@spyglass.fyi>
  • Loading branch information
@travolin
travolin and Joel Bredeson committed May 20, 2023
1 parent 2ae9f28 commit 37601bf
Show file tree
Hide file tree
Showing 4 changed files with 194 additions and 29 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions apps/web/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ strum = "0.24"
strum_macros = "0.24"
thiserror = "1.0"
ui-components = { path = "../../crates/ui-components" }
url = "2.3"
wasm-bindgen = "0.2.83"
wasm-bindgen-futures = "0.4.33"
serde-wasm-bindgen = "0.5"
Expand Down
44 changes: 43 additions & 1 deletion apps/web/src/client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ pub enum LensAddDocType {
/// Token is used to download the document from GDrive.
GDrive { token: String },
/// Normal, web accessible URL.
WebUrl,
WebUrl { include_all_suburls: bool },
}

#[derive(Clone, Debug, Deserialize, Serialize)]
Expand Down Expand Up @@ -247,6 +247,15 @@ pub struct GetLensSourceResponse {
pub results: Vec<LensSource>,
}

#[derive(Deserialize)]
pub struct SourceValidationResponse {
pub url: String,
pub url_count: u64,
pub has_sitemap: Option<bool>,
pub is_valid: bool,
pub validation_msg: Option<String>,
}

#[derive(Error, Debug)]
pub enum ApiError {
#[error("You need to sign in.")]
Expand Down Expand Up @@ -365,6 +374,39 @@ impl ApiClient {
}
}

pub async fn validate_lens_source(
&self,
lens: &str,
request: &LensAddDocument,
) -> Result<SourceValidationResponse, ApiError> {
match &self.token {
Some(token) => {
let resp = self
.client
.post(format!(
"{}/user/lenses/{}/validate/source",
self.endpoint, lens
))
.bearer_auth(token)
.json(request)
.send()
.await?;

match resp.error_for_status_ref() {
Ok(_) => match resp.json::<SourceValidationResponse>().await {
Ok(response) => Ok(response),
Err(msg) => Err(ApiError::Other(msg.to_string())),
},
Err(err) => match resp.json::<ApiErrorMessage>().await {
Ok(msg) => Err(ApiError::ClientError(msg)),
Err(_) => Err(ApiError::RequestError(err)),
},
}
}
None => Err(ApiError::Unauthorized),
}
}

pub async fn lens_update(&self, lens: &str, display_name: &str) -> Result<(), ApiError> {
match &self.token {
Some(token) => {
Expand Down
177 changes: 149 additions & 28 deletions apps/web/src/pages/lens_edit.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ use wasm_bindgen::{
};
use wasm_bindgen_futures::spawn_local;
use web_sys::HtmlInputElement;
use yew::html::Scope;
use yew::prelude::*;
use yew_router::scope_ext::RouterScopeExt;

use crate::{
client::{
ApiError, GetLensSourceResponse, Lens, LensAddDocType, LensAddDocument, LensDocType,
LensSource,
ApiClient, ApiError, GetLensSourceResponse, Lens, LensAddDocType, LensAddDocument,
LensDocType, LensSource,
},
AuthStatus,
};
Expand Down Expand Up @@ -55,6 +56,7 @@ pub struct CreateLensPage {

pub auth_status: AuthStatus,
pub add_url_error: Option<String>,
pub processing_action: Option<Action>,
pub _context_listener: ContextHandle<AuthStatus>,
pub _query_debounce: Option<JsValue>,
pub _name_input_ref: NodeRef,
Expand All @@ -68,9 +70,16 @@ pub struct CreateLensProps {
pub onupdate: Callback<()>,
}

pub enum Action {
AddSingleUrl,
AddAllUrls,
}

pub enum Msg {
AddUrl,
AddUrl { include_all: bool },
AddUrlError(String),
ClearUrlError,
Processing(Option<Action>),
FilePicked { token: String, url: String },
Reload,
ReloadSources(usize),
Expand Down Expand Up @@ -106,6 +115,7 @@ impl Component for CreateLensPage {
is_loading_lens_sources: false,
auth_status,
add_url_error: None,
processing_action: None,
_context_listener: context_listener,
_query_debounce: None,
_name_input_ref: NodeRef::default(),
Expand Down Expand Up @@ -135,42 +145,83 @@ impl Component for CreateLensPage {
fn update(&mut self, ctx: &Context<Self>, msg: Self::Message) -> bool {
let link = ctx.link();
match msg {
Msg::AddUrl => {
Msg::AddUrl { include_all } => {
if let Some(node) = self._url_input_ref.cast::<HtmlInputElement>() {
let url = node.value();
node.set_value("");

let new_source = LensAddDocument {
url,
doc_type: LensAddDocType::WebUrl,
};

// Add to lens
let auth_status = self.auth_status.clone();
let identifier = self.lens_identifier.clone();
let link = link.clone();
spawn_local(async move {
let api = auth_status.get_client();
if let Err(err) = api.lens_add_source(&identifier, &new_source).await {
log::error!("error adding url source: {err}");
match err {
ApiError::ClientError(msg) => {
link.send_message(Msg::AddUrlError(msg.message))

if let Err(_err) = url::Url::parse(&url) {
link.send_message(Msg::AddUrlError("Invalid Url".to_string()));
} else {
let new_source = LensAddDocument {
url,
doc_type: LensAddDocType::WebUrl {
include_all_suburls: include_all,
},
};
// Add to lens
let auth_status = self.auth_status.clone();
let identifier = self.lens_identifier.clone();
let link = link.clone();

if include_all {
spawn_local(async move {
link.send_message(Msg::Processing(Some(Action::AddAllUrls)));
let api = auth_status.get_client();
match api.validate_lens_source(&identifier, &new_source).await {
Ok(response) => {
if response.is_valid {
node.set_value("");
add_lens_source(&api, &new_source, &identifier, link)
.await;
} else if let Some(error_msg) = response.validation_msg {
link.send_message_batch(vec![
Msg::Processing(None),
Msg::AddUrlError(error_msg),
])
} else {
link.send_message_batch(vec![
Msg::Processing(None),
Msg::AddUrlError(
"Unknown error adding url".to_string(),
),
])
}
}
Err(error) => {
log::error!("Unknown error adding url {:?}", error);
link.send_message_batch(vec![
Msg::Processing(None),
Msg::AddUrlError(
"Unknown error adding url".to_string(),
),
])
}
}
_ => link.send_message(Msg::AddUrlError(err.to_string())),
};
})
} else {
// Reload data if successful
link.send_message(Msg::Reload);
node.set_value("");
spawn_local(async move {
link.send_message(Msg::Processing(Some(Action::AddSingleUrl)));
let api = auth_status.get_client();
add_lens_source(&api, &new_source, &identifier, link).await;
});
}
});
}
}
true
}
Msg::AddUrlError(msg) => {
self.add_url_error = Some(msg);
true
}
Msg::ClearUrlError => {
self.add_url_error = None;
true
}
Msg::Processing(action) => {
self.processing_action = action;
true
}
Msg::FilePicked { token, url } => {
let new_source = LensAddDocument {
url,
Expand Down Expand Up @@ -331,6 +382,48 @@ impl Component for CreateLensPage {
.map(|x| html! { <LensSourceComponent source={x.clone()} /> })
.collect::<Html>();

let add_url_actions = if let Some(action) = &self.processing_action {
match action {
Action::AddAllUrls => {
html! {
<>
<Btn disabled=true>{"Add data from URL"}</Btn>
<Btn disabled=true>
<icons::RefreshIcon
classes="mr-1"
width="w-3"
height="h-3"
animate_spin=true/>
{"Add all URLs from Site"}
</Btn>
</>
}
}
Action::AddSingleUrl => {
html! {
<>
<Btn disabled=true>
<icons::RefreshIcon
classes="mr-1"
width="w-3"
height="h-3"
animate_spin=true/>
{"Add data from URL"}
</Btn>
<Btn disabled=true>{"Add all URLs from Site"}</Btn>
</>
}
}
}
} else {
html! {
<>
<Btn onclick={link.callback(|_| Msg::AddUrl {include_all: false})}>{"Add data from URL"}</Btn>
<Btn onclick={link.callback(|_| Msg::AddUrl {include_all: true} )}>{"Add all URLs from Site"}</Btn>
</>
}
};

let is_loading_sources = self.is_loading_lens_sources;
html! {
<div>
Expand Down Expand Up @@ -363,7 +456,7 @@ impl Component for CreateLensPage {
class="rounded p-2 text-sm text-neutral-800"
placeholder="https://example.com"
/>
<Btn onclick={link.callback(|_| Msg::AddUrl)}>{"Add data from URL"}</Btn>
{add_url_actions}
<div class="text-sm text-red-700">{self.add_url_error.clone()}</div>
</div>
<div><Btn onclick={link.callback(|_| Msg::OpenCloudFilePicker)}>{"Add data from Google Drive"}</Btn></div>
Expand Down Expand Up @@ -452,3 +545,31 @@ fn lens_source_comp(props: &LensSourceComponentProps) -> Html {
</div>
}
}

async fn add_lens_source(
api: &ApiClient,
new_source: &LensAddDocument,
identifier: &str,
link: Scope<CreateLensPage>,
) {
if let Err(err) = api.lens_add_source(identifier, new_source).await {
log::error!("error adding url source: {err}");
match err {
ApiError::ClientError(msg) => {
link.send_message_batch(vec![Msg::Processing(None), Msg::AddUrlError(msg.message)])
}
_ => link.send_message_batch(vec![
Msg::Processing(None),
Msg::AddUrlError(err.to_string()),
]),
};
} else {
link.send_message(Msg::ClearUrlError);
// Reload data if successful
link.send_message_batch(vec![
Msg::Processing(None),
Msg::ClearUrlError,
Msg::ReloadSources(0),
]);
}
}

0 comments on commit 37601bf

Please sign in to comment.