Skip to content

Commit

Permalink
Revert "Remove rayon and stdout lock"
Browse files Browse the repository at this point in the history
This reverts commit 9e2a500.
  • Loading branch information
mre committed Dec 10, 2021
1 parent 9e2a500 commit 8e2c676
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 6 deletions.
11 changes: 11 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 8 additions & 4 deletions lychee-bin/src/commands/dump.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::io::{self, Write};
use std::io::{self, StdoutLock, Write};

use lychee_lib::Result;
use lychee_lib::{Client, Request};
Expand All @@ -11,6 +11,10 @@ pub(crate) async fn dump<'a, S>(client: Client, requests: S, verbose: bool) -> R
where
S: futures::Stream<Item = Result<Request>>,
{
// Lock stdout for better performance
let stdout = io::stdout();
let mut handle = stdout.lock();

tokio::pin!(requests);

while let Some(request) = requests.next().await {
Expand All @@ -24,7 +28,7 @@ where
// See https://github.com/rust-lang/rust/issues/46016
// This can occur when piping the output of lychee
// to another program like `grep`.
if let Err(e) = write(&request, verbose) {
if let Err(e) = write(&mut handle, &request, verbose) {
if e.kind() != io::ErrorKind::BrokenPipe {
eprintln!("{}", e);
return Ok(ExitCode::UnexpectedFailure);
Expand All @@ -38,11 +42,11 @@ where
/// Dump request to stdout
/// Only print source in verbose mode. This way the normal link output
/// can be fed into another tool without data mangling.
fn write(request: &Request, verbose: bool) -> io::Result<()> {
fn write(handle: &mut StdoutLock<'_>, request: &Request, verbose: bool) -> io::Result<()> {
let output = if verbose {
request.to_string()
} else {
request.uri.to_string()
};
writeln!(io::stdout(), "{}", output)
writeln!(*handle, "{}", output)
}
2 changes: 2 additions & 0 deletions lychee-lib/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ cached = "0.26.2"
once_cell = "1.8.0"
thiserror = "1.0"
futures = "0.3.18"
rayon = "1.5.1"
ellipse = "0.2.0"

[dependencies.par-stream]
version = "0.7.0"
Expand Down
11 changes: 9 additions & 2 deletions lychee-lib/src/helpers/request.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
use ellipse::Ellipse;
use html5ever::tendril::StrTendril;
use log::info;
use percent_encoding::percent_decode_str;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use reqwest::Url;
use std::{
collections::HashSet,
Expand All @@ -15,6 +17,8 @@ use crate::{
Base, ErrorKind, Input, Request, Result, Uri,
};

const MAX_SOURCE_LEN: usize = 100;

/// Create requests out of the collected URLs.
/// Only keeps "valid" URLs. This filters out anchors for example.
pub(crate) fn create(
Expand All @@ -33,14 +37,17 @@ pub(crate) fn create(
};

let requests: Result<Vec<Option<Request>>> = uris
.into_iter()
.into_par_iter()
.map(|raw_uri| {
let is_anchor = raw_uri.is_anchor();
let text = StrTendril::from(raw_uri.text.clone());
let attribute = raw_uri.attribute.clone();

// Truncate the source in case it gets too long
let input = input_content.input.clone();
let mut input = input_content.input.clone();
if let Input::String(src) = input {
input = Input::String(src.as_str().truncate_ellipse(MAX_SOURCE_LEN).to_string())
}

if let Ok(uri) = Uri::try_from(raw_uri) {
Ok(Some(Request::new(uri, input, attribute)))
Expand Down

0 comments on commit 8e2c676

Please sign in to comment.