Skip to content

Commit

Permalink
Remove rayon and stdout lock
Browse files Browse the repository at this point in the history
  • Loading branch information
mre committed Dec 9, 2021
1 parent 9c1e48a commit 9e2a500
Show file tree
Hide file tree
Showing 4 changed files with 6 additions and 30 deletions.
11 changes: 0 additions & 11 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 4 additions & 8 deletions lychee-bin/src/commands/dump.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::io::{self, StdoutLock, Write};
use std::io::{self, Write};

use lychee_lib::Result;
use lychee_lib::{Client, Request};
Expand All @@ -11,10 +11,6 @@ pub(crate) async fn dump<'a, S>(client: Client, requests: S, verbose: bool) -> R
where
S: futures::Stream<Item = Result<Request>>,
{
// Lock stdout for better performance
let stdout = io::stdout();
let mut handle = stdout.lock();

tokio::pin!(requests);

while let Some(request) = requests.next().await {
Expand All @@ -28,7 +24,7 @@ where
// See https://github.com/rust-lang/rust/issues/46016
// This can occur when piping the output of lychee
// to another program like `grep`.
if let Err(e) = write(&mut handle, &request, verbose) {
if let Err(e) = write(&request, verbose) {
if e.kind() != io::ErrorKind::BrokenPipe {
eprintln!("{}", e);
return Ok(ExitCode::UnexpectedFailure);
Expand All @@ -42,11 +38,11 @@ where
/// Dump request to stdout
/// Only print source in verbose mode. This way the normal link output
/// can be fed into another tool without data mangling.
fn write(handle: &mut StdoutLock<'_>, request: &Request, verbose: bool) -> io::Result<()> {
fn write(request: &Request, verbose: bool) -> io::Result<()> {
let output = if verbose {
request.to_string()
} else {
request.uri.to_string()
};
writeln!(*handle, "{}", output)
writeln!(io::stdout(), "{}", output)
}
2 changes: 0 additions & 2 deletions lychee-lib/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ cached = "0.26.2"
once_cell = "1.8.0"
thiserror = "1.0"
futures = "0.3.18"
rayon = "1.5.1"
ellipse = "0.2.0"

[dependencies.par-stream]
version = "0.7.0"
Expand Down
11 changes: 2 additions & 9 deletions lychee-lib/src/helpers/request.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
use ellipse::Ellipse;
use html5ever::tendril::StrTendril;
use log::info;
use percent_encoding::percent_decode_str;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use reqwest::Url;
use std::{
collections::HashSet,
Expand All @@ -17,8 +15,6 @@ use crate::{
Base, ErrorKind, Input, Request, Result, Uri,
};

const MAX_SOURCE_LEN: usize = 100;

/// Create requests out of the collected URLs.
/// Only keeps "valid" URLs. This filters out anchors for example.
pub(crate) fn create(
Expand All @@ -37,17 +33,14 @@ pub(crate) fn create(
};

let requests: Result<Vec<Option<Request>>> = uris
.into_par_iter()
.into_iter()
.map(|raw_uri| {
let is_anchor = raw_uri.is_anchor();
let text = StrTendril::from(raw_uri.text.clone());
let attribute = raw_uri.attribute.clone();

// Truncate the source in case it gets too long
let mut input = input_content.input.clone();
if let Input::String(src) = input {
input = Input::String(src.as_str().truncate_ellipse(MAX_SOURCE_LEN).to_string())
}
let input = input_content.input.clone();

if let Ok(uri) = Uri::try_from(raw_uri) {
Ok(Some(Request::new(uri, input, attribute)))
Expand Down

0 comments on commit 9e2a500

Please sign in to comment.