From c1dfb047fb93628213d3a8b5de8ee904a33c0308 Mon Sep 17 00:00:00 2001 From: Tomio Date: Sat, 18 Jun 2022 05:22:10 +0000 Subject: [PATCH] feat: add ability to add additional chrome flags (closes #36) --- .devcontainer/devcontainer.json | 78 ++++++++++++++++----------------- README.md | 1 + src/cdp.rs | 8 ++-- src/error.rs | 4 +- src/main.rs | 64 ++++++++++++++++++++------- src/middlewares/auth.rs | 12 ++--- src/routes/get.rs | 2 +- src/routes/screenshot.rs | 75 ++++++++++++++++--------------- src/util.rs | 19 ++++---- 9 files changed, 145 insertions(+), 118 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f93d7f9..db676c6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,46 +1,46 @@ { - "name": "Rust", - "build": { - "dockerfile": "Dockerfile", - "args": { - // Use the VARIANT arg to pick a Debian OS version: buster, bullseye - // Use bullseye when on local on arm64/Apple Silicon. - "VARIANT": "buster" - } - }, - "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + "name": "Rust", + "build": { + "dockerfile": "Dockerfile", + "args": { + // Use the VARIANT arg to pick a Debian OS version: buster, bullseye + // Use bullseye when on local on arm64/Apple Silicon. + "VARIANT": "buster" + } + }, + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], - // Configure tool-specific properties. - "customizations": { - // Configure properties specific to VS Code. - "vscode": { - // Set *default* container specific settings.json values on container create. - "settings": { - "lldb.executable": "/usr/bin/lldb", - // VS Code don't watch files under ./target - "files.watcherExclude": { - "**/target/**": true - } - }, + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Set *default* container specific settings.json values on container create. + "settings": { + "lldb.executable": "/usr/bin/lldb", + // VS Code don't watch files under ./target + "files.watcherExclude": { + "**/target/**": true + } + }, - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "vadimcn.vscode-lldb", - "mutantdino.resourcemonitor", - "rust-lang.rust-analyzer", - "tamasfe.even-better-toml", - "serayuzgur.crates", - "usernamehw.errorlens" - ] - } - }, + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "vadimcn.vscode-lldb", + "mutantdino.resourcemonitor", + "rust-lang.rust-analyzer", + "tamasfe.even-better-toml", + "serayuzgur.crates", + "usernamehw.errorlens" + ] + } + }, - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], - // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "rustc --version", + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "rustc --version", - // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode" + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" } diff --git a/README.md b/README.md index 84a63ee..e1bbb29 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ - `DARK_MODE` - if set, it will take screenshots in dark mode, if the website supports it (optional) - `FORCE_NSFW_CHECK` - if set, force NSFW check (optional) - `FORCE_DARK_MODE` - if set, force dark mode (optional) +- `CHROME_FLAGS` - additional flags to provide to chrome (optional, example: `--a,--b,-c`) ### Railway diff --git a/src/cdp.rs b/src/cdp.rs index 2a34841..298979e 100644 --- a/src/cdp.rs +++ b/src/cdp.rs @@ -87,21 +87,21 @@ impl WebDriverCompatibleCommand for ChromeCommand { match &self { ChromeCommand::LaunchApp(app_id) => { method = Method::POST; - body = Some(json!({ "id": app_id }).to_string()) + body = Some(json!({ "id": app_id }).to_string()); }, ChromeCommand::SetNetworkConditions(conditions) => { method = Method::POST; - body = Some(json!({ "network_conditions": conditions }).to_string()) + body = Some(json!({ "network_conditions": conditions }).to_string()); }, ChromeCommand::ExecuteCdpCommand(command, params) => { method = Method::POST; - body = Some(json!({"cmd": command, "params": params }).to_string()) + body = Some(json!({"cmd": command, "params": params }).to_string()); }, ChromeCommand::SetSinkToUse(sink_name) | ChromeCommand::StartTabMirroring(sink_name) | ChromeCommand::StopCasting(sink_name) => { method = Method::POST; - body = Some(json!({ "sinkName": sink_name }).to_string()) + body = Some(json!({ "sinkName": sink_name }).to_string()); }, _ => {}, } diff --git a/src/error.rs b/src/error.rs index e4172ee..ef0963a 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,5 +1,3 @@ -use std::ops::Deref; - use actix_web::error::ResponseError; use actix_web::http::header::ContentType; use actix_web::http::StatusCode; @@ -38,7 +36,7 @@ impl ResponseError for Error { } fn status_code(&self) -> StatusCode { - match self.deref() { + match self { Error::InvalidUrl | Error::MissingAuthToken | Error::FailedToConnect diff --git a/src/main.rs b/src/main.rs index fae98a6..fe43b84 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,8 +1,20 @@ #![feature(let_chains, pattern)] +#![warn(clippy::pedantic)] +#![allow( + clippy::unreadable_literal, + clippy::module_name_repetitions, + clippy::unused_async, + clippy::too_many_lines, + clippy::cast_possible_truncation, + clippy::missing_errors_doc, + clippy::must_use_candidate, + clippy::wildcard_imports +)] #[macro_use] extern crate tracing; +use std::borrow::ToOwned; use std::env; use std::sync::Arc; @@ -10,7 +22,24 @@ use actix_web::middleware::Compress; use actix_web::{web, App, Error, HttpServer}; use actix_web_static_files::ResourceFiles; use cdp::ChromeCommand; -use evasions::*; +use evasions::{ + evaluate_on_new_document, + CHROME_APP, + CHROME_CSI, + CHROME_LOADTIMES, + CHROME_RUNTIME, + IFRAME_CONTENTWINDOW, + MEDIA_CODECS, + NAVIGATOR_HARDWARECONCURRENCY, + NAVIGATOR_LANGUAGES, + NAVIGATOR_PERMISSIONS, + NAVIGATOR_PLUGINS, + NAVIGATOR_VENDOR, + NAVIGATOR_WEBDRIVER, + UTILS, + WEBGL_VENDOR, + WINDOW_OUTERDIMENSIONS, +}; use fantoccini::{Client, ClientBuilder}; use providers::{Provider, Storage}; use reqwest::Client as ReqwestClient; @@ -44,26 +73,27 @@ async fn main() -> anyhow::Result<()> { initialize_tracing(); let mut capabilities = Map::new(); + let mut args = vec![ + "--disable-gpu".to_owned(), + "--no-sandbox".to_owned(), + "--disable-dev-shm-usage".to_owned(), + "--headless".to_owned(), + "--hide-scrollbars".to_owned(), + "--whitelisted-ips=".to_owned(), + ]; + + if let Ok(flags) = env::var("CHROME_FLAGS") { + let flags = flags.split(',').map(ToOwned::to_owned).collect::>(); + + args.extend_from_slice(&flags); + }; + let chrome_opts = match env::var("GOOGLE_CHROME_PATH") { Ok(path) => serde_json::json!({ "binary": path, - "args": [ - "--disable-gpu", - "--no-sandbox", - "--disable-dev-shm-usage", - "--headless", - "--whitelisted-ips=" - ] - }), - Err(_) => serde_json::json!({ - "args": [ - "--disable-gpu", - "--no-sandbox", - "--disable-dev-shm-usage", - "--headless", - "--whitelisted-ips=" - ] + "args": args }), + Err(_) => serde_json::json!({ "args": args }), }; capabilities.insert("goog:chromeOptions".to_owned(), chrome_opts); diff --git a/src/middlewares/auth.rs b/src/middlewares/auth.rs index e25da7e..a1edbc1 100644 --- a/src/middlewares/auth.rs +++ b/src/middlewares/auth.rs @@ -60,23 +60,23 @@ where let auth = auth.to_str().expect("Failed converting to str").to_owned(); if auth_token != auth { - let res = HttpResponse::from_error(Errors::Unauthorized) + let response = HttpResponse::from_error(Errors::Unauthorized) .map_into_right_body::(); - return Box::pin(async { Ok(ServiceResponse::new(req, res)) }); + return Box::pin(async { Ok(ServiceResponse::new(req, response)) }); } }, None => { - let res = HttpResponse::from_error(Errors::MissingAuthToken) + let response = HttpResponse::from_error(Errors::MissingAuthToken) .map_into_right_body::(); - return Box::pin(async { Ok(ServiceResponse::new(req, res)) }); + return Box::pin(async { Ok(ServiceResponse::new(req, response)) }); }, }; } - let res = self.service.call(ServiceRequest::from_parts(req, pl)); + let response = self.service.call(ServiceRequest::from_parts(req, pl)); - Box::pin(async move { res.await.map(ServiceResponse::map_into_left_body) }) + Box::pin(async move { response.await.map(ServiceResponse::map_into_left_body) }) } } diff --git a/src/routes/get.rs b/src/routes/get.rs index 73c3f3b..09ad60a 100644 --- a/src/routes/get.rs +++ b/src/routes/get.rs @@ -21,6 +21,6 @@ pub async fn get_screenshot( Ok(HttpResponse::Ok() .content_type("image/png") - .append_header(header::CacheControl(vec![header::CacheDirective::MaxAge(31536000)])) + .append_header(header::CacheControl(vec![header::CacheDirective::MaxAge(31_536_000)])) .body(screenshot)) } diff --git a/src/routes/screenshot.rs b/src/routes/screenshot.rs index cde3bc5..df26634 100644 --- a/src/routes/screenshot.rs +++ b/src/routes/screenshot.rs @@ -112,44 +112,43 @@ pub async fn screenshot( client.refresh().await.expect("Failed to refresh"); - let screenshot = match payload.fullscreen { - true => { - let original_size = client.get_window_size().await.expect("Failed to get window size"); - let width = client - .execute("return document.body.parentNode.scrollWidth", vec![]) - .await - .expect("Failed getting scroll width") - .as_u64() - .expect("Failed to convert to u64"); - - let height = client - .execute("return document.body.parentNode.scrollHeight", vec![]) - .await - .expect("Failed getting scroll height") - .as_u64() - .expect("Failed to convert to u64"); - - client - .set_window_size(width as u32, height as u32) - .await - .expect("Failed setting window size"); - - let ss = client - .find(Locator::Css("body")) - .await - .expect("Failed finding body element") - .screenshot() - .await - .expect("Failed screenshoting page"); - - client - .set_window_size(original_size.0 as u32, original_size.1 as u32) - .await - .expect("Failed setting window size"); - - ss - }, - false => client.screenshot().await.expect("Failed screenshoting page"), + let screenshot = if payload.fullscreen { + let original_size = client.get_window_size().await.expect("Failed to get window size"); + let width = client + .execute("return document.body.parentNode.scrollWidth", vec![]) + .await + .expect("Failed getting scroll width") + .as_u64() + .expect("Failed to convert to u64"); + + let height = client + .execute("return document.body.parentNode.scrollHeight", vec![]) + .await + .expect("Failed getting scroll height") + .as_u64() + .expect("Failed to convert to u64"); + + client + .set_window_size(width as u32, height as u32) + .await + .expect("Failed setting window size"); + + let ss = client + .find(Locator::Css("body")) + .await + .expect("Failed finding body element") + .screenshot() + .await + .expect("Failed screenshoting page"); + + client + .set_window_size(original_size.0 as u32, original_size.1 as u32) + .await + .expect("Failed setting window size"); + + ss + } else { + client.screenshot().await.expect("Failed screenshoting page") }; let slug = slug().expect("Failed generating slug"); diff --git a/src/util.rs b/src/util.rs index 7a52d9b..96a925b 100644 --- a/src/util.rs +++ b/src/util.rs @@ -15,7 +15,7 @@ use tracing_subscriber::FmtSubscriber; static URL_REGEX: OnceCell = OnceCell::new(); static NSFW_SITE_LIST: sync::OnceCell> = sync::OnceCell::const_new(); -async fn get_nsfw_list() -> Result> { +async fn get_nsfw_list<'a>() -> Result<&'a Vec> { let list = NSFW_SITE_LIST .get_or_init(|| async { let text = reqwest::get("https://blocklistproject.github.io/Lists/porn.txt") @@ -36,8 +36,7 @@ async fn get_nsfw_list() -> Result> { }) .collect() }) - .await - .to_owned(); + .await; Ok(list) } @@ -48,14 +47,13 @@ pub async fn check_if_nsfw(host: &str) -> Result { Ok(list.par_iter().any(|s| s == host)) } -fn get_url_regex() -> Result { +fn get_url_regex<'a>() -> Result<&'a Regex> { let re = URL_REGEX .get_or_try_init(|| { let re = r"(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})"; Regex::with_flags(re, "i") - })? - .to_owned(); + })?; Ok(re) } @@ -63,9 +61,10 @@ fn get_url_regex() -> Result { pub fn check_if_url(url: &str) -> Result { let re = get_url_regex()?; - match re.find(url).is_some() { - true => Ok(true), - false => Err(anyhow::anyhow!("url not valid")), + if re.find(url).is_some() { + Ok(true) + } else { + Err(anyhow::anyhow!("url not valid")) } } @@ -126,7 +125,7 @@ pub mod test { pub async fn setup_with_state() -> Result<(Client, Data)> { let client = ClientBuilder::rustls() - .capabilities(Lazy::force(&CAPABILITIES).to_owned()) + .capabilities(Lazy::force(&CAPABILITIES).clone()) .connect("http://localhost:9515") .await?;