✨ Option to use a proxy for outgoing upstream search engine requests

[m00nwtchr]

What does this PR do?

Adds a new config option to enable proxying of outgoing (to search engines) requests.

Why is this change important?

It allows for greater privacy of single-user/selfhosted instances by proxying requests via e.g. Tor.

How to test this PR locally?

1. Choose a proxy, e.g. tor
2. Set the proxy config value e.g.: proxy = "socks5h://172.0.0.1:9050"
3. Run websurfx

[github-actions]

Congrats on making your first Pull Request and thanks for taking the time to improve Websurfx! ❤️!
Say hello by joining the conversation in our Discord

[github-actions]

Stale pull request message

[neon-mmd]

Sorry for the late reply.

Thanks ❤️ for working on this issue. Good job!! Keep it up!! 👍 but we would like to suggest a few changes:

• Instead of using an empty string and stringified value for proxy option for config. We would suggest using conditionally compiled feature option of rust to allow the user to either enable/disable the option during the build stage as this will allow the user to for example not include the code for proxy in the final binary which can reduce the final binary size. Also, using this feature can increase the security of the server by disabling the proxy option and removing the code completely during the build stage. I hope you got our point 🙂

Sample Code for the above suggestion

Cargo.toml

[package]
name = "websurfx"
version = "1.17.20"
edition = "2021"
description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
repository = "https://github.com/neon-mmd/websurfx"
license = "AGPL-3.0"

[[bin]]
name = "websurfx"
test = true
bench = false
path = "src/bin/websurfx.rs"

[dependencies]
reqwest = { version = "0.12.5", default-features = false, features = [
    "rustls-tls",
    "brotli",
    "gzip",
    "http2"
] }
tokio = { version = "1.32.0", features = [
    "rt-multi-thread",
    "macros",
    "fs",
    "io-util",
], default-features = false }
serde = { version = "1.0.209", default-features = false, features = ["derive"] }
serde_json = { version = "1.0.122", default-features = false }
maud = { version = "0.26.0", default-features = false, features = [
    "actix-web",
] }
scraper = { version = "0.20.0", default-features = false }
actix-web = { version = "4.9.0", features = [
    "cookies",
    "macros",
    "compress-brotli",
], default-features = false }
actix-files = { version = "0.6.5", default-features = false }
actix-cors = { version = "0.7.0", default-features = false }
fake-useragent = { version = "0.1.3", default-features = false }
env_logger = { version = "0.11.1", default-features = false }
log = { version = "0.4.21", default-features = false }
mlua = { version = "0.9.9", features = [
    "luajit",
    "vendored",
], default-features = false }
redis = { version = "0.25.4", features = [
    "tokio-comp",
    "connection-manager",
], default-features = false, optional = true }
blake3 = { version = "1.5.4", default-features = false }
error-stack = { version = "0.4.0", default-features = false, features = [
    "std",
] }
async-trait = { version = "0.1.80", default-features = false }
regex = { version = "1.9.4", features = ["perf"], default-features = false }
smallvec = { version = "1.13.1", features = [
    "union",
    "serde",
], default-features = false }
futures = { version = "0.3.30", default-features = false, features = ["alloc"] }
dhat = { version = "0.3.2", optional = true, default-features = false }
mimalloc = { version = "0.1.43", default-features = false }
async-once-cell = { version = "0.5.3", default-features = false }
actix-governor = { version = "0.5.0", default-features = false }
mini-moka = { version = "0.10", optional = true, default-features = false, features = [
    "sync",
] }
async-compression = { version = "0.4.12", default-features = false, features = [
    "brotli",
    "tokio",
], optional = true }
chacha20poly1305 = { version = "0.10.1", default-features = false, features = [
    "alloc",
    "getrandom",
], optional = true }
chacha20 = { version = "0.9.1", default-features = false, optional = true }
base64 = { version = "0.21.5", default-features = false, features = [
    "std",
], optional = true }
cfg-if = { version = "1.0.0", default-features = false, optional = true }
keyword_extraction = { version = "1.4.3", default-features = false, features = [
    "tf_idf",
] }

stop-words = { version = "0.8.0", default-features = false, features = ["iso"] }
thesaurus = { version = "0.5.2", default-features = false, optional = true, features = [
    "moby",
]}
itertools = {version = "0.13.0", default-features = false}

[dev-dependencies]
rusty-hook = { version = "^0.11.2", default-features = false }
criterion = { version = "0.5.1", default-features = false }
tempfile = { version = "3.10.1", default-features = false }

[build-dependencies]
lightningcss = { version = "1.0.0-alpha.57", default-features = false, features = [
    "grid",
] }
# Disabled until bug fixing update
# minify-js = { version = "0.6.0", default-features = false }
# Temporary fork with fix
minify-js = { git = "https://github.com/RuairidhWilliamson/minify-js", branch = "master", version = "0.6.0", default-features = false} 



[profile.dev]
opt-level = 0
debug = true
split-debuginfo = 'unpacked'
debug-assertions = true
overflow-checks = true
lto = false
panic = 'unwind'
incremental = true
codegen-units = 256
rpath = false

[profile.release]
opt-level = 3
debug = false # This should only be commented when testing with dhat profiler
# debug = 1 # This should only be uncommented when testing with dhat profiler
split-debuginfo = '...'
debug-assertions = false
overflow-checks = false
lto = 'thin'
panic = 'abort'
incremental = false
codegen-units = 1
rpath = false
strip = "symbols"

[profile.bsr1]
inherits = "release"
opt-level = "s"

[profile.bsr2]
inherits = "bsr1"
opt-level = "z"

[profile.lpcb1]
inherits = "release"
codegen-units = 16

[profile.lpcb2]
inherits = "lpcb1"
lto = "off"

[profile.lpcb3]
inherits = "lpcb2"
opt-level = 2

[profile.bsr_and_lpcb1]
inherits = "lpcb1"
opt-level = "s"

[profile.bsr_and_lpcb2]
inherits = "lpcb2"
opt-level = "s"

[profile.bsr_and_lpcb3]
inherits = "lpcb3"
opt-level = "s"

[profile.bsr_and_lpcb4]
inherits = "lpcb1"
opt-level = "z"

[profile.bsr_and_lpcb5]
inherits = "lpcb1"
opt-level = "z"

[profile.bsr_and_lpcb6]
inherits = "lpcb1"
opt-level = "z"

[features]
use-synonyms-search = ["thesaurus/static"]
default = ["memory-cache"]
+torified-search-queries = [reqwest/proxy]
dhat-heap = ["dep:dhat"]
memory-cache = ["dep:mini-moka"]
redis-cache = ["dep:redis", "dep:base64"]
compress-cache-results = ["dep:async-compression", "dep:cfg-if"]
encrypt-cache-results = ["dep:chacha20poly1305", "dep:chacha20"]
cec-cache-results = ["compress-cache-results", "encrypt-cache-results"]
experimental-io-uring = ["actix-web/experimental-io-uring"]
use-non-static-synonyms-search = ["thesaurus"]

[neon-mmd]

Thanks ❤️ for working on the previous suggestions and making some really cool changes but we would like to make a few more suggestions before we merge this PR in. 🙂

[neon-mmd]

@m00nwtchr it has been nearly one week, Any updates on this PR so far? Would love to hear 🙂

[m00nwtchr]

Alright, I merged in the changes, should be good to go now.

[neon-mmd]

Thanks ❤️ again for working on such a cool PR. We really appreciate it. Good job!! Keep it up!! 👍. Since everything looks good. We will merge this PR as soon as possible 🙂 .