feat(sre): implement critical infrastructure hardening (V1, V2, V4, V5)

V1 - WAF/CAPTCHA Detection: Add detect_waf_challenge() with 19 WAF
signatures (Cloudflare, reCAPTCHA, hCaptcha, DataDome, PerimeterX,
Akamai) to detect false HTTP 200 responses. Retries with UA rotation
once before returning WafBlocked error.

V2 - Race Condition Fix: Add fs2 file locking to state_store.rs.
Exclusive lock on save(), shared lock on load() prevents data
corruption with parallel instances.

V4 - TUI Panic Hook: Each restoration step runs independently.
Partial failures no longer corrupt terminal state.

V5 - OOM Protection: Streaming size limits in sitemap_parser.rs.
HTTP response capped at 50MB, GZIP decompression at 100MB.

Tests: 267/267 passing (+15 new WAF tests), 0 clippy warnings.

Author: Developer

Cargo.lock

@@ -107,6 +107,9 @@ rand = "0.8"
 # Cache directory management (TASK-001: User-Agent lazy update)
 dirs = "5"
 
+# File locking for concurrent state access (SRE hotfix: Vector 2)
+fs2 = "0.4"
+
 # Walking directory for tests
 walkdir = "2"
 

Cargo.toml

@@ -4,6 +4,7 @@
 //! Follows security-no-unwrap-prod rule - no unwrap() in production code.
 
 use crossterm::{
+    cursor::Show,
     execute,
     terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen},
 };
@@ -43,18 +44,26 @@ pub fn setup_terminal() -> io::Result<Terminal<CrosstermBackend<io::Stdout>>> {
 pub fn restore_terminal() -> io::Result<()> {
     disable_raw_mode()?;
     execute!(io::stdout(), LeaveAlternateScreen)?;
+    execute!(io::stdout(), Show)?;
     Ok(())
 }
 
 /// Setup panic hook to restore terminal on panic
 ///
 /// This ensures the terminal is not left in a corrupted state
 /// if the application panics during TUI mode.
+///
+/// Each restoration step runs independently so that a partial
+/// failure (e.g., broken stdout) doesn't prevent other steps.
 fn setup_panic_hook() {
     let original_hook = std::panic::take_hook();
     std::panic::set_hook(Box::new(move |panic_info| {
-        // Restore terminal before panic to prevent corruption
-        let _ = restore_terminal();
+        // Each step runs independently — failure in one doesn't block others
+        // Following **err-result-over-panic**: ignore errors in cleanup path
+        let _ = disable_raw_mode();
+        let _ = execute!(io::stdout(), LeaveAlternateScreen);
+        let _ = execute!(io::stdout(), Show);
+        eprintln!("Application panicked. Terminal restored.");
         original_hook(panic_info);
     }));
 }

src/adapters/tui/terminal.rs

@@ -86,6 +86,8 @@ pub enum HttpError {
     Connection(String),
     /// Request building/error - contains error message
     Request(String),
+    /// WAF/CAPTCHA challenge detected in HTTP 200 (false positive)
+    WafChallenge(String),
 }
 
 impl std::fmt::Display for HttpError {
@@ -100,12 +102,77 @@ impl std::fmt::Display for HttpError {
             HttpError::Timeout => write!(f, "Request Timeout"),
             HttpError::Connection(msg) => write!(f, "Connection Error: {}", msg),
             HttpError::Request(msg) => write!(f, "Request Error: {}", msg),
+            HttpError::WafChallenge(provider) => {
+                write!(f, "WAF/CAPTCHA challenge detected ({})", provider)
+            }
         }
     }
 }
 
 impl std::error::Error for HttpError {}
 
+/// WAF/CAPTCHA challenge signature for detection in HTTP 200 responses
+///
+/// Each entry is (signature, provider_name). The detector scans the response body
+/// for these substrings — zero allocations, O(N*M) where M is the signature count.
+///
+/// Following **perf-iter-over-index**: iterator-based scanning, no regex needed.
+const WAF_SIGNATURES: &[(&str, &str)] = &[
+    // Cloudflare Turnstile / JS Challenge
+    ("cf-turnstile", "Cloudflare Turnstile"),
+    ("challenge-platform", "Cloudflare JS Challenge"),
+    ("Just a moment...", "Cloudflare"),
+    ("Checking your browser", "Cloudflare"),
+    ("__cf_chl_f_tk", "Cloudflare"),
+    // Google reCAPTCHA
+    ("g-recaptcha", "reCAPTCHA"),
+    ("recaptcha/api.js", "reCAPTCHA"),
+    ("grecaptcha.execute", "reCAPTCHA"),
+    // hCaptcha
+    ("hcaptcha.com", "hCaptcha"),
+    ("h-captcha", "hCaptcha"),
+    // DataDome
+    ("datadome", "DataDome"),
+    ("dd-captcha", "DataDome"),
+    // PerimeterX / HUMAN Security
+    ("perimeterx", "PerimeterX"),
+    ("_pxCaptcha", "PerimeterX"),
+    // Akamai Bot Manager
+    ("_abck", "Akamai Bot Manager"),
+    ("SensorData", "Akamai Bot Manager"),
+    // Generic challenge phrases
+    ("Please verify you are a human", "Generic Challenge"),
+    ("verify you are human", "Generic Challenge"),
+    ("bot detection", "Generic Detection"),
+];
+
+/// Detect WAF/CAPTCHA challenge pages disguised as HTTP 200
+///
+/// Scans the response body for known WAF signatures. Returns the provider name
+/// if a challenge is detected, or `None` if the content appears legitimate.
+///
+/// # Arguments
+///
+/// * `body` - The HTTP response body as a string
+///
+/// # Returns
+///
+/// * `Some(provider_name)` if a WAF challenge signature is found
+/// * `None` if no challenge detected
+///
+/// # Performance
+///
+/// O(N * M) where N = body length, M = signature count (currently 19).
+/// Zero allocations — uses `str::contains()` with static string slices.
+/// For M > 20, consider upgrading to `aho-corasick` for O(N) DFA matching.
+#[inline]
+pub fn detect_waf_challenge(body: &str) -> Option<&'static str> {
+    // Following **perf-iter-over-index**: iterator scan, no intermediate collections
+    WAF_SIGNATURES
+        .iter()
+        .find_map(|(sig, provider)| body.contains(sig).then_some(*provider))
+}
+
 /// HTTP client wrapper with configurable retry behavior
 ///
 /// Wraps `reqwest::Client` and adds:
@@ -204,10 +271,26 @@ impl HttpClient {
 
             match status.as_u16() {
                 200..=299 => {
-                    return response
+                    let body = response
                         .text()
                         .await
-                        .map_err(|e| HttpError::Request(e.to_string()));
+                        .map_err(|e| HttpError::Request(e.to_string()))?;
+
+                    // Detect WAF/CAPTCHA challenges disguised as HTTP 200
+                    if let Some(provider) = detect_waf_challenge(&body) {
+                        warn!(
+                            "WAF challenge detected from {} ({}), rotating UA",
+                            url, provider
+                        );
+                        // Same retry logic as 403: rotate UA once
+                        if ua_index == 0 {
+                            ua_index += 1;
+                            continue;
+                        }
+                        return Err(HttpError::WafChallenge(provider.to_string()));
+                    }
+
+                    return Ok(body);
                 }
                 403 => {
                     warn!("403 Forbidden from {}", url);
@@ -733,3 +816,171 @@ mod wiremock_tests {
         assert_eq!(result.unwrap(), expected_body);
     }
 }
+
+#[cfg(test)]
+mod waf_detection_tests {
+    use super::*;
+
+    // =========================================================================
+    // detect_waf_challenge unit tests
+    // =========================================================================
+
+    #[test]
+    fn test_detect_cloudflare_turnstile() {
+        let html = r#"<div id="cf-turnstile" data-sitekey="abc123"></div>"#;
+        assert_eq!(detect_waf_challenge(html), Some("Cloudflare Turnstile"));
+    }
+
+    #[test]
+    fn test_detect_cloudflare_just_a_moment() {
+        let html = "<html><body><h1>Just a moment...</h1></body></html>";
+        assert_eq!(detect_waf_challenge(html), Some("Cloudflare"));
+    }
+
+    #[test]
+    fn test_detect_cloudflare_checking_browser() {
+        let html = "<html><body>Checking your browser before accessing...</body></html>";
+        assert_eq!(detect_waf_challenge(html), Some("Cloudflare"));
+    }
+
+    #[test]
+    fn test_detect_recaptcha() {
+        let html = r#"<script src="https://www.google.com/recaptcha/api.js?render=abc"></script>"#;
+        assert_eq!(detect_waf_challenge(html), Some("reCAPTCHA"));
+    }
+
+    #[test]
+    fn test_detect_g_recaptcha() {
+        let html = r#"<div class="g-recaptcha" data-sitekey="abc"></div>"#;
+        assert_eq!(detect_waf_challenge(html), Some("reCAPTCHA"));
+    }
+
+    #[test]
+    fn test_detect_hcaptcha() {
+        let html = r#"<div class="h-captcha" data-sitekey="abc"></div>"#;
+        assert_eq!(detect_waf_challenge(html), Some("hCaptcha"));
+    }
+
+    #[test]
+    fn test_detect_datadome() {
+        let html = r#"<script src="https://js.datadome.co/captcha.js"></script>"#;
+        assert_eq!(detect_waf_challenge(html), Some("DataDome"));
+    }
+
+    #[test]
+    fn test_detect_perimeterx() {
+        let html = r#"<script>var _pxCaptcha = {};</script>"#;
+        assert_eq!(detect_waf_challenge(html), Some("PerimeterX"));
+    }
+
+    #[test]
+    fn test_detect_akamai() {
+        let html = r#"<input type="hidden" name="_abck" value="xxx">"#;
+        assert_eq!(detect_waf_challenge(html), Some("Akamai Bot Manager"));
+    }
+
+    #[test]
+    fn test_detect_generic_challenge() {
+        let html = "<p>Please verify you are a human to continue.</p>";
+        assert_eq!(detect_waf_challenge(html), Some("Generic Challenge"));
+    }
+
+    #[test]
+    fn test_clean_html_no_detection() {
+        let html = r#"
+            <html>
+                <head><title>Normal Page</title></head>
+                <body>
+                    <article>
+                        <h1>Welcome</h1>
+                        <p>This is a normal page with real content.</p>
+                    </article>
+                </body>
+            </html>
+        "#;
+        assert_eq!(detect_waf_challenge(html), None);
+    }
+
+    #[test]
+    fn test_empty_body_no_detection() {
+        assert_eq!(detect_waf_challenge(""), None);
+    }
+
+    // =========================================================================
+    // Wiremock integration: HTTP 200 with WAF body
+    // =========================================================================
+
+    use wiremock::matchers::{method, path};
+    use wiremock::{Mock, MockServer, ResponseTemplate};
+
+    #[tokio::test]
+    async fn test_200_cloudflare_challenge_returns_waf_error() {
+        let mock_server = MockServer::start().await;
+
+        // Cloudflare challenge page returns 200
+        let challenge_body = r#"<html><head><title>Just a moment...</title></head>
+        <body><div id="challenge-running">Checking your browser...</div></body></html>"#;
+
+        Mock::given(method("GET"))
+            .and(path("/"))
+            .respond_with(ResponseTemplate::new(200).set_body_string(challenge_body))
+            .mount(&mock_server)
+            .await;
+
+        let config = HttpClientConfig {
+            max_retries: 1, // Only 1 retry
+            ..Default::default()
+        };
+        let client = HttpClient::new(config).unwrap();
+
+        let result = client.get(&mock_server.uri()).await;
+
+        assert!(result.is_err());
+        assert!(matches!(result.unwrap_err(), HttpError::WafChallenge(_)));
+    }
+
+    #[tokio::test]
+    async fn test_200_recaptcha_challenge_returns_waf_error() {
+        let mock_server = MockServer::start().await;
+
+        let challenge_body = r#"<html><body><div class="g-recaptcha" data-sitekey="abc"></div></body></html>"#;
+
+        Mock::given(method("GET"))
+            .and(path("/"))
+            .respond_with(ResponseTemplate::new(200).set_body_string(challenge_body))
+            .mount(&mock_server)
+            .await;
+
+        let config = HttpClientConfig {
+            max_retries: 1,
+            ..Default::default()
+        };
+        let client = HttpClient::new(config).unwrap();
+
+        let result = client.get(&mock_server.uri()).await;
+
+        assert!(result.is_err());
+        assert!(matches!(result.unwrap_err(), HttpError::WafChallenge(_)));
+    }
+
+    #[tokio::test]
+    async fn test_200_normal_page_returns_body() {
+        let mock_server = MockServer::start().await;
+
+        let normal_body = "<html><body><article><h1>Real Content</h1><p>Normal page.</p></article></body></html>";
+
+        Mock::given(method("GET"))
+            .and(path("/"))
+            .respond_with(ResponseTemplate::new(200).set_body_string(normal_body))
+            .mount(&mock_server)
+            .await;
+
+        let config = HttpClientConfig::default();
+        let client = HttpClient::new(config).unwrap();
+
+        let result = client.get(&mock_server.uri()).await;
+
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap(), normal_body);
+    }
+}

src/application/http_client.rs

@@ -14,6 +14,7 @@
 use crate::domain::{DownloadedAsset, ScrapedContent, ValidUrl};
 use crate::error::{Result, ScraperError};
 use crate::ScraperConfig;
+use super::http_client::detect_waf_challenge;
 use futures::stream::{self, StreamExt};
 use reqwest_middleware::ClientWithMiddleware;
 use tracing::{debug, info, warn};
@@ -78,6 +79,15 @@ pub async fn scrape_with_config(
     let html = response.text().await.map_err(ScraperError::Network)?;
     debug!("📄 Downloaded {} bytes from {}", html.len(), url);
 
+    // Detect WAF/CAPTCHA challenges disguised as HTTP 200
+    if let Some(provider) = detect_waf_challenge(&html) {
+        warn!("WAF challenge detected from {}: {}", url, provider);
+        return Err(ScraperError::WafBlocked {
+            url: url.to_string(),
+            provider: provider.to_string(),
+        });
+    }
+
     // Try Readability first, fallback to plain text extraction
     match crate::infrastructure::scraper::readability::parse(&html, Some(url.as_str())) {
         Ok(article) => {