From 16aab7d7b3cc23d2ae0fdef33b0925eb013a7eae Mon Sep 17 00:00:00 2001 From: Captain Mirage <87281406+CaptainMirage@users.noreply.github.com> Date: Fri, 22 May 2026 23:36:21 +0330 Subject: [PATCH 1/8] feat(quota_tracker): add per-account quota tracking with config fields --- src/bin/ui.rs | 11 ++ src/config.rs | 19 ++ src/lib.rs | 3 + src/quota_tracker.rs | 433 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 466 insertions(+) create mode 100644 src/quota_tracker.rs diff --git a/src/bin/ui.rs b/src/bin/ui.rs index c5f9ed63..44587964 100644 --- a/src/bin/ui.rs +++ b/src/bin/ui.rs @@ -298,6 +298,9 @@ struct FormState { /// claude.ai / grok.com / x.com). Config-only — no UI editor yet. /// See `assets/exit_node/` for the generic exit-node handler. exit_node: mhrv_rs::config::ExitNodeConfig, + /// Quota config — config-only round-trip fields. No UI editor yet. + quota_daily_limit: u64, + quota_safety_buffer: u64, } #[derive(Clone, Debug)] @@ -392,6 +395,8 @@ fn load_form() -> (FormState, Option) { auto_blacklist_cooldown_secs: c.auto_blacklist_cooldown_secs, request_timeout_secs: c.request_timeout_secs, exit_node: c.exit_node.clone(), + quota_daily_limit: c.quota_daily_limit, + quota_safety_buffer: c.quota_safety_buffer, } } else { FormState { @@ -434,6 +439,8 @@ fn load_form() -> (FormState, Option) { auto_blacklist_cooldown_secs: 120, request_timeout_secs: 30, exit_node: mhrv_rs::config::ExitNodeConfig::default(), + quota_daily_limit: 20_000, + quota_safety_buffer: 500, } }; (form, load_err) @@ -622,6 +629,10 @@ impl FormState { // / grok.com / x.com). Round-trip through FormState — config-only // editing for now, UI editor planned for v1.9.x desktop UI batch. exit_node: self.exit_node.clone(), + // Quota limits: config-only for now, round-tripped through FormState + // so Save doesn't drop hand-edited values. + quota_daily_limit: self.quota_daily_limit, + quota_safety_buffer: self.quota_safety_buffer, }) } } diff --git a/src/config.rs b/src/config.rs index cd63b8b8..b858b715 100644 --- a/src/config.rs +++ b/src/config.rs @@ -398,6 +398,21 @@ pub struct Config { /// Setup walkthrough at `assets/exit_node/README.md`. Default off. #[serde(default)] pub exit_node: ExitNodeConfig, + + /// Daily request quota per account bucket. Each configured script_id is + /// treated as one separate account. Default 20_000 matches the free-tier + /// Apps Script UrlFetchApp limit. Set to 100_000 for Workspace accounts. + #[serde(default = "default_quota_daily_limit")] + pub quota_daily_limit: u64, + + /// Per-account safety buffer. An account is considered effectively + /// exhausted when its remaining requests for the current 24-hour window + /// drop below this value. The reserve intentionally keeps calls away from + /// Google's hard quota edge to avoid triggering anti-abuse heuristics. + /// Aggregate hard-stop reserve = account_count × quota_safety_buffer. + /// Default 500. + #[serde(default = "default_quota_safety_buffer")] + pub quota_safety_buffer: u64, } /// Configuration for the optional second-hop exit node. @@ -526,6 +541,8 @@ fn default_block_doh() -> bool { true } fn default_auto_blacklist_strikes() -> u32 { 3 } fn default_auto_blacklist_window_secs() -> u64 { 30 } fn default_auto_blacklist_cooldown_secs() -> u64 { 120 } +fn default_quota_daily_limit() -> u64 { 20_000 } +fn default_quota_safety_buffer() -> u64 { 500 } /// Default for `request_timeout_secs`: 30s, matching the historical /// hard-coded `BATCH_TIMEOUT` and Apps Script's typical response cliff. @@ -920,6 +937,8 @@ impl From for Config { auto_blacklist_cooldown_secs: t.relay.auto_blacklist_cooldown_secs, request_timeout_secs: t.relay.request_timeout_secs, exit_node: t.exit_node, + quota_daily_limit: default_quota_daily_limit(), + quota_safety_buffer: default_quota_safety_buffer(), } } } diff --git a/src/lib.rs b/src/lib.rs index 6b53a32b..0d9edb8a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,7 @@ pub mod domain_fronter; pub mod lan_utils; pub mod mitm; pub mod proxy_server; +pub mod quota_tracker; pub mod rlimit; pub mod tunnel_client; pub mod scan_ips; @@ -15,5 +16,7 @@ pub mod scan_sni; pub mod test_cmd; pub mod update_check; +pub use quota_tracker::QuotaSummary; + #[cfg(target_os = "android")] pub mod android_jni; diff --git a/src/quota_tracker.rs b/src/quota_tracker.rs new file mode 100644 index 00000000..95a3fddd --- /dev/null +++ b/src/quota_tracker.rs @@ -0,0 +1,433 @@ +// Quota tracking for Apps Script mode. +// +// Model assumption: each script_id in the configured list represents one +// separate Google account. Apps Script's UrlFetchApp quota is per-user/account, +// not per-script-deployment. This tool treats each configured deployment as a +// distinct account bucket so quotas are tracked independently. The structure is +// flexible enough for future refinement if a single account has multiple +// deployments. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; + +use portable_atomic::AtomicU64; +use portable_atomic::Ordering; +use serde::{Deserialize, Serialize}; + +use crate::data_dir; + +// ── Persisted per-account state ────────────────────────────────────────────── + +/// State for one Apps Script account bucket (one configured script_id). +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct AccountBucket { + /// Masked script id for log output (first 4 + "..." + last 4 chars). + pub masked_id: String, + /// Requests used in the current 24-hour window. + pub requests_used: u64, + /// Failed requests in the current 24-hour window. + pub failed_requests: u64, + /// Bytes of JSON payload uploaded to Apps Script (all attempts). + pub bytes_up: u64, + /// Bytes of response body received from Apps Script (successful only). + pub bytes_down: u64, + /// Total bytes transferred (bytes_up + bytes_down). + pub bytes_total: u64, + /// Unix timestamp of the last recorded request for this account. + pub last_request_at: Option, + /// Unix timestamp when this bucket's 24-hour window resets. + /// Set to first_request_time + 86400 on first use in each window. + /// Follows Apps Script's actual quota model: resets 24 h after first + /// request, not at a fixed midnight boundary. + pub next_reset_at: Option, + /// Whether this account has been flagged as quota-exhausted. + pub exhausted: bool, + /// Whether routing to this account is hard-stopped (no more dispatches). + /// Once set, stays set across restarts until manually cleared in the JSON + /// file or until the rolling window resets on the next recorded request. + pub hard_stopped: bool, + /// Human-readable reason this account was exhausted/stopped. + pub exhaustion_reason: Option, + /// Count of responses with quota-like error messages from this account. + /// Separate from failed_requests so callers can distinguish quota signals + /// from generic upstream failures. + pub quota_error_count: u64, +} + +// ── In-memory aggregate summary (not persisted) ────────────────────────────── + +/// Aggregate quota view across all account buckets. +/// Cheap to clone — used to pass a snapshot to the UI and terminal logs. +#[derive(Debug, Clone, Default)] +pub struct QuotaSummary { + /// Number of tracked account buckets (= number of configured script_ids). + pub account_count: usize, + /// Total daily capacity across all accounts (account_count × daily_limit). + pub daily_capacity_total: u64, + /// Total requests used across all active windows. + pub requests_used_total: u64, + /// Total requests remaining before the aggregate safety reserve is hit. + pub requests_remaining_total: u64, + /// Total failed requests across all buckets. + pub failed_requests_total: u64, + /// Total bytes uploaded across all buckets. + pub bytes_up_total: u64, + /// Total bytes downloaded across all buckets. + pub bytes_down_total: u64, + /// Total bytes transferred (up + down) across all buckets. + pub bytes_total: u64, + /// Number of accounts currently marked exhausted. + pub exhausted_count: usize, + /// Number of accounts currently hard-stopped. + pub hard_stopped_count: usize, + /// Whether a global hard stop is active (all buckets exhausted, or + /// aggregate remaining quota has crossed the collective safety threshold + /// with confirmed quota error signals). + pub global_hard_stop: bool, + /// Unix timestamp of the soonest window reset across all non-exhausted buckets. + pub next_reset_at: Option, +} + +// ── Disk state wrapper ──────────────────────────────────────────────────────── + +#[derive(Serialize, Deserialize, Default)] +struct QuotaState { + buckets: HashMap, +} + +// ── Tracker ─────────────────────────────────────────────────────────────────── + +pub struct QuotaTracker { + state: Mutex, + /// Ordered list of script_ids from config (determines account_count). + script_ids: Vec, + /// Daily request limit per account. Default 20_000 (free tier). + /// Set 100_000 for Google Workspace accounts. + daily_limit: u64, + /// Per-account safety buffer. An account is considered effectively done + /// when its remaining requests drop below this value. This reserve keeps + /// calls away from Google's hard quota edge, staying on the safer side + /// of anti-abuse heuristics and ToS gray areas. + /// Aggregate hard-stop reserve = account_count × safety_buffer. + safety_buffer: u64, + /// Incremented on every mutation; used to decide when to auto-flush. + dirty_count: AtomicU64, + state_path: PathBuf, +} + +fn quota_state_path() -> PathBuf { + data_dir::data_dir().join("quota_state.json") +} + +fn mask_id(id: &str) -> String { + let n = id.chars().count(); + if n <= 8 { + return "***".into(); + } + let head: String = id.chars().take(4).collect(); + let tail: String = id.chars().skip(n - 4).collect(); + format!("{}...{}", head, tail) +} + +fn now_unix() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +impl QuotaTracker { + /// Load persisted quota state from disk, or create a fresh tracker if + /// the file is absent or unreadable. Validates loaded buckets against + /// the current script_id list and inserts missing entries. + pub fn load(script_ids: &[String], daily_limit: u64, safety_buffer: u64) -> Self { + let state_path = quota_state_path(); + let mut qs = std::fs::read_to_string(&state_path) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .unwrap_or_default(); + + // Ensure every configured script_id has a bucket entry. + for sid in script_ids { + qs.buckets.entry(sid.clone()).or_insert_with(|| AccountBucket { + masked_id: mask_id(sid), + ..Default::default() + }); + } + + Self { + state: Mutex::new(qs), + script_ids: script_ids.to_vec(), + daily_limit, + safety_buffer, + dirty_count: AtomicU64::new(0), + state_path, + } + } + + // ── Recording ──────────────────────────────────────────────────────────── + + /// Record one Apps Script fetch attempt for `script_id`. Called once per + /// `do_relay_once_with` call, including retries — every call here maps + /// to one real `UrlFetchApp.fetch()` on Google's side. + /// + /// Rolls the 24-hour window forward if needed (rolling from first request, + /// not from midnight — matches Apps Script's actual quota reset cadence). + pub fn record_attempt(&self, script_id: &str, bytes_up: u64) { + let now = now_unix(); + let mut st = self.state.lock().unwrap(); + let bucket = st.buckets.entry(script_id.to_string()).or_insert_with(|| { + AccountBucket { + masked_id: mask_id(script_id), + ..Default::default() + } + }); + + // Roll the window if the 24-hour period has elapsed. + if let Some(reset_at) = bucket.next_reset_at { + if now >= reset_at { + bucket.requests_used = 0; + bucket.failed_requests = 0; + bucket.bytes_up = 0; + bucket.bytes_down = 0; + bucket.bytes_total = 0; + bucket.next_reset_at = Some(now + 86_400); + // Clear exhaustion flags on window reset so the account gets a + // fresh chance in the new quota period. + bucket.exhausted = false; + bucket.hard_stopped = false; + bucket.exhaustion_reason = None; + bucket.quota_error_count = 0; + } + } else { + // First request for this account — open the rolling window. + bucket.next_reset_at = Some(now + 86_400); + } + + bucket.requests_used += 1; + bucket.bytes_up += bytes_up; + bucket.bytes_total += bytes_up; + bucket.last_request_at = Some(now); + + // Check if the safety buffer threshold is crossed (soft exhaustion). + // This is separate from mark_exhausted which is called on quota error + // signals; this handles the case where the counter itself reaches the + // limit without a quota error message being returned yet. + let remaining = self.daily_limit.saturating_sub(bucket.requests_used); + if !bucket.hard_stopped && remaining < self.safety_buffer { + bucket.exhausted = true; + bucket.hard_stopped = true; + bucket.exhaustion_reason = Some(format!( + "safety buffer crossed: {}/{} requests used (limit {}, buffer {})", + bucket.requests_used, self.daily_limit, + self.daily_limit, self.safety_buffer, + )); + tracing::warn!( + "[quota] account {} safety buffer reached ({} remaining < {}): marking hard-stopped", + bucket.masked_id, remaining, self.safety_buffer, + ); + } + + drop(st); + let n = self.dirty_count.fetch_add(1, Ordering::Relaxed); + if n % 50 == 0 { + self.save(); + } + } + + /// Record that a relay attempt succeeded and the response body was `bytes_down` bytes. + pub fn record_success(&self, script_id: &str, bytes_down: u64) { + let mut st = self.state.lock().unwrap(); + if let Some(bucket) = st.buckets.get_mut(script_id) { + bucket.bytes_down += bytes_down; + bucket.bytes_total += bytes_down; + } + drop(st); + self.dirty_count.fetch_add(1, Ordering::Relaxed); + } + + /// Record a failed relay attempt for `script_id`. + /// `is_quota_error` should be true only when the failure is confidently a + /// quota-related error from Apps Script (not a local transport/network failure). + /// Does NOT mark the account exhausted — callers do that separately via + /// `mark_exhausted` when they are confident the account is done. + pub fn record_failure(&self, script_id: &str, is_quota_error: bool) { + let mut st = self.state.lock().unwrap(); + if let Some(bucket) = st.buckets.get_mut(script_id) { + bucket.failed_requests += 1; + if is_quota_error { + bucket.quota_error_count += 1; + } + } + drop(st); + self.dirty_count.fetch_add(1, Ordering::Relaxed); + } + + /// Hard-stop a specific account bucket with an explicit reason. + /// Force-saves to disk immediately so the state survives a crash/restart. + pub fn mark_exhausted(&self, script_id: &str, reason: &str) { + let mut st = self.state.lock().unwrap(); + let bucket = st.buckets.entry(script_id.to_string()).or_insert_with(|| { + AccountBucket { + masked_id: mask_id(script_id), + ..Default::default() + } + }); + bucket.exhausted = true; + bucket.hard_stopped = true; + bucket.exhaustion_reason = Some(reason.to_string()); + drop(st); + self.save(); + } + + // ── Routing queries ────────────────────────────────────────────────────── + + /// Returns true if this account should be excluded from relay dispatch. + pub fn is_hard_stopped(&self, script_id: &str) -> bool { + let st = self.state.lock().unwrap(); + st.buckets + .get(script_id) + .map(|b| b.hard_stopped) + .unwrap_or(false) + } + + /// Returns true when all tracked account buckets are hard-stopped, OR when + /// the aggregate remaining quota has crossed the collective safety threshold + /// AND at least one confirmed quota error has been seen. + /// + /// Conservative by design: random transport failures or local disconnects do + /// NOT trigger a global stop. Only exhaustion of every individual account + /// bucket OR a confirmed aggregate-quota crossing with quota error evidence + /// activates this. + pub fn is_globally_hard_stopped(&self) -> bool { + if self.script_ids.is_empty() { + return false; + } + let st = self.state.lock().unwrap(); + let all_stopped = self.script_ids.iter().all(|sid| { + st.buckets.get(sid).map(|b| b.hard_stopped).unwrap_or(false) + }); + if all_stopped { + return true; + } + // Secondary check: aggregate remaining < N × safety_buffer + // AND at least one quota error signal has been seen (not just network + // failures or local disconnects). + let total_quota_errors: u64 = st.buckets.values().map(|b| b.quota_error_count).sum(); + if total_quota_errors == 0 { + return false; + } + let total_used: u64 = st.buckets.values().map(|b| b.requests_used).sum(); + let total_cap = self.daily_limit * self.script_ids.len() as u64; + let total_remaining = total_cap.saturating_sub(total_used); + let aggregate_reserve = self.safety_buffer * self.script_ids.len() as u64; + total_remaining < aggregate_reserve + } + + // ── Summary ────────────────────────────────────────────────────────────── + + /// Build a point-in-time aggregate summary across all tracked buckets. + pub fn summary(&self) -> QuotaSummary { + let n = self.script_ids.len(); + if n == 0 { + return QuotaSummary::default(); + } + let st = self.state.lock().unwrap(); + let mut used_total = 0u64; + let mut failed_total = 0u64; + let mut bytes_up = 0u64; + let mut bytes_down = 0u64; + let mut bytes_total = 0u64; + let mut exhausted = 0usize; + let mut hard_stopped = 0usize; + let mut next_reset: Option = None; + + for sid in &self.script_ids { + let Some(b) = st.buckets.get(sid) else { continue }; + used_total += b.requests_used; + failed_total += b.failed_requests; + bytes_up += b.bytes_up; + bytes_down += b.bytes_down; + bytes_total += b.bytes_total; + if b.exhausted { exhausted += 1; } + if b.hard_stopped { hard_stopped += 1; } + if !b.hard_stopped { + if let Some(r) = b.next_reset_at { + next_reset = Some(match next_reset { + None => r, + Some(prev) => prev.min(r), + }); + } + } + } + drop(st); + + let capacity = self.daily_limit * n as u64; + // Remaining is capacity minus used, floored at zero. + let remaining = capacity.saturating_sub(used_total); + let global_stop = self.is_globally_hard_stopped(); + + QuotaSummary { + account_count: n, + daily_capacity_total: capacity, + requests_used_total: used_total, + requests_remaining_total: remaining, + failed_requests_total: failed_total, + bytes_up_total: bytes_up, + bytes_down_total: bytes_down, + bytes_total, + exhausted_count: exhausted, + hard_stopped_count: hard_stopped, + global_hard_stop: global_stop, + next_reset_at: next_reset, + } + } + + // ── Persistence ────────────────────────────────────────────────────────── + + /// Write current state to `quota_state.json`. Non-fatal: logs on IO error. + pub fn save(&self) { + let st = self.state.lock().unwrap(); + match serde_json::to_string(&*st) { + Ok(json) => { + if let Err(e) = std::fs::write(&self.state_path, json) { + tracing::warn!("[quota] failed to save state to {}: {}", self.state_path.display(), e); + } + } + Err(e) => { + tracing::warn!("[quota] failed to serialize quota state: {}", e); + } + } + } + + /// Save if any mutations have occurred since the last flush. + /// Called from the periodic 60-second stats task. + pub fn save_if_needed(&self) { + if self.dirty_count.load(Ordering::Relaxed) > 0 { + self.save(); + // Reset after save so next call knows it's clean. + self.dirty_count.store(0, Ordering::Relaxed); + } + } + + /// Build a human-readable startup summary line. + pub fn startup_summary(&self) -> String { + let s = self.summary(); + let now = now_unix(); + let reset_str = s.next_reset_at.map(|r| { + let secs = r.saturating_sub(now); + format!(" next_reset=in {}h {}m", secs / 3600, (secs / 60) % 60) + }).unwrap_or_default(); + + format!( + "[quota] {} account(s) capacity={}/day used={} remaining={}{}", + s.account_count, + s.daily_capacity_total, + s.requests_used_total, + s.requests_remaining_total, + reset_str, + ) + } +} From b010a6acdc9bdcd104688b5b3593d35d257d5925 Mon Sep 17 00:00:00 2001 From: Captain Mirage <87281406+CaptainMirage@users.noreply.github.com> Date: Sat, 23 May 2026 00:13:04 +0330 Subject: [PATCH 2/8] feat: wire quota tracking into relay dispatch and startup logging --- src/domain_fronter.rs | 153 +++++++++++++++++++++++++++++++++++++++--- src/main.rs | 5 ++ src/proxy_server.rs | 28 ++++++++ 3 files changed, 176 insertions(+), 10 deletions(-) diff --git a/src/domain_fronter.rs b/src/domain_fronter.rs index 0e11e764..6e41ca5f 100644 --- a/src/domain_fronter.rs +++ b/src/domain_fronter.rs @@ -42,6 +42,7 @@ use rustls::{ClientConfig, DigitallySignedStruct, SignatureScheme}; use crate::cache::{cache_key, is_cacheable_method, parse_ttl, ResponseCache}; use crate::config::Config; +use crate::quota_tracker::{QuotaSummary, QuotaTracker}; #[derive(Debug, thiserror::Error)] pub enum FronterError { @@ -420,6 +421,10 @@ pub struct DomainFronter { auto_blacklist_strikes: u32, auto_blacklist_window: Duration, auto_blacklist_cooldown: Duration, + /// Per-account quota tracker. One bucket per configured script_id, + /// each treated as a separate Google account per the model assumption. + /// Persists to quota_state.json so quota state survives restarts. + quota_tracker: Arc, /// Per-batch HTTP timeout. Mirrors `Config::request_timeout_secs` /// (#430, masterking32 PR #25). Read by `tunnel_client::fire_batch` /// so a single config field tunes the timeout used everywhere. @@ -594,6 +599,13 @@ impl DomainFronter { tls_h1.alpn_protocols = vec![b"http/1.1".to_vec()]; let tls_connector_h1 = TlsConnector::from(Arc::new(tls_h1)); + // Build quota tracker before script_ids is moved into the struct. + let quota_tracker_arc = Arc::new(QuotaTracker::load( + &script_ids, + config.quota_daily_limit, + config.quota_safety_buffer, + )); + Ok(Self { connect_host: config.google_ip.clone(), sni_hosts: build_sni_pool_for( @@ -639,6 +651,7 @@ impl DomainFronter { auto_blacklist_cooldown: Duration::from_secs( config.auto_blacklist_cooldown_secs.clamp(1, 86400), ), + quota_tracker: quota_tracker_arc, batch_timeout: Duration::from_secs( config.request_timeout_secs.clamp(5, 300), ), @@ -778,9 +791,15 @@ impl DomainFronter { h2_calls: self.h2_calls.load(Ordering::Relaxed), h2_fallbacks: self.h2_fallbacks.load(Ordering::Relaxed), h2_disabled: self.h2_disabled.load(Ordering::Relaxed), + quota: self.quota_tracker.summary(), } } + /// Access the quota tracker for periodic saves and startup logging. + pub fn quota_tracker(&self) -> &Arc { + &self.quota_tracker + } + pub fn num_scripts(&self) -> usize { self.script_ids.len() } @@ -806,11 +825,25 @@ impl DomainFronter { for _ in 0..n { let idx = self.script_idx.fetch_add(1, Ordering::Relaxed); let sid = &self.script_ids[idx % n]; - if !bl.contains_key(sid) { + if !bl.contains_key(sid) && !self.quota_tracker.is_hard_stopped(sid) { return sid.clone(); } } - // All blacklisted: pick whichever comes off cooldown soonest. + // Fallback: prefer a blacklisted-but-not-quota-exhausted account + // over a fully quota-exhausted one (blacklist is transient, quota + // exhaustion is per-window). + let not_exhausted: Vec<_> = bl + .iter() + .filter(|(sid, _)| !self.quota_tracker.is_hard_stopped(sid)) + .collect(); + if let Some((sid, _)) = not_exhausted.iter().min_by_key(|(_, t)| **t) { + let sid = sid.to_string(); + bl.remove(&sid); + return sid; + } + // All accounts are either quota-exhausted or blacklisted. The global + // hard-stop check in do_relay_with_retry will handle the quota case. + // Fall back to soonest-off-blacklist cooldown as a last resort. if let Some((sid, _)) = bl.iter().min_by_key(|(_, t)| **t) { let sid = sid.clone(); bl.remove(&sid); @@ -839,7 +872,10 @@ impl DomainFronter { } let idx = self.script_idx.fetch_add(1, Ordering::Relaxed); let sid = &self.script_ids[idx % n]; - if !bl.contains_key(sid) && !picked.iter().any(|p| p == sid) { + if !bl.contains_key(sid) + && !self.quota_tracker.is_hard_stopped(sid) + && !picked.iter().any(|p| p == sid) + { picked.push(sid.clone()); } } @@ -2349,6 +2385,21 @@ impl DomainFronter { headers: &[(String, String)], body: &[u8], ) -> Result, FronterError> { + // Refuse immediately if every configured account bucket is exhausted. + // Conservative: only triggers when all buckets are hard-stopped OR the + // aggregate remaining quota has crossed the collective safety threshold + // with confirmed quota error evidence (not random network failures). + if self.quota_tracker.is_globally_hard_stopped() { + tracing::error!( + "[quota] global hard stop active — all Apps Script account buckets exhausted" + ); + return Err(FronterError::Relay( + "All Apps Script accounts quota exhausted; hard stop active. \ + Quota resets on a rolling 24-hour window per account." + .into(), + )); + } + // Fan-out path: fire N instances in parallel, return first Ok, cancel // the rest. Clamps to number of available script IDs so the single-ID // case is a no-op even if parallel_relay>1 was configured. @@ -2445,6 +2496,14 @@ impl DomainFronter { self.do_relay_once_with(script_id, method, url, headers, body).await } + /// Quota-recording wrapper around `do_relay_once_inner`. Counts every + /// Apps Script fetch attempt (including retries) against the per-account + /// bucket, records byte metrics on success, and marks an account as + /// hard-stopped when the response carries a confirmed quota error message. + /// + /// Local transport failures (Io, Tls, Timeout) are recorded as failed + /// attempts but do NOT trigger exhaustion — only quota-like Relay errors + /// qualify, keeping transient network issues from false-stopping accounts. async fn do_relay_once_with( &self, script_id: String, @@ -2453,12 +2512,49 @@ impl DomainFronter { headers: &[(String, String)], body: &[u8], ) -> Result, FronterError> { - // Build once, wrap in Bytes (zero-copy move). h2 takes a clone - // (Arc bump, not memcpy); h1 fallback uses the same Bytes via - // Deref<&[u8]>. Saves a full payload allocation+copy per call - // — meaningful on range-parallel fan-out where N copies fire - // in parallel for one user-facing GET. let payload: Bytes = Bytes::from(self.build_payload_json(method, url, headers, body)?); + let bytes_up = payload.len() as u64; + + // Count ALL attempts, including retries. Each call here maps to one + // real UrlFetchApp.fetch() on Google's side — that's the unit Google + // bills against the daily quota. + self.quota_tracker.record_attempt(&script_id, bytes_up); + + let result = self + .do_relay_once_inner(script_id.clone(), method, url, payload) + .await; + + match &result { + Ok(bytes) => { + self.quota_tracker + .record_success(&script_id, bytes.len() as u64); + } + Err(e) => { + let is_quota = is_quota_like_fronter_error(e); + self.quota_tracker.record_failure(&script_id, is_quota); + if is_quota { + self.quota_tracker + .mark_exhausted(&script_id, &e.to_string()); + tracing::warn!( + "[quota] account {} exhausted: {}", + mask_script_id(&script_id), + e + ); + } + } + } + + result + } + + async fn do_relay_once_inner( + &self, + script_id: String, + method: &str, + url: &str, + payload: Bytes, + ) -> Result, FronterError> { + // payload already built by the caller; path derived from script_id. let path = format!("/macros/s/{}/exec", script_id); // h2 fast path: one shared TCP/TLS connection multiplexes all @@ -4831,6 +4927,9 @@ pub struct StatsSnapshot { /// switch set, or peer refused h2 during ALPN). All traffic on the /// h1 path. pub h2_disabled: bool, + /// Quota state snapshot. Only meaningful in AppsScript/Full modes where + /// a DomainFronter is active; defaults to zero values in Direct mode. + pub quota: QuotaSummary, } impl StatsSnapshot { @@ -4863,8 +4962,22 @@ impl StatsSnapshot { ) } }; + let q = &self.quota; + let quota_seg = if q.account_count > 0 && (q.exhausted_count > 0 || q.global_hard_stop) { + format!( + " quota={}/{} remaining={} exhausted={}/{}{}", + q.requests_used_total, + q.daily_capacity_total, + q.requests_remaining_total, + q.exhausted_count, + q.account_count, + if q.global_hard_stop { " HARD-STOP" } else { "" }, + ) + } else { + String::new() + }; format!( - "stats: relay={} ({}KB) failures={} coalesced={} cache={}/{} ({:.0}% hit, {}KB) scripts={}/{} active{}", + "stats: relay={} ({}KB) failures={} coalesced={} cache={}/{} ({:.0}% hit, {}KB) scripts={}/{} active{}{}", self.relay_calls, self.bytes_relayed / 1024, self.relay_failures, @@ -4876,6 +4989,7 @@ impl StatsSnapshot { self.total_scripts - self.blacklisted_scripts, self.total_scripts, h2_seg, + quota_seg, ) } @@ -4887,8 +5001,9 @@ impl StatsSnapshot { fn esc(s: &str) -> String { s.replace('\\', "\\\\").replace('"', "\\\"") } + let q = &self.quota; format!( - r#"{{"relay_calls":{},"relay_failures":{},"coalesced":{},"bytes_relayed":{},"cache_hits":{},"cache_misses":{},"cache_bytes":{},"blacklisted_scripts":{},"total_scripts":{},"today_calls":{},"today_bytes":{},"today_key":"{}","today_reset_secs":{},"h2_calls":{},"h2_fallbacks":{},"h2_disabled":{}}}"#, + r#"{{"relay_calls":{},"relay_failures":{},"coalesced":{},"bytes_relayed":{},"cache_hits":{},"cache_misses":{},"cache_bytes":{},"blacklisted_scripts":{},"total_scripts":{},"today_calls":{},"today_bytes":{},"today_key":"{}","today_reset_secs":{},"h2_calls":{},"h2_fallbacks":{},"h2_disabled":{},"quota_account_count":{},"quota_capacity":{},"quota_used":{},"quota_remaining":{},"quota_exhausted":{},"quota_hard_stop":{}}}"#, self.relay_calls, self.relay_failures, self.coalesced, @@ -4905,6 +5020,12 @@ impl StatsSnapshot { self.h2_calls, self.h2_fallbacks, self.h2_disabled, + q.account_count, + q.daily_capacity_total, + q.requests_used_total, + q.requests_remaining_total, + q.exhausted_count, + q.global_hard_stop, ) } } @@ -4916,6 +5037,18 @@ fn should_blacklist(status: u16, body: &str) -> bool { looks_like_quota_error(body) } +/// True only when the error is a Relay-level message that looks like a quota +/// signal from Apps Script. Io/Tls/Timeout errors are local transport issues +/// and must NOT trigger account exhaustion — that would false-stop accounts on +/// any network glitch. +fn is_quota_like_fronter_error(e: &FronterError) -> bool { + match e { + FronterError::Relay(msg) => looks_like_quota_error(msg), + FronterError::NonRetryable(inner) => is_quota_like_fronter_error(inner), + _ => false, + } +} + fn looks_like_quota_error(msg: &str) -> bool { let lower = msg.to_ascii_lowercase(); lower.contains("quota") diff --git a/src/main.rs b/src/main.rs index 72e5aefb..bebd45d6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -357,6 +357,11 @@ async fn main() -> ExitCode { } }; + // Log quota state on startup so the user knows where their daily budget stands. + if let Some(summary) = server.quota_startup_summary() { + tracing::info!("{}", summary); + } + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); let run = server.run(shutdown_rx); diff --git a/src/proxy_server.rs b/src/proxy_server.rs index 209bbc58..7c82ca93 100644 --- a/src/proxy_server.rs +++ b/src/proxy_server.rs @@ -533,6 +533,14 @@ impl ProxyServer { pub fn fronter(&self) -> Option> { self.fronter.clone() } + /// Returns a formatted quota startup summary for logging, or None in + /// Direct mode (no fronter / no quota to report). + pub fn quota_startup_summary(&self) -> Option { + self.fronter + .as_ref() + .map(|f| f.quota_tracker().startup_summary()) + } + pub async fn run( mut self, mut shutdown_rx: tokio::sync::oneshot::Receiver<()>, @@ -617,6 +625,26 @@ impl ProxyServer { if s.relay_calls > 0 || s.cache_hits > 0 { tracing::info!("{}", s.fmt_line()); } + // Log quota warnings and flush persisted state. + let q = &s.quota; + if q.global_hard_stop { + tracing::error!( + "[quota] GLOBAL HARD STOP — all {} account(s) exhausted", + q.account_count + ); + } else if q.exhausted_count > 0 { + tracing::warn!( + "[quota] {}/{} account(s) exhausted used={}/{} remaining={}", + q.exhausted_count, + q.account_count, + q.requests_used_total, + q.daily_capacity_total, + q.requests_remaining_total, + ); + } + // Periodic flush so quota state survives a crash between + // the every-50-requests auto-saves in record_attempt. + stats_fronter.quota_tracker().save_if_needed(); } }) } else { From 425e309b4ae6bfb14034088100d3e63dcd243321 Mon Sep 17 00:00:00 2001 From: Captain Mirage <87281406+CaptainMirage@users.noreply.github.com> Date: Sat, 23 May 2026 00:27:00 +0330 Subject: [PATCH 3/8] feat(ui): add quota usage display to Usage Today section --- src/bin/ui.rs | 244 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 193 insertions(+), 51 deletions(-) diff --git a/src/bin/ui.rs b/src/bin/ui.rs index 44587964..86d44d17 100644 --- a/src/bin/ui.rs +++ b/src/bin/ui.rs @@ -154,6 +154,11 @@ struct UiState { /// One-line status of the most recent download (Ok(path) or Err(msg)). last_download: Option>, last_download_at: Option, + /// Quota state from the most recent PollStats snapshot. None until the + /// first poll completes. Used by the quota display and hard-stop indicator. + /// TODO(quota-dashboard): feed this into a dedicated QuotaWidget once + /// the UI is remodeled. + quota: Option, } #[derive(Clone, Debug)] @@ -1167,7 +1172,7 @@ impl eframe::App for App { ui.add_space(8.0); // ── Status + stats card ──────────────────────────────────────── - let (running, started_at, stats, ca_trusted, last_test_msg, per_site) = { + let (running, started_at, stats, ca_trusted, last_test_msg, per_site, quota_state) = { let s = self.shared.state.lock().unwrap(); ( s.running, @@ -1176,6 +1181,7 @@ impl eframe::App for App { s.ca_trusted, s.last_test_msg.clone(), s.last_per_site.clone(), + s.quota.clone(), ) }; @@ -1251,66 +1257,200 @@ impl eframe::App for App { }); // ── Usage today (estimated) — daily budget tracker ─────────────── - // Client-side estimate from our own atomic counters. Counts only - // successful relay calls this process saw since 00:00 UTC. Google's - // actual quota bucket is per-Apps-Script-project and per-Google - // account — if multiple devices share the same deployment, each - // client only sees its own share. We link to the Google dashboard - // for the authoritative number. + // TODO(quota-dashboard): Replace this inline grid with a dedicated + // QuotaWidget / QuotaDashboard when the UI is remodeled. The quota + // state is already wired through UiState.quota and StatsSnapshot.quota. if let Some(s) = &stats { ui.add_space(2.0); - section(ui, "Usage today (estimated)", |ui| { - // Free-tier Apps Script UrlFetchApp quota. Workspace / - // paid accounts get 100k but most users are on free. - const FREE_QUOTA_PER_DAY: u64 = 20_000; - let pct = if FREE_QUOTA_PER_DAY > 0 { - (s.today_calls as f64 / FREE_QUOTA_PER_DAY as f64) * 100.0 - } else { 0.0 }; - let reset = s.today_reset_secs; - let reset_str = format!( - "{}h {}m", - reset / 3600, - (reset / 60) % 60, - ); - let rows: Vec<(&str, String)> = vec![ - ( - "calls today", + + // Show a red hard-stop banner when all accounts are exhausted. + if let Some(q) = "a_state { + if q.global_hard_stop { + ui.colored_label( + egui::Color32::from_rgb(220, 80, 80), + "⚠ All account quota exhausted — Apps Script relay hard-stopped", + ); + // TODO(quota-dashboard): disable the Start button here once + // the button state wiring supports conditional disabling. + ui.add_space(4.0); + } else if q.exhausted_count > 0 { + ui.colored_label( + egui::Color32::from_rgb(220, 170, 80), format!( - "{} / {} ({:.1}%)", - s.today_calls, FREE_QUOTA_PER_DAY, pct + "⚠ {}/{} account(s) exhausted — routing to remaining accounts", + q.exhausted_count, q.account_count ), - ), - ("bytes today", fmt_bytes(s.today_bytes)), - ("PT day", s.today_key.clone()), - ("resets in", reset_str), - ]; + ); + ui.add_space(2.0); + } + } + + section(ui, "Usage today (estimated)", |ui| { + // Use quota-tracked daily capacity when available, fall back to + // the free-tier default for display purposes. + let (quota_cap, quota_used, quota_remaining, any_exhausted, global_stop) = + if let Some(q) = "a_state { + ( + q.daily_capacity_total.max(1), + q.requests_used_total, + q.requests_remaining_total, + q.exhausted_count > 0, + q.global_hard_stop, + ) + } else { + (20_000u64, s.today_calls, 20_000u64.saturating_sub(s.today_calls), false, false) + }; + + let pct = (quota_used as f64 / quota_cap as f64) * 100.0; + let alert = any_exhausted || global_stop; + + // calls today — turns red when any account is exhausted + let calls_str = format!("{} / {} ({:.1}%)", quota_used, quota_cap, pct); + let calls_text = if alert { + egui::RichText::new(&calls_str) + .monospace() + .color(egui::Color32::from_rgb(220, 80, 80)) + } else { + egui::RichText::new(&calls_str).monospace() + }; + + // quota remaining — same red treatment + let remaining_str = quota_remaining.to_string(); + let remaining_text = if alert { + egui::RichText::new(&remaining_str) + .monospace() + .color(egui::Color32::from_rgb(220, 80, 80)) + } else { + egui::RichText::new(&remaining_str).monospace() + }; + + // Next reset from the quota tracker's rolling window, or PT midnight + let reset_str = if let Some(q) = "a_state { + if let Some(next_reset) = q.next_reset_at { + use std::time::{SystemTime, UNIX_EPOCH}; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let secs = next_reset.saturating_sub(now); + format!("{}h {}m (rolling)", secs / 3600, (secs / 60) % 60) + } else { + let reset = s.today_reset_secs; + format!("{}h {}m (PT midnight)", reset / 3600, (reset / 60) % 60) + } + } else { + let reset = s.today_reset_secs; + format!("{}h {}m", reset / 3600, (reset / 60) % 60) + }; + egui::Grid::new("usage_today") .num_columns(4) .spacing([16.0, 4.0]) .show(ui, |ui| { - for chunk in rows.chunks(2) { - for (label, value) in chunk.iter() { - ui.add_sized( - [110.0, 18.0], - egui::Label::new( - egui::RichText::new(*label) - .color(egui::Color32::from_gray(150)), - ), - ); - ui.add_sized( - [140.0, 18.0], - egui::Label::new( - egui::RichText::new(value).monospace(), - ), - ); - } - if chunk.len() == 1 { - ui.label(""); - ui.label(""); - } - ui.end_row(); + // Row 1: calls today | bytes today + ui.add_sized( + [110.0, 18.0], + egui::Label::new( + egui::RichText::new("calls today") + .color(egui::Color32::from_gray(150)), + ), + ); + ui.add_sized( + [140.0, 18.0], + egui::Label::new(calls_text), + ); + ui.add_sized( + [110.0, 18.0], + egui::Label::new( + egui::RichText::new("bytes today") + .color(egui::Color32::from_gray(150)), + ), + ); + ui.add_sized( + [140.0, 18.0], + egui::Label::new( + egui::RichText::new(fmt_bytes(s.today_bytes)).monospace(), + ), + ); + ui.end_row(); + + // Row 2: quota remaining | next reset + ui.add_sized( + [110.0, 18.0], + egui::Label::new( + egui::RichText::new("remaining") + .color(egui::Color32::from_gray(150)), + ), + ); + ui.add_sized( + [140.0, 18.0], + egui::Label::new(remaining_text), + ); + ui.add_sized( + [110.0, 18.0], + egui::Label::new( + egui::RichText::new("resets in") + .color(egui::Color32::from_gray(150)), + ), + ); + ui.add_sized( + [140.0, 18.0], + egui::Label::new( + egui::RichText::new(&reset_str).monospace(), + ), + ); + ui.end_row(); + + // Row 3: PT day | accounts (if quota available) + ui.add_sized( + [110.0, 18.0], + egui::Label::new( + egui::RichText::new("PT day") + .color(egui::Color32::from_gray(150)), + ), + ); + ui.add_sized( + [140.0, 18.0], + egui::Label::new( + egui::RichText::new(&s.today_key).monospace(), + ), + ); + if let Some(q) = "a_state { + ui.add_sized( + [110.0, 18.0], + egui::Label::new( + egui::RichText::new("accounts") + .color(egui::Color32::from_gray(150)), + ), + ); + let acct_str = if q.exhausted_count > 0 { + format!( + "{}/{} ({} exhausted)", + q.account_count - q.exhausted_count, + q.account_count, + q.exhausted_count, + ) + } else { + format!("{}/{} active", q.account_count, q.account_count) + }; + let acct_text = if q.exhausted_count > 0 { + egui::RichText::new(&acct_str) + .monospace() + .color(egui::Color32::from_rgb(220, 170, 80)) + } else { + egui::RichText::new(&acct_str).monospace() + }; + ui.add_sized( + [140.0, 18.0], + egui::Label::new(acct_text), + ); + } else { + ui.label(""); + ui.label(""); } + ui.end_row(); }); + ui.add_space(4.0); ui.horizontal(|ui| { ui.hyperlink_to( @@ -1997,9 +2137,11 @@ fn background_thread(shared: Arc, rx: Receiver) { if let Some(fronter) = f.as_ref() { let s = fronter.snapshot_stats(); let per_site = fronter.snapshot_per_site(); + let quota = s.quota.clone(); let mut st = shared.state.lock().unwrap(); st.last_stats = Some(s); st.last_per_site = per_site; + st.quota = Some(quota); } }); } From a80a69e9f41b50f755bbb38a027bebe548cd7a66 Mon Sep 17 00:00:00 2001 From: Captain Mirage <87281406+CaptainMirage@users.noreply.github.com> Date: Sat, 23 May 2026 20:34:15 +0330 Subject: [PATCH 4/8] fix: quota safety buffer on load, idle window rollover, startup flush --- src/domain_fronter.rs | 19 ++++++++ src/proxy_server.rs | 10 ++-- src/quota_tracker.rs | 104 ++++++++++++++++++++++++++++++++++++------ 3 files changed, 115 insertions(+), 18 deletions(-) diff --git a/src/domain_fronter.rs b/src/domain_fronter.rs index 6e41ca5f..a81b2f01 100644 --- a/src/domain_fronter.rs +++ b/src/domain_fronter.rs @@ -2512,6 +2512,16 @@ impl DomainFronter { headers: &[(String, String)], body: &[u8], ) -> Result, FronterError> { + // Defense-in-depth: if next_script_id's last-resort fallback handed us + // a hard-stopped account (all exhausted, none in the blacklist), refuse + // here before building the payload or touching the network. + if self.quota_tracker.is_hard_stopped(&script_id) { + return Err(FronterError::Relay(format!( + "account {} is quota-hard-stopped; skipping dispatch", + mask_script_id(&script_id), + ))); + } + let payload: Bytes = Bytes::from(self.build_payload_json(method, url, headers, body)?); let bytes_up = payload.len() as u64; @@ -2544,6 +2554,15 @@ impl DomainFronter { } } + tracing::debug!( + "[quota] {} dispatch result: {}", + mask_script_id(&script_id), + match &result { + Ok(_) => "Ok".to_string(), + Err(e) => format!("Err({})", e), + }, + ); + result } diff --git a/src/proxy_server.rs b/src/proxy_server.rs index 7c82ca93..a1658ef2 100644 --- a/src/proxy_server.rs +++ b/src/proxy_server.rs @@ -642,9 +642,13 @@ impl ProxyServer { q.requests_remaining_total, ); } - // Periodic flush so quota state survives a crash between - // the every-50-requests auto-saves in record_attempt. - stats_fronter.quota_tracker().save_if_needed(); + // Roll any expired 24-hour windows so idle accounts come + // back online even without inbound traffic. + stats_fronter.quota_tracker().roll_expired_windows(); + // Always flush so the file is up-to-date even when idle. + // save_if_needed() skips the write when dirty_count == 0, + // which means zero-traffic sessions never update the file. + stats_fronter.quota_tracker().save(); } }) } else { diff --git a/src/quota_tracker.rs b/src/quota_tracker.rs index 95a3fddd..24acbf22 100644 --- a/src/quota_tracker.rs +++ b/src/quota_tracker.rs @@ -88,6 +88,10 @@ pub struct QuotaSummary { pub global_hard_stop: bool, /// Unix timestamp of the soonest window reset across all non-exhausted buckets. pub next_reset_at: Option, + /// Unix timestamp of the soonest window reset across ALL buckets, including + /// hard-stopped ones. Used by the UI to show a meaningful reset time even + /// when all accounts are exhausted. + pub next_reset_at_any: Option, } // ── Disk state wrapper ──────────────────────────────────────────────────────── @@ -121,6 +125,26 @@ fn quota_state_path() -> PathBuf { data_dir::data_dir().join("quota_state.json") } +/// Mark any bucket that is already past the safety buffer as hard-stopped. +/// Called at load time so accounts near the limit are blocked before the +/// first request arrives, not after it fires. +fn check_all_safety_buffers(qs: &mut QuotaState, daily_limit: u64, safety_buffer: u64) { + for bucket in qs.buckets.values_mut() { + if bucket.hard_stopped { + continue; + } + let remaining = daily_limit.saturating_sub(bucket.requests_used); + if remaining < safety_buffer { + bucket.exhausted = true; + bucket.hard_stopped = true; + bucket.exhaustion_reason = Some(format!( + "safety buffer crossed on load: {}/{} requests used (limit {}, buffer {})", + bucket.requests_used, daily_limit, daily_limit, safety_buffer, + )); + } + } +} + fn mask_id(id: &str) -> String { let n = id.chars().count(); if n <= 8 { @@ -157,14 +181,24 @@ impl QuotaTracker { }); } - Self { + // Pre-check safety buffers on loaded state so accounts that are already + // near the limit are marked hard_stopped before the first request arrives, + // not after it fires. + check_all_safety_buffers(&mut qs, daily_limit, safety_buffer); + + let tracker = Self { state: Mutex::new(qs), script_ids: script_ids.to_vec(), daily_limit, safety_buffer, dirty_count: AtomicU64::new(0), state_path, - } + }; + // Always write on startup so the file exists and reflects current state + // before any relay traffic arrives. Without this, the file is only created + // after the first dirty_count increment (i.e. after real traffic). + tracker.save(); + tracker } // ── Recording ──────────────────────────────────────────────────────────── @@ -206,22 +240,16 @@ impl QuotaTracker { bucket.next_reset_at = Some(now + 86_400); } - bucket.requests_used += 1; - bucket.bytes_up += bytes_up; - bucket.bytes_total += bytes_up; - bucket.last_request_at = Some(now); - - // Check if the safety buffer threshold is crossed (soft exhaustion). - // This is separate from mark_exhausted which is called on quota error - // signals; this handles the case where the counter itself reaches the - // limit without a quota error message being returned yet. - let remaining = self.daily_limit.saturating_sub(bucket.requests_used); + // Check safety buffer BEFORE incrementing so the account is stopped on + // the request that would exceed the limit, not the one after it. + let next_used = bucket.requests_used + 1; + let remaining = self.daily_limit.saturating_sub(next_used); if !bucket.hard_stopped && remaining < self.safety_buffer { bucket.exhausted = true; bucket.hard_stopped = true; bucket.exhaustion_reason = Some(format!( "safety buffer crossed: {}/{} requests used (limit {}, buffer {})", - bucket.requests_used, self.daily_limit, + next_used, self.daily_limit, self.daily_limit, self.safety_buffer, )); tracing::warn!( @@ -230,6 +258,11 @@ impl QuotaTracker { ); } + bucket.requests_used = next_used; + bucket.bytes_up += bytes_up; + bucket.bytes_total += bytes_up; + bucket.last_request_at = Some(now); + drop(st); let n = self.dirty_count.fetch_add(1, Ordering::Relaxed); if n % 50 == 0 { @@ -343,6 +376,7 @@ impl QuotaTracker { let mut exhausted = 0usize; let mut hard_stopped = 0usize; let mut next_reset: Option = None; + let mut next_reset_any: Option = None; for sid in &self.script_ids { let Some(b) = st.buckets.get(sid) else { continue }; @@ -353,8 +387,13 @@ impl QuotaTracker { bytes_total += b.bytes_total; if b.exhausted { exhausted += 1; } if b.hard_stopped { hard_stopped += 1; } - if !b.hard_stopped { - if let Some(r) = b.next_reset_at { + if let Some(r) = b.next_reset_at { + // next_reset_any covers all accounts including stopped ones. + next_reset_any = Some(match next_reset_any { + None => r, + Some(prev) => prev.min(r), + }); + if !b.hard_stopped { next_reset = Some(match next_reset { None => r, Some(prev) => prev.min(r), @@ -382,6 +421,7 @@ impl QuotaTracker { hard_stopped_count: hard_stopped, global_hard_stop: global_stop, next_reset_at: next_reset, + next_reset_at_any: next_reset_any, } } @@ -412,6 +452,40 @@ impl QuotaTracker { } } + /// Roll any expired 24-hour windows for all tracked buckets. + /// Called from the 60-second stats task so windows reset even when the + /// proxy is idle and no new requests arrive to trigger record_attempt. + pub fn roll_expired_windows(&self) { + let now = now_unix(); + let mut st = self.state.lock().unwrap(); + let mut rolled = false; + for bucket in st.buckets.values_mut() { + if let Some(reset_at) = bucket.next_reset_at { + if now >= reset_at { + bucket.requests_used = 0; + bucket.failed_requests = 0; + bucket.bytes_up = 0; + bucket.bytes_down = 0; + bucket.bytes_total = 0; + bucket.next_reset_at = Some(now + 86_400); + bucket.exhausted = false; + bucket.hard_stopped = false; + bucket.exhaustion_reason = None; + bucket.quota_error_count = 0; + rolled = true; + tracing::info!( + "[quota] account {} window rolled (idle expiry) — quota reset", + bucket.masked_id, + ); + } + } + } + drop(st); + if rolled { + self.dirty_count.fetch_add(1, Ordering::Relaxed); + } + } + /// Build a human-readable startup summary line. pub fn startup_summary(&self) -> String { let s = self.summary(); From c7e860c18a6baea0c52a1f05ae4661d99934902b Mon Sep 17 00:00:00 2001 From: Captain Mirage <87281406+CaptainMirage@users.noreply.github.com> Date: Sat, 23 May 2026 20:34:27 +0330 Subject: [PATCH 5/8] =?UTF-8?q?fix(ui):=20usage=20today=20section=20?= =?UTF-8?q?=E2=80=94=20hard-stop=20banner,=20resets,=20data=20estimate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/ui.rs | 243 ++++++++++++++++++++++++-------------------------- 1 file changed, 119 insertions(+), 124 deletions(-) diff --git a/src/bin/ui.rs b/src/bin/ui.rs index 86d44d17..52da83e7 100644 --- a/src/bin/ui.rs +++ b/src/bin/ui.rs @@ -1185,110 +1185,46 @@ impl eframe::App for App { ) }; - let status_title = if running { - let up = started_at.map(|t| t.elapsed()).unwrap_or_default(); - format!("Traffic · uptime {}", fmt_duration(up)) - } else { - "Traffic · (not running)".to_string() - }; - section(ui, &status_title, |ui| { - if let Some(s) = &stats { - // Compact two-column layout so 7 metrics fit in ~4 rows - // instead of a tall vertical strip. - let rows: Vec<(&str, String)> = vec![ - ("relay calls", s.relay_calls.to_string()), - ("failures", s.relay_failures.to_string()), - ("coalesced", s.coalesced.to_string()), - ( - "cache hits", - format!( - "{} / {} ({:.0}%)", - s.cache_hits, - s.cache_hits + s.cache_misses, - s.hit_rate() - ), - ), - ("cache size", format!("{} KB", s.cache_bytes / 1024)), - ("bytes relayed", fmt_bytes(s.bytes_relayed)), - ( - "active scripts", - format!( - "{} / {}", - s.total_scripts - s.blacklisted_scripts, - s.total_scripts - ), - ), - ]; - egui::Grid::new("stats") - .num_columns(4) - .spacing([16.0, 4.0]) - .show(ui, |ui| { - for chunk in rows.chunks(2) { - for (label, value) in chunk.iter() { - ui.add_sized( - [110.0, 18.0], - egui::Label::new( - egui::RichText::new(*label) - .color(egui::Color32::from_gray(150)), - ), - ); - ui.add_sized( - [140.0, 18.0], - egui::Label::new( - egui::RichText::new(value).monospace(), - ), - ); - } - // Pad the final short row so grid columns stay aligned. - if chunk.len() == 1 { - ui.label(""); - ui.label(""); - } - ui.end_row(); - } - }); - } else { - ui.label( - egui::RichText::new("No traffic yet — click Start and send a request.") - .color(egui::Color32::from_gray(150)) - .italics(), - ); - } - }); - - // ── Usage today (estimated) — daily budget tracker ─────────────── + // ── Usage today — daily budget tracker + traffic stats ─────────── // TODO(quota-dashboard): Replace this inline grid with a dedicated // QuotaWidget / QuotaDashboard when the UI is remodeled. The quota // state is already wired through UiState.quota and StatsSnapshot.quota. if let Some(s) = &stats { ui.add_space(2.0); - // Show a red hard-stop banner when all accounts are exhausted. - if let Some(q) = "a_state { - if q.global_hard_stop { - ui.colored_label( - egui::Color32::from_rgb(220, 80, 80), - "⚠ All account quota exhausted — Apps Script relay hard-stopped", - ); - // TODO(quota-dashboard): disable the Start button here once - // the button state wiring supports conditional disabling. - ui.add_space(4.0); - } else if q.exhausted_count > 0 { - ui.colored_label( - egui::Color32::from_rgb(220, 170, 80), - format!( - "⚠ {}/{} account(s) exhausted — routing to remaining accounts", - q.exhausted_count, q.account_count - ), - ); - ui.add_space(2.0); + let usage_title = if running { + let up = started_at.map(|t| t.elapsed()).unwrap_or_default(); + format!("Usage today · uptime {}", fmt_duration(up)) + } else { + "Usage today · (not running)".to_string() + }; + + section(ui, &usage_title, |ui| { + // Hard-stop / exhaustion banners at the top of the section. + if let Some(q) = "a_state { + if q.global_hard_stop { + ui.colored_label( + egui::Color32::from_rgb(220, 80, 80), + "⚠ All account quota exhausted — Apps Script relay hard-stopped", + ); + // TODO(quota-dashboard): disable the Start button here once + // the button state wiring supports conditional disabling. + ui.add_space(4.0); + } else if q.exhausted_count > 0 { + ui.colored_label( + egui::Color32::from_rgb(220, 170, 80), + format!( + "⚠ {}/{} account(s) exhausted — routing to remaining accounts", + q.exhausted_count, q.account_count + ), + ); + ui.add_space(2.0); + } } - } - section(ui, "Usage today (estimated)", |ui| { // Use quota-tracked daily capacity when available, fall back to // the free-tier default for display purposes. - let (quota_cap, quota_used, quota_remaining, any_exhausted, global_stop) = + let (quota_cap, quota_used, _quota_remaining, any_exhausted, global_stop) = if let Some(q) = "a_state { ( q.daily_capacity_total.max(1), @@ -1304,7 +1240,7 @@ impl eframe::App for App { let pct = (quota_used as f64 / quota_cap as f64) * 100.0; let alert = any_exhausted || global_stop; - // calls today — turns red when any account is exhausted + // fetches today — turns red when any account is exhausted let calls_str = format!("{} / {} ({:.1}%)", quota_used, quota_cap, pct); let calls_text = if alert { egui::RichText::new(&calls_str) @@ -1314,25 +1250,18 @@ impl eframe::App for App { egui::RichText::new(&calls_str).monospace() }; - // quota remaining — same red treatment - let remaining_str = quota_remaining.to_string(); - let remaining_text = if alert { - egui::RichText::new(&remaining_str) - .monospace() - .color(egui::Color32::from_rgb(220, 80, 80)) - } else { - egui::RichText::new(&remaining_str).monospace() - }; - - // Next reset from the quota tracker's rolling window, or PT midnight + // Next reset: use rolling window from active accounts, fall back to + // soonest reset across ALL accounts (including stopped ones) when all + // are exhausted, then finally PT midnight as last resort. let reset_str = if let Some(q) = "a_state { - if let Some(next_reset) = q.next_reset_at { - use std::time::{SystemTime, UNIX_EPOCH}; - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - let secs = next_reset.saturating_sub(now); + use std::time::{SystemTime, UNIX_EPOCH}; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let reset_ts = q.next_reset_at.or(q.next_reset_at_any); + if let Some(ts) = reset_ts { + let secs = ts.saturating_sub(now); format!("{}h {}m (rolling)", secs / 3600, (secs / 60) % 60) } else { let reset = s.today_reset_secs; @@ -1347,11 +1276,11 @@ impl eframe::App for App { .num_columns(4) .spacing([16.0, 4.0]) .show(ui, |ui| { - // Row 1: calls today | bytes today + // Row 1: fetches today | bytes relayed ui.add_sized( [110.0, 18.0], egui::Label::new( - egui::RichText::new("calls today") + egui::RichText::new("fetches today") .color(egui::Color32::from_gray(150)), ), ); @@ -1362,46 +1291,73 @@ impl eframe::App for App { ui.add_sized( [110.0, 18.0], egui::Label::new( - egui::RichText::new("bytes today") + egui::RichText::new("bytes relayed") .color(egui::Color32::from_gray(150)), ), ); ui.add_sized( [140.0, 18.0], egui::Label::new( - egui::RichText::new(fmt_bytes(s.today_bytes)).monospace(), + egui::RichText::new(fmt_bytes(s.bytes_relayed)).monospace(), ), ); ui.end_row(); - // Row 2: quota remaining | next reset + // Row 2: next reset (remaining removed — already shown in fetches today X/Y) ui.add_sized( [110.0, 18.0], egui::Label::new( - egui::RichText::new("remaining") + egui::RichText::new("resets in") .color(egui::Color32::from_gray(150)), ), ); ui.add_sized( [140.0, 18.0], - egui::Label::new(remaining_text), + egui::Label::new( + egui::RichText::new(&reset_str).monospace(), + ), ); + ui.label(""); + ui.label(""); + ui.end_row(); + + // Row 3: relay calls+failures | cache + let relay_str = if s.relay_failures > 0 { + format!("{} ({} failed)", s.relay_calls, s.relay_failures) + } else { + s.relay_calls.to_string() + }; ui.add_sized( [110.0, 18.0], egui::Label::new( - egui::RichText::new("resets in") + egui::RichText::new("relay calls") .color(egui::Color32::from_gray(150)), ), ); ui.add_sized( [140.0, 18.0], + egui::Label::new(egui::RichText::new(&relay_str).monospace()), + ); + let cache_total = s.cache_hits + s.cache_misses; + let cache_str = if cache_total > 0 { + format!("{}/{} ({:.0}% hit)", s.cache_hits, cache_total, s.hit_rate()) + } else { + "—".to_string() + }; + ui.add_sized( + [110.0, 18.0], egui::Label::new( - egui::RichText::new(&reset_str).monospace(), + egui::RichText::new("cache") + .color(egui::Color32::from_gray(150)), ), ); + ui.add_sized( + [140.0, 18.0], + egui::Label::new(egui::RichText::new(&cache_str).monospace()), + ); ui.end_row(); - // Row 3: PT day | accounts (if quota available) + // Row 4: PT day | accounts (if quota available) ui.add_sized( [110.0, 18.0], egui::Label::new( @@ -1449,7 +1405,40 @@ impl eframe::App for App { ui.label(""); } ui.end_row(); - }); + + // Row 5: data transferred with estimated daily total + if let Some(q) = "a_state { + if q.bytes_total > 0 { + let data_str = if q.requests_used_total > 0 { + let avg = q.bytes_total as f64 / q.requests_used_total as f64; + let est = (avg * q.daily_capacity_total as f64) as u64; + format!( + "{} / {} (est.)", + fmt_bytes_approx(q.bytes_total), + fmt_bytes_approx(est), + ) + } else { + fmt_bytes_approx(q.bytes_total) + }; + ui.add_sized( + [110.0, 18.0], + egui::Label::new( + egui::RichText::new("data transferred") + .color(egui::Color32::from_gray(150)), + ), + ); + ui.add_sized( + [140.0, 18.0], + egui::Label::new( + egui::RichText::new(&data_str).monospace(), + ), + ); + ui.label(""); + ui.label(""); + ui.end_row(); + } + } + }); // end egui::Grid "usage_today" ui.add_space(4.0); ui.horizontal(|ui| { @@ -2115,6 +2104,12 @@ fn fmt_bytes(b: u64) -> String { } } +/// Approximate bytes display with a `~` prefix, used for quota tracker totals +/// where the value is an estimate (bytes_up + bytes_down from relay payloads). +fn fmt_bytes_approx(b: u64) -> String { + format!("~{}", fmt_bytes(b)) +} + // ---------- Background thread: owns the tokio runtime + proxy lifecycle ---------- fn background_thread(shared: Arc, rx: Receiver) { From 92b20d068d13013a67ea0f2e535b8108ddba6840 Mon Sep 17 00:00:00 2001 From: Captain Mirage <87281406+CaptainMirage@users.noreply.github.com> Date: Sun, 24 May 2026 02:40:02 +0330 Subject: [PATCH 6/8] fix: global hard stop coverage, 1s save task, exhaustion detail logging --- src/bin/ui.rs | 6 ++++-- src/domain_fronter.rs | 27 +++++++++++++++++++++++++++ src/main.rs | 5 ----- src/proxy_server.rs | 37 ++++++++++++++++++++++++++++++------- src/quota_tracker.rs | 29 ++++++++++++++++++++++++++++- 5 files changed, 89 insertions(+), 15 deletions(-) diff --git a/src/bin/ui.rs b/src/bin/ui.rs index 52da83e7..44900346 100644 --- a/src/bin/ui.rs +++ b/src/bin/ui.rs @@ -1224,17 +1224,19 @@ impl eframe::App for App { // Use quota-tracked daily capacity when available, fall back to // the free-tier default for display purposes. + // quota_used: total relay() calls (exit node + Apps Script combined) + // so "fetches today" reflects all proxied traffic, not just Apps Script. let (quota_cap, quota_used, _quota_remaining, any_exhausted, global_stop) = if let Some(q) = "a_state { ( q.daily_capacity_total.max(1), - q.requests_used_total, + s.total_relay_calls, q.requests_remaining_total, q.exhausted_count > 0, q.global_hard_stop, ) } else { - (20_000u64, s.today_calls, 20_000u64.saturating_sub(s.today_calls), false, false) + (20_000u64, s.total_relay_calls, 20_000u64.saturating_sub(s.total_relay_calls), false, false) }; let pct = (quota_used as f64 / quota_cap as f64) * 100.0; diff --git a/src/domain_fronter.rs b/src/domain_fronter.rs index a81b2f01..9635c5a1 100644 --- a/src/domain_fronter.rs +++ b/src/domain_fronter.rs @@ -365,6 +365,11 @@ pub struct DomainFronter { /// strike state is per-deployment health bookkeeping, not the /// permanent ban list. script_timeouts: Arc>>, + /// Every call to `relay()` increments this — exit node AND Apps Script. + /// Use this for UI "fetches today" display. Distinct from `relay_calls` + /// (Apps-Script-direct only) and from the quota tracker's `requests_used` + /// (also Apps-Script-only). + total_relay_calls: AtomicU64, relay_calls: AtomicU64, relay_failures: AtomicU64, bytes_relayed: AtomicU64, @@ -633,6 +638,7 @@ impl DomainFronter { coalesced: AtomicU64::new(0), blacklist: Arc::new(std::sync::Mutex::new(HashMap::new())), script_timeouts: Arc::new(std::sync::Mutex::new(HashMap::new())), + total_relay_calls: AtomicU64::new(0), relay_calls: AtomicU64::new(0), relay_failures: AtomicU64::new(0), bytes_relayed: AtomicU64::new(0), @@ -775,6 +781,7 @@ impl DomainFronter { guard.clone() }; StatsSnapshot { + total_relay_calls: self.total_relay_calls.load(Ordering::Relaxed), relay_calls: self.relay_calls.load(Ordering::Relaxed), relay_failures: self.relay_failures.load(Ordering::Relaxed), coalesced: self.coalesced.load(Ordering::Relaxed), @@ -1784,6 +1791,23 @@ impl DomainFronter { headers: &[(String, String)], body: &[u8], ) -> Vec { + self.total_relay_calls.fetch_add(1, Ordering::Relaxed); + + // Block ALL relay paths (exit node + Apps Script) when every account + // bucket is quota-exhausted. Checked here so the exit node short-circuit + // below can't bypass the global hard stop. + if self.quota_tracker.is_globally_hard_stopped() { + self.relay_failures.fetch_add(1, Ordering::Relaxed); + tracing::error!( + "[quota] global hard stop active — all Apps Script account buckets exhausted" + ); + return error_response( + 502, + "All Apps Script accounts quota exhausted; hard stop active. \ + Quota resets on a rolling 24-hour window per account.", + ); + } + // Optional URL rewrite for X/Twitter GraphQL (issue #16). Applied // here, at the top of relay(), so it affects BOTH the cache key // (so matching requests collapse into one entry) AND the URL that @@ -4907,6 +4931,9 @@ fn decode_js_string_escapes(s: &str) -> Option { #[derive(Debug, Clone)] pub struct StatsSnapshot { + /// Total calls to `relay()` — all traffic through this fronter including + /// exit node and Apps Script. Use for "fetches today" display. + pub total_relay_calls: u64, pub relay_calls: u64, pub relay_failures: u64, pub coalesced: u64, diff --git a/src/main.rs b/src/main.rs index bebd45d6..72e5aefb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -357,11 +357,6 @@ async fn main() -> ExitCode { } }; - // Log quota state on startup so the user knows where their daily budget stands. - if let Some(summary) = server.quota_startup_summary() { - tracing::info!("{}", summary); - } - let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); let run = server.run(shutdown_rx); diff --git a/src/proxy_server.rs b/src/proxy_server.rs index a1658ef2..a8617d58 100644 --- a/src/proxy_server.rs +++ b/src/proxy_server.rs @@ -564,6 +564,9 @@ impl ProxyServer { "Listening SOCKS5 on {} — xray / Telegram / app-level SOCKS5 clients use this.", socks_addr ); + if let Some(summary) = self.quota_startup_summary() { + tracing::info!("{}", summary); + } // Pre-warm the outbound connection pool so the user's first request // doesn't pay a fresh TLS handshake to Google edge. Best-effort; // failures are logged and ignored. Skipped in `direct` mode — @@ -616,22 +619,25 @@ impl ProxyServer { let stats_task = if let Some(stats_fronter) = self.fronter.clone() { tokio::spawn(async move { - let mut ticker = tokio::time::interval(std::time::Duration::from_secs(60)); + let mut ticker = tokio::time::interval(std::time::Duration::from_secs(15)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - ticker.tick().await; + let mut was_hard_stopped = false; loop { ticker.tick().await; let s = stats_fronter.snapshot_stats(); if s.relay_calls > 0 || s.cache_hits > 0 { tracing::info!("{}", s.fmt_line()); } - // Log quota warnings and flush persisted state. let q = &s.quota; if q.global_hard_stop { tracing::error!( "[quota] GLOBAL HARD STOP — all {} account(s) exhausted", q.account_count ); + // Log per-account reasons only on the first tick after transition. + if !was_hard_stopped { + stats_fronter.quota_tracker().log_exhaustion_details(); + } } else if q.exhausted_count > 0 { tracing::warn!( "[quota] {}/{} account(s) exhausted used={}/{} remaining={}", @@ -642,13 +648,29 @@ impl ProxyServer { q.requests_remaining_total, ); } + was_hard_stopped = q.global_hard_stop; // Roll any expired 24-hour windows so idle accounts come // back online even without inbound traffic. stats_fronter.quota_tracker().roll_expired_windows(); - // Always flush so the file is up-to-date even when idle. - // save_if_needed() skips the write when dirty_count == 0, - // which means zero-traffic sessions never update the file. - stats_fronter.quota_tracker().save(); + // Safety-net flush for idle sessions (1s save task handles + // active-traffic saves; this covers the zero-traffic case). + stats_fronter.quota_tracker().save_if_needed(); + } + }) + } else { + tokio::spawn(async move { std::future::pending::<()>().await }) + }; + + // Flush quota state to disk every second so the JSON file stays within + // ~1s of in-memory state. The Mutex-backed in-memory state is always + // real-time; this just keeps the on-disk snapshot current. + let save_task = if let Some(save_fronter) = self.fronter.clone() { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(std::time::Duration::from_secs(1)); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + ticker.tick().await; + save_fronter.quota_tracker().save_if_needed(); } }) } else { @@ -744,6 +766,7 @@ impl ProxyServer { biased; _ = &mut shutdown_rx => { tracing::info!("Shutdown signal received, stopping listeners"); + save_task.abort(); stats_task.abort(); keepalive_task.abort(); refill_task.abort(); diff --git a/src/quota_tracker.rs b/src/quota_tracker.rs index 24acbf22..0f071191 100644 --- a/src/quota_tracker.rs +++ b/src/quota_tracker.rs @@ -487,6 +487,19 @@ impl QuotaTracker { } /// Build a human-readable startup summary line. + /// Log the masked ID and exhaustion reason for every hard-stopped bucket. + /// Called once when global hard stop transitions from false to true. + pub fn log_exhaustion_details(&self) { + let st = self.state.lock().unwrap(); + for sid in &self.script_ids { + let Some(b) = st.buckets.get(sid) else { continue }; + if b.hard_stopped { + let reason = b.exhaustion_reason.as_deref().unwrap_or("no reason recorded"); + tracing::warn!("[quota] {} exhausted: {}", b.masked_id, reason); + } + } + } + pub fn startup_summary(&self) -> String { let s = self.summary(); let now = now_unix(); @@ -494,14 +507,28 @@ impl QuotaTracker { let secs = r.saturating_sub(now); format!(" next_reset=in {}h {}m", secs / 3600, (secs / 60) % 60) }).unwrap_or_default(); + let stop_suffix = if s.global_hard_stop { + format!(" exhausted={}/{} HARD-STOP", s.exhausted_count, s.account_count) + } else if s.exhausted_count > 0 { + format!(" exhausted={}/{}", s.exhausted_count, s.account_count) + } else { + String::new() + }; format!( - "[quota] {} account(s) capacity={}/day used={} remaining={}{}", + "[quota] {} account(s) capacity={}/day used={} remaining={}{}{}", s.account_count, s.daily_capacity_total, s.requests_used_total, s.requests_remaining_total, reset_str, + stop_suffix, ) } } + +impl Drop for QuotaTracker { + fn drop(&mut self) { + self.save(); + } +} From 957208ab5322d6340bf7bd903ece283ae762fdec Mon Sep 17 00:00:00 2001 From: Captain Mirage <87281406+CaptainMirage@users.noreply.github.com> Date: Sun, 24 May 2026 12:00:42 +0330 Subject: [PATCH 7/8] fix: persist relay call count, exit node bytes tracking, UI data display cleanup --- src/bin/ui.rs | 36 +++++++++++------------------------- src/domain_fronter.rs | 21 ++++++++++----------- src/quota_tracker.rs | 33 ++++++++++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 39 deletions(-) diff --git a/src/bin/ui.rs b/src/bin/ui.rs index 44900346..343d78c5 100644 --- a/src/bin/ui.rs +++ b/src/bin/ui.rs @@ -1278,7 +1278,7 @@ impl eframe::App for App { .num_columns(4) .spacing([16.0, 4.0]) .show(ui, |ui| { - // Row 1: fetches today | bytes relayed + // Row 1: fetches today | resets in ui.add_sized( [110.0, 18.0], egui::Label::new( @@ -1290,22 +1290,6 @@ impl eframe::App for App { [140.0, 18.0], egui::Label::new(calls_text), ); - ui.add_sized( - [110.0, 18.0], - egui::Label::new( - egui::RichText::new("bytes relayed") - .color(egui::Color32::from_gray(150)), - ), - ); - ui.add_sized( - [140.0, 18.0], - egui::Label::new( - egui::RichText::new(fmt_bytes(s.bytes_relayed)).monospace(), - ), - ); - ui.end_row(); - - // Row 2: next reset (remaining removed — already shown in fetches today X/Y) ui.add_sized( [110.0, 18.0], egui::Label::new( @@ -1319,8 +1303,6 @@ impl eframe::App for App { egui::RichText::new(&reset_str).monospace(), ), ); - ui.label(""); - ui.label(""); ui.end_row(); // Row 3: relay calls+failures | cache @@ -1408,19 +1390,23 @@ impl eframe::App for App { } ui.end_row(); - // Row 5: data transferred with estimated daily total + // Row 5: data transferred with stable estimated daily total. + // Uses bytes_relayed (all relay paths: exit node + Apps Script). + // Estimate clamps avg bytes/call to 50 KB–500 KB so early sparse + // samples don't make the projection swing wildly. if let Some(q) = "a_state { - if q.bytes_total > 0 { - let data_str = if q.requests_used_total > 0 { - let avg = q.bytes_total as f64 / q.requests_used_total as f64; + if s.bytes_relayed > 0 { + let data_str = if s.total_relay_calls >= 5 { + let raw_avg = s.bytes_relayed as f64 / s.total_relay_calls as f64; + let avg = raw_avg.clamp(50_000.0, 500_000.0); let est = (avg * q.daily_capacity_total as f64) as u64; format!( "{} / {} (est.)", - fmt_bytes_approx(q.bytes_total), + fmt_bytes_approx(s.bytes_relayed), fmt_bytes_approx(est), ) } else { - fmt_bytes_approx(q.bytes_total) + fmt_bytes_approx(s.bytes_relayed) }; ui.add_sized( [110.0, 18.0], diff --git a/src/domain_fronter.rs b/src/domain_fronter.rs index 9635c5a1..0b3a57ec 100644 --- a/src/domain_fronter.rs +++ b/src/domain_fronter.rs @@ -365,11 +365,6 @@ pub struct DomainFronter { /// strike state is per-deployment health bookkeeping, not the /// permanent ban list. script_timeouts: Arc>>, - /// Every call to `relay()` increments this — exit node AND Apps Script. - /// Use this for UI "fetches today" display. Distinct from `relay_calls` - /// (Apps-Script-direct only) and from the quota tracker's `requests_used` - /// (also Apps-Script-only). - total_relay_calls: AtomicU64, relay_calls: AtomicU64, relay_failures: AtomicU64, bytes_relayed: AtomicU64, @@ -638,7 +633,6 @@ impl DomainFronter { coalesced: AtomicU64::new(0), blacklist: Arc::new(std::sync::Mutex::new(HashMap::new())), script_timeouts: Arc::new(std::sync::Mutex::new(HashMap::new())), - total_relay_calls: AtomicU64::new(0), relay_calls: AtomicU64::new(0), relay_failures: AtomicU64::new(0), bytes_relayed: AtomicU64::new(0), @@ -780,8 +774,9 @@ impl DomainFronter { } guard.clone() }; + let quota = self.quota_tracker.summary(); StatsSnapshot { - total_relay_calls: self.total_relay_calls.load(Ordering::Relaxed), + total_relay_calls: quota.total_relay_calls, relay_calls: self.relay_calls.load(Ordering::Relaxed), relay_failures: self.relay_failures.load(Ordering::Relaxed), coalesced: self.coalesced.load(Ordering::Relaxed), @@ -798,7 +793,7 @@ impl DomainFronter { h2_calls: self.h2_calls.load(Ordering::Relaxed), h2_fallbacks: self.h2_fallbacks.load(Ordering::Relaxed), h2_disabled: self.h2_disabled.load(Ordering::Relaxed), - quota: self.quota_tracker.summary(), + quota, } } @@ -1791,7 +1786,7 @@ impl DomainFronter { headers: &[(String, String)], body: &[u8], ) -> Vec { - self.total_relay_calls.fetch_add(1, Ordering::Relaxed); + self.quota_tracker.record_relay(); // Block ALL relay paths (exit node + Apps Script) when every account // bucket is quota-exhausted. Checked here so the exit node short-circuit @@ -1842,6 +1837,10 @@ impl DomainFronter { bytes.len() as u64, t0.elapsed().as_nanos() as u64, ); + self.bytes_relayed.fetch_add( + (body.len() + bytes.len()) as u64, + Ordering::Relaxed, + ); return bytes; } Err(e) if !e.is_retryable() => { @@ -4931,8 +4930,8 @@ fn decode_js_string_escapes(s: &str) -> Option { #[derive(Debug, Clone)] pub struct StatsSnapshot { - /// Total calls to `relay()` — all traffic through this fronter including - /// exit node and Apps Script. Use for "fetches today" display. + /// Total relay() calls today (exit node + Apps Script). Sourced from the + /// persisted quota tracker so this survives proxy restarts. pub total_relay_calls: u64, pub relay_calls: u64, pub relay_failures: u64, diff --git a/src/quota_tracker.rs b/src/quota_tracker.rs index 0f071191..f36177bc 100644 --- a/src/quota_tracker.rs +++ b/src/quota_tracker.rs @@ -92,6 +92,9 @@ pub struct QuotaSummary { /// hard-stopped ones. Used by the UI to show a meaningful reset time even /// when all accounts are exhausted. pub next_reset_at_any: Option, + /// Total relay() calls today (all paths). Persisted across restarts. + /// Resets at UTC midnight. + pub total_relay_calls: u64, } // ── Disk state wrapper ──────────────────────────────────────────────────────── @@ -99,6 +102,14 @@ pub struct QuotaSummary { #[derive(Serialize, Deserialize, Default)] struct QuotaState { buckets: HashMap, + /// Total relay() calls today across all paths (exit node + Apps Script). + /// Persisted so restarts don't reset the "fetches today" counter. + #[serde(default)] + total_relay_calls: u64, + /// UTC day number (unix_secs / 86400) when total_relay_calls was last reset. + /// When the day changes, total_relay_calls is zeroed on the next record_relay(). + #[serde(default)] + relay_today_day: u64, } // ── Tracker ─────────────────────────────────────────────────────────────────── @@ -377,6 +388,7 @@ impl QuotaTracker { let mut hard_stopped = 0usize; let mut next_reset: Option = None; let mut next_reset_any: Option = None; + let total_relay_calls = st.total_relay_calls; for sid in &self.script_ids { let Some(b) = st.buckets.get(sid) else { continue }; @@ -422,6 +434,7 @@ impl QuotaTracker { global_hard_stop: global_stop, next_reset_at: next_reset, next_reset_at_any: next_reset_any, + total_relay_calls, } } @@ -443,7 +456,7 @@ impl QuotaTracker { } /// Save if any mutations have occurred since the last flush. - /// Called from the periodic 60-second stats task. + /// Called from the 1-second save task and periodic stats task. pub fn save_if_needed(&self) { if self.dirty_count.load(Ordering::Relaxed) > 0 { self.save(); @@ -453,7 +466,7 @@ impl QuotaTracker { } /// Roll any expired 24-hour windows for all tracked buckets. - /// Called from the 60-second stats task so windows reset even when the + /// Called from the periodic stats task so windows reset even when the /// proxy is idle and no new requests arrive to trigger record_attempt. pub fn roll_expired_windows(&self) { let now = now_unix(); @@ -486,7 +499,20 @@ impl QuotaTracker { } } - /// Build a human-readable startup summary line. + /// Record one call to relay() — all paths (exit node + Apps Script). + /// Persisted so "fetches today" survives proxy restarts. Resets at UTC midnight. + pub fn record_relay(&self) { + let today = now_unix() / 86_400; + let mut st = self.state.lock().unwrap(); + if st.relay_today_day != today { + st.relay_today_day = today; + st.total_relay_calls = 0; + } + st.total_relay_calls += 1; + drop(st); + self.dirty_count.fetch_add(1, Ordering::Relaxed); + } + /// Log the masked ID and exhaustion reason for every hard-stopped bucket. /// Called once when global hard stop transitions from false to true. pub fn log_exhaustion_details(&self) { @@ -500,6 +526,7 @@ impl QuotaTracker { } } + /// Build a human-readable startup summary line. pub fn startup_summary(&self) -> String { let s = self.summary(); let now = now_unix(); From f05601619ac550484f70300801fb2b45077b29bf Mon Sep 17 00:00:00 2001 From: Captain Mirage <87281406+CaptainMirage@users.noreply.github.com> Date: Mon, 25 May 2026 21:28:15 +0330 Subject: [PATCH 8/8] fix(quota_tracker): filter global hard stop aggregate to configured IDs, fix 503 response is_globally_hard_stopped() was summing quota_error_count and requests_used over all persisted buckets, including stale ones from script IDs removed from config. A user rotating away exhausted IDs would still carry their usage in quota_state.json, causing the aggregate check to falsely trip a global hard stop against fresh accounts. Fixed by filtering both sums to self.script_ids only. The all_stopped primary check was already correct (iterates script_ids, not bucket values). Also corrects the hard-stop HTTP response from 502 Bad Gateway to 503 Service Unavailable, which is the accurate status for a deliberately refused request due to resource exhaustion. Regression test: one stale exhausted persisted bucket not in the current config plus one fresh configured bucket must not trigger a global hard stop. --- src/domain_fronter.rs | 2 +- src/quota_tracker.rs | 74 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/domain_fronter.rs b/src/domain_fronter.rs index 0b3a57ec..ff6d4cd1 100644 --- a/src/domain_fronter.rs +++ b/src/domain_fronter.rs @@ -1797,7 +1797,7 @@ impl DomainFronter { "[quota] global hard stop active — all Apps Script account buckets exhausted" ); return error_response( - 502, + 503, "All Apps Script accounts quota exhausted; hard stop active. \ Quota resets on a rolling 24-hour window per account.", ); diff --git a/src/quota_tracker.rs b/src/quota_tracker.rs index f36177bc..5614684b 100644 --- a/src/quota_tracker.rs +++ b/src/quota_tracker.rs @@ -359,11 +359,20 @@ impl QuotaTracker { // Secondary check: aggregate remaining < N × safety_buffer // AND at least one quota error signal has been seen (not just network // failures or local disconnects). - let total_quota_errors: u64 = st.buckets.values().map(|b| b.quota_error_count).sum(); + // Only sum over currently configured script_ids so stale buckets from + // removed accounts don't inflate the used count or error tally and + // falsely trip this check. + let total_quota_errors: u64 = self.script_ids.iter() + .filter_map(|sid| st.buckets.get(sid)) + .map(|b| b.quota_error_count) + .sum(); if total_quota_errors == 0 { return false; } - let total_used: u64 = st.buckets.values().map(|b| b.requests_used).sum(); + let total_used: u64 = self.script_ids.iter() + .filter_map(|sid| st.buckets.get(sid)) + .map(|b| b.requests_used) + .sum(); let total_cap = self.daily_limit * self.script_ids.len() as u64; let total_remaining = total_cap.saturating_sub(total_used); let aggregate_reserve = self.safety_buffer * self.script_ids.len() as u64; @@ -559,3 +568,64 @@ impl Drop for QuotaTracker { self.save(); } } + +#[cfg(test)] +impl QuotaTracker { + fn new_for_test( + script_ids: Vec, + daily_limit: u64, + safety_buffer: u64, + state: QuotaState, + ) -> Self { + Self { + state: Mutex::new(state), + script_ids, + daily_limit, + safety_buffer, + dirty_count: AtomicU64::new(0), + state_path: std::path::PathBuf::from("/dev/null"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A stale exhausted bucket from a removed script ID must not cause a + /// global hard stop when the currently configured ID is fresh and healthy. + #[test] + fn stale_exhausted_bucket_does_not_trigger_global_hard_stop() { + let stale_id = "stale_removed_aaaa1111bbbb2222cccc".to_string(); + let active_id = "active_fresh_xxxx9999yyyy8888zzzz".to_string(); + + let mut state = QuotaState::default(); + state.buckets.insert(stale_id.clone(), AccountBucket { + masked_id: mask_id(&stale_id), + requests_used: 19_500, + quota_error_count: 5, + exhausted: true, + hard_stopped: true, + exhaustion_reason: Some("quota exhausted".into()), + ..Default::default() + }); + state.buckets.insert(active_id.clone(), AccountBucket { + masked_id: mask_id(&active_id), + requests_used: 100, + ..Default::default() + }); + + // Only active_id is in the live config — stale_id was removed. + let tracker = QuotaTracker::new_for_test( + vec![active_id], + 20_000, + 500, + state, + ); + + assert!( + !tracker.is_globally_hard_stopped(), + "stale exhausted bucket from a removed script_id should not trigger global hard stop" + ); + } +}