Persist GUI history and robustify training

toptek/config/app.yml

@@ -1,4 +1,5 @@
 polling_interval_seconds: 5
 cache_directory: data/cache
 models_directory: models
+user_data_directory: data/user
 log_level: INFO

toptek/core/model.py

@@ -10,9 +10,12 @@
 import numpy as np
 from sklearn.calibration import CalibratedClassifierCV
 from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.impute import SimpleImputer
 from sklearn.linear_model import LogisticRegression
 from sklearn.metrics import accuracy_score, roc_auc_score
 from sklearn.model_selection import train_test_split
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
 
 
 @dataclass
@@ -34,12 +37,27 @@ def train_classifier(
 ) -> TrainResult:
     """Train a basic classifier and persist it to ``models_dir``."""
 
+    X = np.asarray(X, dtype=float)
+    X = np.where(np.isfinite(X), X, np.nan)
+    y = np.asarray(y)
+
     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, shuffle=True, random_state=42)
 
     if model_type == "logistic":
-        model = LogisticRegression(max_iter=1000)
+        model = Pipeline(
+            [
+                ("imputer", SimpleImputer(strategy="median")),
+                ("scaler", StandardScaler()),
+                ("clf", LogisticRegression(max_iter=1000)),
+            ]
+        )
     elif model_type == "gbm":
-        model = GradientBoostingClassifier()
+        model = Pipeline(
+            [
+                ("imputer", SimpleImputer(strategy="median")),
+                ("clf", GradientBoostingClassifier()),
+            ]
+        )
     else:
         raise ValueError("Unknown model type")
 
@@ -92,7 +110,7 @@ def calibrate_classifier(
 
     X_cal, y_cal = calibration_data
     pipeline = load_model(model_path)
-    calibrator = CalibratedClassifierCV(base_estimator=pipeline, method=method, cv="prefit")
+    calibrator = CalibratedClassifierCV(estimator=pipeline, method=method, cv="prefit")
     calibrator.fit(X_cal, y_cal)
     target_path = output_path or model_path.with_name(f"{model_path.stem}_calibrated.pkl")
     with target_path.open("wb") as handle:

toptek/core/storage.py

@@ -0,0 +1,133 @@
+"""Persistent user storage for GUI state and activity history.
+
+This module provides a light-weight persistence layer for the GUI so that
+session preferences, trained model summaries, and activity history survive
+across program restarts. Data is written to JSON on disk and exposed via a
+simple publish/subscribe API for real-time UI updates.
+
+Example:
+    >>> from pathlib import Path
+    >>> from core.storage import UserStorage
+    >>> store = UserStorage(Path("data/user/state.json"))
+    >>> store.update_section("preferences", {"theme": "dark"})
+    >>> store.append_history("login", "Signed in", {"username": "demo"})
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from threading import RLock
+from typing import Any, Callable, Dict, List, MutableMapping
+
+
+@dataclass
+class UserEvent:
+    """Represents a persisted user event."""
+
+    timestamp: str
+    event_type: str
+    message: str
+    payload: Dict[str, Any]
+
+
+class UserStorage:
+    """Manage durable user state and publish updates to listeners."""
+
+    def __init__(self, state_path: Path, *, history_limit: int = 200) -> None:
+        self.state_path = state_path
+        self.history_limit = history_limit
+        self.state_path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = RLock()
+        self._listeners: List[Callable[[Dict[str, Any]], None]] = []
+        self._state: Dict[str, Any] = {"sections": {}, "history": []}
+        self._last_loaded = 0.0
+        self._load()
+
+    def subscribe(self, callback: Callable[[Dict[str, Any]], None]) -> None:
+        """Register *callback* to receive state snapshots when changes occur."""
+
+        with self._lock:
+            self._listeners.append(callback)
+        callback(self.snapshot())
+
+    def update_section(self, name: str, data: MutableMapping[str, Any]) -> None:
+        """Persist a structured section under *name*."""
+
+        with self._lock:
+            self._state.setdefault("sections", {})[name] = dict(data)
+            self._persist_locked()
+            self._broadcast()
+
+    def append_history(self, event_type: str, message: str, payload: Dict[str, Any] | None = None) -> None:
+        """Append a history event with metadata."""
+
+        event = UserEvent(
+            timestamp=datetime.now(tz=timezone.utc).isoformat(),
+            event_type=event_type,
+            message=message,
+            payload=payload or {},
+        )
+        with self._lock:
+            history: List[Dict[str, Any]] = self._state.setdefault("history", [])
+            history.append(event.__dict__)
+            if len(history) > self.history_limit:
+                del history[: len(history) - self.history_limit]
+            self._persist_locked()
+            self._broadcast()
+
+    def snapshot(self) -> Dict[str, Any]:
+        """Return a shallow copy of the storage state."""
+
+        with self._lock:
+            return json.loads(json.dumps(self._state))
+
+    def get_section(self, name: str, default: Dict[str, Any] | None = None) -> Dict[str, Any] | None:
+        """Retrieve a stored section."""
+
+        with self._lock:
+            sections = self._state.get("sections", {})
+            if name not in sections:
+                return default
+            return json.loads(json.dumps(sections[name]))
+
+    def reload_if_changed(self) -> bool:
+        """Reload state from disk when the backing file changes."""
+
+        try:
+            mtime = self.state_path.stat().st_mtime
+        except FileNotFoundError:
+            return False
+        if mtime <= self._last_loaded:
+            return False
+        self._load()
+        self._broadcast()
+        return True
+
+    # Internal helpers -------------------------------------------------
+
+    def _broadcast(self) -> None:
+        snapshot = self.snapshot()
+        for listener in list(self._listeners):
+            listener(snapshot)
+
+    def _load(self) -> None:
+        if self.state_path.exists():
+            with self.state_path.open("r", encoding="utf-8") as handle:
+                try:
+                    self._state = json.load(handle)
+                except json.JSONDecodeError:
+                    self._state = {"sections": {}, "history": []}
+            self._last_loaded = self.state_path.stat().st_mtime
+        else:
+            self._persist_locked()
+
+    def _persist_locked(self) -> None:
+        with self.state_path.open("w", encoding="utf-8") as handle:
+            json.dump(self._state, handle, indent=2)
+        self._last_loaded = self.state_path.stat().st_mtime
+
+
+__all__ = ["UserStorage", "UserEvent"]

toptek/core/utils.py

@@ -34,11 +34,13 @@ class AppPaths:
         root: Base directory for the project.
         cache: Directory path for cached data files.
         models: Directory path for persisted models.
+        user_data: Directory path for long-lived user state and history.
     """
 
     root: Path
     cache: Path
     models: Path
+    user_data: Path
 
 
 def build_logger(name: str, level: str = "INFO") -> logging.Logger:
@@ -84,7 +86,7 @@ def load_yaml(path: Path) -> Dict[str, Any]:
 def ensure_directories(paths: AppPaths) -> None:
     """Ensure application directories exist."""
 
-    for directory in (paths.cache, paths.models):
+    for directory in (paths.cache, paths.models, paths.user_data):
         directory.mkdir(parents=True, exist_ok=True)
 
 
@@ -119,4 +121,5 @@ def build_paths(root: Path, app_config: Dict[str, Any]) -> AppPaths:
 
     cache_dir = root / app_config.get("cache_directory", "data/cache")
     models_dir = root / app_config.get("models_directory", "models")
-    return AppPaths(root=root, cache=cache_dir, models=models_dir)
+    user_dir = root / app_config.get("user_data_directory", "data/user")
+    return AppPaths(root=root, cache=cache_dir, models=models_dir, user_data=user_dir)

toptek/gui/app.py

@@ -4,43 +4,159 @@
 
 import tkinter as tk
 from tkinter import ttk
-from typing import Dict
+from typing import Dict, List
 
-from core import utils
+from core import storage as storage_mod, utils
 
 
 class ToptekApp(ttk.Notebook):
     """Main application notebook containing all tabs."""
 
-    def __init__(self, master: tk.Tk, configs: Dict[str, Dict[str, object]], paths: utils.AppPaths) -> None:
+    def __init__(
+        self,
+        master: tk.Misc,
+        configs: Dict[str, Dict[str, object]],
+        paths: utils.AppPaths,
+        storage: storage_mod.UserStorage,
+        *,
+        on_tab_change: callable | None = None,
+    ) -> None:
         super().__init__(master)
         self.configs = configs
         self.paths = paths
+        self.storage = storage
+        self._on_tab_change = on_tab_change
+        self._tab_names: List[str] = []
+        self._tab_guidance: Dict[str, str] = {}
         self._build_tabs()
+        self.bind("<<NotebookTabChanged>>", self._handle_tab_change)
 
     def _build_tabs(self) -> None:
         from . import widgets
 
         tabs = {
-            "Login": widgets.LoginTab,
-            "Research": widgets.ResearchTab,
-            "Train": widgets.TrainTab,
-            "Backtest": widgets.BacktestTab,
-            "Trade": widgets.TradeTab,
+            "Login": (
+                widgets.LoginTab,
+                "Step 1 · Secure your API keys and verify environment access.",
+            ),
+            "Research": (
+                widgets.ResearchTab,
+                "Step 2 · Explore market structure and feature snapshots.",
+            ),
+            "Train": (
+                widgets.TrainTab,
+                "Step 3 · Fit and calibrate models before risking capital.",
+            ),
+            "Backtest": (
+                widgets.BacktestTab,
+                "Step 4 · Validate expectancy and drawdown resilience.",
+            ),
+            "Trade": (
+                widgets.TradeTab,
+                "Step 5 · Check Topstep guardrails and plan manual execution.",
+            ),
         }
-        for name, cls in tabs.items():
-            frame = cls(self, self.configs, self.paths)
+        for name, (cls, guidance) in tabs.items():
+            frame = cls(self, self.configs, self.paths, self.storage)
             self.add(frame, text=name)
+            self._tab_names.append(name)
+            self._tab_guidance[name] = guidance
+
+    def initialise_guidance(self) -> None:
+        """Invoke the guidance callback for the default tab."""
+
+        if not self._tab_names:
+            return
+        self._dispatch_tab_change(0)
+
+    def _handle_tab_change(self, _: tk.Event) -> None:
+        index = self.index("current")
+        self._dispatch_tab_change(index)
+
+    def _dispatch_tab_change(self, index: int) -> None:
+        if self._on_tab_change is None:
+            return
+        name = self._tab_names[index]
+        guidance = self._tab_guidance.get(name, "")
+        self._on_tab_change(index, name, guidance)
 
 
 def launch_app(*, configs: Dict[str, Dict[str, object]], paths: utils.AppPaths) -> None:
     """Initialise and start the Tkinter main loop."""
 
     root = tk.Tk()
-    root.title("Toptek Starter")
-    root.geometry("900x600")
-    notebook = ToptekApp(root, configs, paths)
+    root.title("Toptek Mission Control")
+    root.geometry("1024x680")
+
+    storage = storage_mod.UserStorage(paths.user_data / "user_state.json")
+
+    style = ttk.Style()
+    try:
+        style.theme_use("clam")
+    except tk.TclError:
+        # ``clam`` is widely available, but gracefully fallback if missing.
+        pass
+    style.configure("Header.TLabel", font=("Segoe UI", 20, "bold"))
+    style.configure("SubHeader.TLabel", font=("Segoe UI", 12))
+    style.configure("Step.TLabel", font=("Segoe UI", 11))
+    style.configure("Guidance.TLabelframe", padding=(12, 10))
+    style.configure("Guidance.TLabelframe.Label", font=("Segoe UI", 11, "bold"))
+    style.configure("TNotebook.Tab", padding=(14, 8))
+
+    container = ttk.Frame(root, padding=16)
+    container.pack(fill=tk.BOTH, expand=True)
+
+    header = ttk.Frame(container)
+    header.pack(fill=tk.X, pady=(0, 12))
+    ttk.Label(header, text="Project X · Manual Trading Copilot", style="Header.TLabel").pack(anchor=tk.W)
+    ttk.Label(
+        header,
+        text="Follow the guided workflow from credentials to Topstep-compliant trade plans.",
+        style="SubHeader.TLabel",
+    ).pack(anchor=tk.W)
+
+    guidance_card = ttk.Labelframe(container, text="Mission Checklist", style="Guidance.TLabelframe")
+    guidance_card.pack(fill=tk.X, pady=(0, 12))
+
+    step_label = ttk.Label(guidance_card, style="Step.TLabel")
+    step_label.pack(anchor=tk.W)
+    progress = ttk.Progressbar(guidance_card, maximum=4, mode="determinate", length=220)
+    progress.pack(anchor=tk.W, pady=(8, 0))
+
+    def handle_tab_change(index: int, name: str, guidance: str) -> None:
+        step_label.config(text=f"{guidance}\n→ Current focus: {name} tab")
+        progress["value"] = index
+
+    history_card = ttk.Labelframe(container, text="Activity timeline", padding=12)
+    history_card.pack(fill=tk.BOTH, expand=False, pady=(0, 12))
+    history_tree = ttk.Treeview(history_card, columns=("time", "event", "detail"), show="headings", height=6)
+    history_tree.heading("time", text="Timestamp (UTC)")
+    history_tree.heading("event", text="Event")
+    history_tree.heading("detail", text="Message")
+    history_tree.column("time", width=190, anchor=tk.W)
+    history_tree.column("event", width=120, anchor=tk.W)
+    history_tree.column("detail", width=520, anchor=tk.W)
+    history_tree.pack(fill=tk.BOTH, expand=True)
+
+    def sync_history(state: Dict[str, object]) -> None:
+        history = state.get("history", []) if isinstance(state, dict) else []
+        history_tree.delete(*history_tree.get_children())
+        for item in history[-12:]:
+            history_tree.insert("", tk.END, values=(item.get("timestamp", ""), item.get("event_type", ""), item.get("message", "")))
+
+    storage.subscribe(sync_history)
+
+    def poll_storage() -> None:
+        storage.reload_if_changed()
+        interval_ms = int(configs.get("app", {}).get("polling_interval_seconds", 5) * 1000)
+        root.after(max(1000, interval_ms), poll_storage)
+
+    poll_storage()
+
+    notebook = ToptekApp(container, configs, paths, storage, on_tab_change=handle_tab_change)
     notebook.pack(fill=tk.BOTH, expand=True)
+    notebook.initialise_guidance()
+
     root.mainloop()