Replace curl-download with paginated inline results + widget JSON endpoint

everyrow-mcp/deploy/docker-compose.local.yaml

@@ -7,5 +7,5 @@ services:
 
   mcp-server:
     environment:
-      MCP_SERVER_URL: "http://localhost:8000"
-      TRUST_PROXY_HEADERS: "false"
+      MCP_SERVER_URL: "${MCP_SERVER_URL:-http://localhost:8000}"
+      TRUST_PROXY_HEADERS: "${TRUST_PROXY_HEADERS:-false}"

everyrow-mcp/src/everyrow_mcp/app.py

@@ -131,7 +131,13 @@ async def no_auth_http_lifespan(_server: FastMCP):
 - Do NOT pass local file paths to `everyrow_upload_data` — it will fail in remote mode.
 
 ## Results
-- `everyrow_results(task_id)` returns a paginated preview with a download link.
+- IMPORTANT: When a task completes, you MUST ask the user how many rows they want loaded into \
+your context BEFORE calling everyrow_results. Do NOT call everyrow_results without asking first.
+- `everyrow_results(task_id, page_size=N)` loads N rows into your context so you can read them. \
+The user always has access to all rows via the widget and download link.
+- After retrieving results, tell the user how many rows you can see vs the total, and that \
+they have access to the full dataset via the widget above and the download link.
+- Use offset to paginate through larger datasets.
 """
 )
 

everyrow-mcp/src/everyrow_mcp/models.py

@@ -654,8 +654,14 @@ def validate_task_id(cls, v: str) -> str:
         ge=0,
     )
     page_size: int = Field(
-        default=1000,
-        description="Number of rows per page. Default 1000. Max 10000.",
+        default=50,
+        description=(
+            "Number of result rows to load into your context so you can read them. "
+            "The user has access to all rows via the widget regardless of this value. "
+            "REQUIRED: You must ask the user how many rows they want before calling this tool. "
+            "Do not use the default without asking. "
+            "Use offset to paginate through larger datasets."
+        ),
         ge=1,
         le=10000,
     )

everyrow-mcp/src/everyrow_mcp/result_store.py

@@ -105,40 +105,55 @@ def _build_result_response(
     *requested_page_size*, when provided, is the user's original page_size
     and is used in the "next page" hint so the server can re-clamp
     independently on each call.
+
+    The widget fetches full results on demand by minting a fresh download
+    token via the ``download-token`` endpoint — no pre-minted URL is baked
+    into the response, avoiding stale-token issues on re-render.
     """
     col_names = _format_columns(columns)
     hint_page_size = (
         requested_page_size if requested_page_size is not None else page_size
     )
 
-    widget_data: dict[str, Any] = {
-        "csv_url": csv_url,
-        "preview": preview_records,
-        "total": total,
-    }
-    if session_url:
-        widget_data["session_url"] = session_url
-    if poll_token:
-        widget_data["poll_token"] = poll_token
-        widget_data["download_token_url"] = (
-            f"{mcp_server_url}/api/results/{task_id}/download-token"
-        )
-    widget_json = json.dumps(widget_data)
-
     has_more = offset + page_size < total
     next_offset = offset + page_size if has_more else None
 
+    # Only emit widget JSON on the first page — the widget already fetches
+    # the full dataset independently, so subsequent pages only need the
+    # text summary for the LLM.
+    # Alternative: track a per-task call counter in Redis and only emit on
+    # the first call. Rejected because it adds state, and re-fetching
+    # offset=0 (e.g. "show me the results again") should show the widget.
+    contents: list[TextContent] = []
+    if offset == 0:
+        widget_data: dict[str, Any] = {
+            "csv_url": csv_url,
+            "preview": preview_records,
+            "total": total,
+            "fetch_full_results": True,
+        }
+        if session_url:
+            widget_data["session_url"] = session_url
+        if poll_token:
+            widget_data["poll_token"] = poll_token
+            widget_data["download_token_url"] = (
+                f"{mcp_server_url}/api/results/{task_id}/download-token"
+            )
+        contents.append(TextContent(type="text", text=json.dumps(widget_data)))
+
     if has_more:
         page_size_arg = f", page_size={hint_page_size}"
         summary = (
             f"Results: {total} rows, {len(columns)} columns ({col_names}). "
             f"Showing rows {offset + 1}-{min(offset + page_size, total)} of {total}.\n"
+            f"IMPORTANT: Tell the user that you can only see {min(page_size, total)} of the {total} rows in your context, "
+            f"but they have access to all {total} rows via the widget above.\n"
             f"Call everyrow_results(task_id='{task_id}', offset={next_offset}{page_size_arg}) for the next page."
         )
         if offset == 0:
             summary += (
                 f"\nFull CSV download: {csv_url}\n"
-                "IMPORTANT: Display this download link to the user as a clickable URL in your response."
+                "Display this download link to the user as a clickable URL in your response."
             )
     elif offset == 0:
         summary = (
@@ -153,10 +168,8 @@ def _build_result_response(
             f"of {total} (final page)."
         )
 
-    return [
-        TextContent(type="text", text=widget_json),
-        TextContent(type="text", text=summary),
-    ]
+    contents.append(TextContent(type="text", text=summary))
+    return contents
 
 
 async def _get_csv_url(
@@ -247,12 +260,8 @@ async def try_store_result(
     page_size: int,
     session_url: str = "",
     mcp_server_url: str = "",
-) -> list[TextContent] | None:
-    """Store a DataFrame in Redis and return a response.
-
-    Returns None if Redis is not available (caller should fall back to
-    inline results).
-    """
+) -> list[TextContent]:
+    """Store a DataFrame in Redis and return a paginated response."""
     try:
         # Store full CSV in Redis
         await redis_store.store_result_csv(task_id, df.to_csv(index=False))
@@ -279,10 +288,9 @@ async def try_store_result(
 
         csv_url, poll_token = await _get_csv_url(task_id, mcp_server_url)
         if csv_url is None:
-            logger.warning(
-                "Poll token expired for task %s, cannot build download URL", task_id
+            raise RuntimeError(
+                f"Poll token expired for task {task_id}, cannot build download URL"
             )
-            return None
 
         preview_records, effective_page_size = clamp_page_to_budget(
             preview_records=preview_records,
@@ -302,10 +310,6 @@ async def try_store_result(
             mcp_server_url=mcp_server_url,
             requested_page_size=page_size,
         )
-    except Exception as exc:
-        logger.error(
-            "Failed to store results in Redis for task %s, falling back to inline: %s",
-            task_id,
-            type(exc).__name__,
-        )
-        return None
+    except Exception:
+        logger.exception("Failed to store results in Redis for task %s", task_id)
+        raise

everyrow-mcp/src/everyrow_mcp/routes.py

@@ -2,10 +2,12 @@
 
 from __future__ import annotations
 
+import io
 import logging
 import secrets
 from uuid import UUID
 
+import pandas as pd
 from everyrow.api_utils import handle_response
 from everyrow.generated.api.tasks import get_task_status_tasks_task_id_status_get
 from everyrow.generated.client import AuthenticatedClient
@@ -14,6 +16,7 @@
 
 from everyrow_mcp import redis_store
 from everyrow_mcp.config import settings
+from everyrow_mcp.result_store import _sanitize_records
 from everyrow_mcp.tool_helpers import _UI_EXCLUDE, TaskState
 
 logger = logging.getLogger(__name__)
@@ -243,7 +246,7 @@ async def api_download_token(request: Request) -> Response:
     return JSONResponse({"download_url": download_url}, headers=cors)
 
 
-async def api_download(request: Request) -> Response:
+async def api_download(request: Request) -> Response:  # noqa: PLR0911
     """REST endpoint to download task results as CSV.
 
     Authenticates via a short-lived, single-use download token (not the
@@ -290,6 +293,38 @@ async def api_download(request: Request) -> Response:
             {"error": "Results not found or expired"}, status_code=404, headers=cors
         )
 
+    # Validate format parameter
+    fmt = request.query_params.get("format", "csv")
+    if fmt not in ("csv", "json"):
+        return JSONResponse(
+            {"error": "Unsupported format"}, status_code=400, headers=cors
+        )
+
+    # Return JSON array if requested (used by the widget for full data fetch).
+    if fmt == "json":
+        # Guard against memory exhaustion — JSON conversion holds ~4x the data
+        # in memory (csv string, DataFrame, records list, JSON response body).
+        # Use a conservative 10 MB threshold to keep peak memory manageable.
+        max_json_size = settings.max_upload_size_bytes // 5
+        if len(csv_text) > max_json_size:
+            logger.warning(
+                "CSV too large for JSON conversion (%d chars, limit %d) for task %s",
+                len(csv_text),
+                max_json_size,
+                task_id,
+            )
+            return JSONResponse(
+                {"error": "Result too large for JSON format"},
+                status_code=413,
+                headers=cors,
+            )
+        df = pd.read_csv(io.StringIO(csv_text))
+        records = _sanitize_records(df.to_dict(orient="records"))
+        return JSONResponse(
+            records,
+            headers={**cors, "X-Content-Type-Options": "nosniff"},
+        )
+
     safe_prefix = "".join(c for c in task_id[:8] if c.isalnum() or c == "-")
     return Response(
         content=csv_text,

everyrow-mcp/src/everyrow_mcp/templates.py

@@ -32,7 +32,7 @@
   --input-bg:#2d2d2d;--input-border:#555;--input-focus:#64b5f6;
 }}
 *{box-sizing:border-box}
-body{font-family:system-ui,-apple-system,sans-serif;margin:0;padding:12px;color:var(--text);background:var(--bg);font-size:13px}
+body{font-family:system-ui,-apple-system,sans-serif;margin:0;padding:0;color:var(--text);background:var(--bg);font-size:13px;height:0;overflow:hidden}
 #toolbar{display:flex;align-items:center;gap:8px;padding:8px 4px;margin-bottom:8px;flex-wrap:wrap}
 #toolbar #sum{font-weight:600;font-size:13px;flex:1;min-width:150px;color:var(--text-sec)}
 #toolbar button{padding:5px 12px;border:1px solid var(--border);border-radius:5px;font-size:12px;cursor:pointer;background:var(--btn-bg);color:var(--btn-text);transition:background .15s}
@@ -82,8 +82,9 @@
 .copy-modal-box textarea{width:100%;height:300px;font-family:monospace;font-size:12px;border:1px solid var(--border);border-radius:4px;padding:8px;background:var(--input-bg);color:var(--text);resize:vertical}
 .copy-modal-box .modal-btns{display:flex;gap:8px;justify-content:flex-end}
 .copy-modal-box button{padding:6px 16px;border:1px solid var(--border);border-radius:5px;background:var(--btn-bg);color:var(--btn-text);cursor:pointer;font-size:12px}
-.session-link{margin-bottom:6px;font-size:12px}
+.session-link{margin-bottom:6px;font-size:12px;display:flex;align-items:center;gap:8px}
 .session-link a{font-weight:500}
+.session-link .spacer{flex:1}
 .col-resize-handle{position:absolute;top:0;right:-2px;width:4px;height:100%;cursor:col-resize;z-index:5;user-select:none}
 .col-resize-handle:hover{background:var(--accent);opacity:.3}
 body.col-resizing,body.col-resizing *{cursor:col-resize!important;user-select:none!important}
@@ -93,6 +94,9 @@
 .cell-more:hover,.cell-less:hover{text-decoration:underline}
 .export-btns{display:inline-flex;gap:2px}
 .export-btns button{padding:3px 8px;font-size:11px}
+#globalSearch{padding:4px 8px;border:1px solid var(--input-border);border-radius:5px;font-size:12px;background:var(--input-bg);color:var(--text);outline:none;width:160px;transition:border-color .15s,width .2s}
+#globalSearch:focus{border-color:var(--input-focus);width:220px}
+#globalSearch::placeholder{color:var(--text-dim)}
 .col-ghost{position:fixed;background:var(--bg-toolbar);border:1px solid var(--accent);border-radius:4px;padding:4px 8px;font-size:12px;font-weight:600;opacity:.85;pointer-events:none;z-index:200;white-space:nowrap}
 body.col-dragging,body.col-dragging *{cursor:grabbing!important;user-select:none!important}
 .hdr-row th.drag-over-left{box-shadow:inset 3px 0 0 var(--accent)}
@@ -107,13 +111,13 @@
 .settings-drop input[type="radio"]{margin:0}
 .settings-drop .drop-sep{border-top:1px solid var(--border-light);margin:4px 0}
 </style></head><body>
-<div id="sessionLink" class="session-link"></div>
+<div id="sessionLink" class="session-link"><span class="spacer"></span><input id="globalSearch" type="text" placeholder="Search all columns..."></div>
 <div id="toolbar">
   <span id="sum">Loading...</span>
   <button id="selAllBtn">Select all</button>
-  <button id="copyBtn" disabled>Copy TSV (0)</button>
+  <button id="copyBtn" disabled>Copy CSV (0)</button>
   <span class="export-btns"><button id="exportLink" title="Copy CSV download link to clipboard">Copy link</button></span>
-  <span class="settings-wrap"><button id="settingsBtn" title="Settings">Settings</button><div id="settingsDrop" class="settings-drop"><div class="drop-hdr">Copy format</div><label><input type="radio" name="cfmt" value="tsv" checked> TSV (tabs)</label><label><input type="radio" name="cfmt" value="csv"> CSV</label><label><input type="radio" name="cfmt" value="json"> JSON</label><div class="drop-sep"></div><div class="drop-hdr">Table height</div><label><input type="radio" name="tsize" value="250"> Small</label><label><input type="radio" name="tsize" value="420" checked> Medium</label><label><input type="radio" name="tsize" value="700"> Large</label></div></span>
+  <span class="settings-wrap"><button id="settingsBtn" title="Settings">Settings</button><div id="settingsDrop" class="settings-drop"><div class="drop-hdr">Copy format</div><label><input type="radio" name="cfmt" value="csv" checked> CSV</label><label><input type="radio" name="cfmt" value="tsv"> TSV (tabs)</label><label><input type="radio" name="cfmt" value="json"> JSON</label><div class="drop-sep"></div><div class="drop-hdr">Table height</div><label><input type="radio" name="tsize" value="250"> Small</label><label><input type="radio" name="tsize" value="420" checked> Medium</label><label><input type="radio" name="tsize" value="700"> Large</label></div></span>
   <button id="expandBtn" title="Toggle fullscreen">&#x2922;</button>
 </div>
 <div class="wrap" id="wrap"><table id="tbl"></table></div>
@@ -146,10 +150,11 @@
 let sessionUrl="",csvUrl="",pollToken="",downloadTokenUrl="";
 const TRUNC=200;
 let didDrag=false;
-let copyFmt="tsv";
+let copyFmt="csv";
+let widgetActive=false;
 const settingsBtn=document.getElementById("settingsBtn");
 const settingsDrop=document.getElementById("settingsDrop");
-const S={rows:[],allCols:[],filteredIdx:[],sortCol:null,sortDir:0,filters:{},selected:new Set(),lastClick:null,isFullscreen:false,focusedCell:null};
+const S={rows:[],allCols:[],filteredIdx:[],sortCol:null,sortDir:0,filters:{},globalQuery:"",selected:new Set(),lastClick:null,isFullscreen:false,focusedCell:null};
 
 /* --- theming & display mode --- */
 app.onhostcontextchanged=(ctx)=>{
@@ -198,14 +203,18 @@
   const all=[...colSet];
   const visible=all.filter(k=>!k.startsWith("research."));
   S.allCols=[...visible.filter(k=>!k.includes(".")),...visible.filter(k=>k.includes("."))];
-  S.sortCol=null;S.sortDir=0;S.filters={};S.selected.clear();S.lastClick=null;
+  S.sortCol=null;S.sortDir=0;S.filters={};S.globalQuery="";globalSearchEl.value="";S.selected.clear();S.lastClick=null;
   S.filteredIdx=S.rows.map((_,i)=>i);
   renderTable();
 }
 
 /* --- filter & sort --- */
 function applyFilterAndSort(){
   let idx=S.rows.map((_,i)=>i);
+  if(S.globalQuery){
+    const gq=S.globalQuery.toLowerCase();
+    idx=idx.filter(i=>{const row=S.rows[i].display;return Object.values(row).some(v=>v!=null&&String(v).toLowerCase().includes(gq));});
+  }
   for(const[col,q]of Object.entries(S.filters)){
     if(!q)continue;
     const lq=q.toLowerCase();
@@ -227,6 +236,8 @@
 
 let filterTimer=null;
 function onFilterInput(col,val){S.filters[col]=val;clearTimeout(filterTimer);filterTimer=setTimeout(()=>applyFilterAndSort(),150);}
+const globalSearchEl=document.getElementById("globalSearch");
+globalSearchEl.addEventListener("input",()=>{S.globalQuery=globalSearchEl.value;clearTimeout(filterTimer);filterTimer=setTimeout(()=>applyFilterAndSort(),150);});
 
 /* --- research lookup --- */
 function getResearch(row,col){
@@ -707,11 +718,13 @@
 }
 
 /* --- session URL display --- */
+const linksEl=document.createElement("span");linksEl.id="sessionLinks";
+sessionLinkEl.insertBefore(linksEl,sessionLinkEl.querySelector(".spacer"));
 function updateSessionLink(){
   let h="";
   if(sessionUrl)h+='<a href="#" id="sessionOpenLink">Open everyrow session &#x2197;</a>';
   if(csvUrl){if(h)h+=" &nbsp;|&nbsp; ";h+='<a href="#" id="csvOpenLink">Download CSV &#x2913;</a>';}
-  sessionLinkEl.innerHTML=h;
+  linksEl.innerHTML=h;
   document.getElementById("sessionOpenLink")?.addEventListener("click",e=>{
     e.preventDefault();
     if(!/^https?:\\/\\//i.test(sessionUrl))return;app.openLink({url:sessionUrl}).catch(()=>window.open(sessionUrl,"_blank"));
@@ -725,6 +738,12 @@
 }
 
 /* --- data loading --- */
+async function fetchFullResultsWithFreshToken(hasPreview,total){
+  const base=await getFreshDownloadUrl();
+  if(!base){if(!hasPreview)sum.textContent="Download link expired";return;}
+  const url=base+(base.includes("?")?"&":"?")+"format=json";
+  fetchFullResults(url,{},hasPreview,total);
+}
 function fetchFullResults(url,opts,hasPreview,total){
   if(!hasPreview)sum.textContent="Loading"+(total?" "+total+" rows":"")+"...";
   fetch(url,opts).then(r=>{
@@ -734,24 +753,30 @@
     if(hasPreview){showToast("Full load failed, showing preview");}
     else{
       sum.innerHTML=esc("Failed to load: "+err.message)+' <button id="retryBtn" style="margin-left:8px;padding:2px 10px;border:1px solid var(--border);border-radius:4px;background:var(--btn-bg);color:var(--btn-text);cursor:pointer;font-size:12px">Retry</button>';
-      document.getElementById("retryBtn")?.addEventListener("click",()=>fetchFullResults(url,opts,hasPreview,total));
+      document.getElementById("retryBtn")?.addEventListener("click",()=>fetchFullResultsWithFreshToken(hasPreview,total));
     }
   });
 }
 app.ontoolresult=({content})=>{
   const t=content?.find(c=>c.type==="text");if(!t)return;
-  let meta;try{meta=JSON.parse(t.text);}catch{sum.textContent=t.text;return;}
+  let meta;try{meta=JSON.parse(t.text);}catch{
+    /* Non-JSON (e.g. summary text) — ignore if widget already active */
+    return;
+  }
+  /* Only show the widget for recognized data shapes */
+  const isWidget=meta.fetch_full_results||meta.preview||Array.isArray(meta);
+  if(!isWidget){return;}
+  widgetActive=true;
+  document.body.style.height="auto";document.body.style.overflow="visible";document.body.style.padding="12px";
   if(meta.session_url&&!sessionUrl){sessionUrl=meta.session_url;updateSessionLink();}
   if(meta.poll_token){pollToken=meta.poll_token;}
   if(meta.download_token_url){downloadTokenUrl=meta.download_token_url;}
   if(meta.csv_url){csvUrl=meta.csv_url;updateDownloadLink();}
-  if(meta.results_url){
+  if(meta.fetch_full_results){
     if(meta.preview)processData(meta.preview);
-    const opts=meta.download_token?{headers:{"Authorization":"Bearer "+meta.download_token}}:{};
-    fetchFullResults(meta.results_url,opts,!!meta.preview,meta.total);
+    fetchFullResultsWithFreshToken(!!meta.preview,meta.total);
   }else if(meta.preview){processData(meta.preview);}
   else if(Array.isArray(meta)){processData(meta);}
-  else{sum.textContent=JSON.stringify(meta);}
 };
 
 await app.connect();