fix: init card database before accepting connections

api.js

@@ -15,7 +15,7 @@ import { gradeImage, medianGrade } from "./lib/grading/grading.js";
 import { parseListingLanguagesFromInput, filterByCondition, detectCondition, flagPriceOutliers, filterRelevantResults, isGradedCard } from "./lib/search/filters.js";
 import { buildEbaySearchQuery } from "./lib/search/listingQuery.js";
 import { EBAY_CATEGORY_TCG_SINGLE_CARDS_US } from "./lib/search/ebayCategories.js";
-import { saveGradeLog, getGradeLogs, getGradeLogsByUser, getGradeLog, deleteGradeLog, saveDrop, getDrops, getDrop, saveWebhook, getWebhooks, deleteWebhook, getFirestoreStatus, saveAlert, getActiveAlerts, updateAlert, getAlertsByEmail, saveErrorLog, getErrorLogs, clearErrorLogs, getPortfolio, addToPortfolio, removeFromPortfolio, updatePortfolioCard, savePortfolioSnapshot, getPortfolioSnapshots, listPortfolioUserIds, trackSearchFrequency, getTopSearchedCards } from "./lib/data/firestore.js";
+import { saveGradeLog, getGradeLogs, getGradeLogsByUser, getGradeLog, deleteGradeLog, saveDrop, getDrops, getDrop, saveWebhook, getWebhooks, deleteWebhook, getFirestoreStatus, saveAlert, getActiveAlerts, updateAlert, getAlertsByEmail, saveErrorLog, getErrorLogs, clearErrorLogs, getPortfolio, addToPortfolio, removeFromPortfolio, updatePortfolioCard, savePortfolioSnapshot, getPortfolioSnapshots, listPortfolioUserIds, trackSearchFrequency, getTopSearchedCards, recordMilestone, getFunnelStats } from "./lib/data/firestore.js";
 import { getDemoSearchResult, getDemoResult, listDemoCards, findDemoByNumber } from "./lib/cards/demo.js";
 import { csvEscape, csvRow } from "./lib/data/csv.js";
 import { createApiKey, listApiKeys, listAllKeys, listKeysByOwner, getApiKey, updateApiKey, deleteApiKey, rotateApiKey, validateApiKey } from "./lib/auth/api-keys.js";
@@ -26,7 +26,7 @@ import { saveGradedImages } from "./lib/cards/grading-dataset.js";
 import { verifyGoogleToken, generateJwt, verifyJwt } from "./lib/auth/auth.js";
 import { seedFromTCGPlayer } from "./lib/sources/tcgplayer.js";
 import { getOrCreateCard, findCardByQuery, parseCardIdentity, resolveCardIdToQuery, SET_NAME_MAP } from "./lib/cards/card-identity.js";
-import { initCardDatabase, searchCards, refreshCardDatabase, getAllSets, getSetWithCards, findCardByCardId } from "./lib/cards/card-database.js";
+import { initCardDatabase, searchCards, refreshCardDatabase, getAllSets, getSetWithCards, findCardByCardId, isCardDatabaseReady } from "./lib/cards/card-database.js";
 import { raspMiddleware, getSecurityEvents } from "./lib/security/rasp.js";
 import { fileURLToPath } from "url";
 import path from "path";
@@ -184,6 +184,11 @@ const clientSecret = process.env.EBAY_CLIENT_SECRET;
 async function getToken() { return getAccessToken(clientId, clientSecret); }
 async function on401() { invalidateToken(); }
 
+function requireCardDb(req, res, next) {
+  if (!isCardDatabaseReady()) return res.status(503).json({ error: "Card database loading, try again shortly", retryAfter: 5 });
+  next();
+}
+
 function validateQuery(q, res) {
   if (!q) { res.status(400).json({ error: "Missing required parameter: q" }); return false; }
   if (q.length > 200) { res.status(400).json({ error: "Query too long (max 200 characters)" }); return false; }
@@ -372,6 +377,7 @@ app.get("/api/search", apiAuthMiddleware, (req, res, next) => { req._errorType =
     }
     getOrCreateCard(q, { source: result.source, lang: config.language }).catch(() => {});
     trackSearchFrequency(q).catch(() => {});
+    if (req.userId) recordMilestone(req.userId, "firstSearch").catch(() => {});
     res.json(result);
   } catch (e) {
     logError(req._errorType || "api", e.message, req.originalUrl, req.requestId);
@@ -503,6 +509,7 @@ app.post("/api/grade", authMiddleware, (req, res, next) => { req._errorType = "g
       }
     }
 
+    if (req.userId && grade && !grade.error) recordMilestone(req.userId, "firstGrade").catch(() => {});
     res.json({ grade, gradeId, stored: !!(grade && !grade.error) });
   } catch (e) {
     logError(req._errorType || "api", e.message, req.originalUrl, req.requestId);
@@ -613,8 +620,9 @@ app.get("/api/grades", authMiddleware, async (req, res) => {
 // GET /api/errors
 app.get("/api/errors", authMiddleware, async (req, res) => {
   const limit = Math.min(100, Math.max(1, Number(req.query.limit) || 20));
+  const type = req.query.type || undefined;
   try {
-    const errors = await getErrorLogs({ limit });
+    const errors = await getErrorLogs({ limit, type });
     res.json({ errors, count: errors.length });
   } catch (e) {
     res.status(500).json({ error: safeErrorMessage(e), requestId: req.requestId });
@@ -642,6 +650,16 @@ app.get("/api/analytics", ownerOnly, async (req, res) => {
   }
 });
 
+// GET /api/funnel
+app.get("/api/funnel", ownerOnly, async (req, res) => {
+  try {
+    const stats = await getFunnelStats();
+    res.json(stats);
+  } catch (e) {
+    res.status(500).json({ error: safeErrorMessage(e), requestId: req.requestId });
+  }
+});
+
 // GET /api/grading-dataset/stats
 app.get("/api/grading-dataset/stats", ownerOnly, async (req, res) => {
   try {
@@ -708,6 +726,7 @@ app.post("/auth/google", authLimiter, async (req, res) => {
     }
 
     const isAdmin = gUser.sub === process.env.CASECOMP_ADMIN_SUB;
+    recordMilestone(gUser.sub, "signup").catch(() => {});
     res.json({ jwt, apiKey, isAdmin, user: { id: gUser.sub, email: gUser.email, name: gUser.name, picture: gUser.picture } });
   } catch (e) {
     res.status(401).json({ error: "Invalid Google token" });
@@ -860,7 +879,7 @@ app.delete("/api/admin/keys/:id", authMiddleware, async (req, res) => {
 });
 
 // GET /api/autocomplete
-app.get("/api/autocomplete", (req, res) => {
+app.get("/api/autocomplete", requireCardDb, (req, res) => {
   const q = (req.query.q || "").trim();
   if (!q || q.length < 2) return res.status(400).json({ error: "Query must be at least 2 characters" });
   if (q.length > 100) return res.status(400).json({ error: "Query too long (max 100 characters)" });
@@ -870,15 +889,15 @@ app.get("/api/autocomplete", (req, res) => {
 });
 
 // GET /api/sets
-app.get("/api/sets", (req, res) => {
+app.get("/api/sets", requireCardDb, (req, res) => {
   const sets = getAllSets();
   const era = req.query.era;
   const filtered = era ? sets.filter(s => s.era === era) : sets;
   res.json({ sets: filtered, count: filtered.length });
 });
 
 // GET /api/sets/:setCode
-app.get("/api/sets/:setCode", (req, res) => {
+app.get("/api/sets/:setCode", requireCardDb, (req, res) => {
   const result = getSetWithCards(req.params.setCode);
   if (!result) return res.status(404).json({ error: "Set not found" });
   res.json(result);
@@ -1999,6 +2018,7 @@ app.post("/api/portfolio", authMiddleware, async (req, res) => {
       purchaseSource: purchaseSource || "",
       quantity: quantity != null ? Number(quantity) : 1,
     });
+    recordMilestone(userId, "firstPortfolioAdd").catch(() => {});
     res.status(201).json({ ok: true, card });
   } catch (e) {
     logError("portfolio", e.message, req.originalUrl, req.requestId);
@@ -2413,9 +2433,8 @@ app.use((err, req, res, _next) => {
 });
 
 const PORT = process.env.API_PORT || 3000;
-app.listen(PORT, async () => {
-  console.log(`Casecomp API listening on http://localhost:${PORT}`);
-  console.log(`Swagger docs: http://localhost:${PORT}/docs`);
+
+async function startup() {
   if (clientId && clientSecret) {
     try {
       await getToken();
@@ -2424,5 +2443,25 @@ app.listen(PORT, async () => {
       console.warn(`eBay token warmup failed: ${e.message}`);
     }
   }
-  initCardDatabase().catch(() => {});
+  try {
+    await initCardDatabase();
+  } catch (e) {
+    console.warn(`Card database init failed: ${e.message}`);
+  }
+  app.listen(PORT, () => {
+    console.log(`Casecomp API listening on http://localhost:${PORT}`);
+    console.log(`Swagger docs: http://localhost:${PORT}/docs`);
+  });
+}
+
+startup();
+
+process.on("unhandledRejection", (reason) => {
+  const msg = reason instanceof Error ? reason.message : String(reason);
+  logError("unhandledRejection", msg, reason instanceof Error ? reason.stack?.split("\n")[1]?.trim() : "");
+});
+
+process.on("uncaughtException", (err) => {
+  logError("uncaughtException", err.message, err.stack?.split("\n")[1]?.trim());
+  setTimeout(() => process.exit(1), 1000);
 });

lib/cards/card-database.js

@@ -60,6 +60,7 @@ const RARITY_MAP = {
 const RARITY_TIERS = Object.keys(RARITY_MAP);
 
 let cardIndex = [];
+let cardDbReady = false;
 const tcgdexSetMeta = new Map();
 
 async function fetchSetMetadata(timeout = 15000) {
@@ -306,6 +307,7 @@ export async function initCardDatabase() {
   if (cached && cached.length > 0) {
     cardIndex = cached;
     deriveSetTotals(cardIndex);
+    cardDbReady = true;
     console.log(`Card database loaded from cache: ${cardIndex.length} cards`);
 
     fetchAndBuildIndex().then(async (freshIndex) => {
@@ -324,6 +326,7 @@ export async function initCardDatabase() {
   if (freshIndex && freshIndex.length > 0) {
     cardIndex = freshIndex;
     deriveSetTotals(cardIndex);
+    cardDbReady = true;
     console.log(`Card database loaded from TCGdex: ${cardIndex.length} cards`);
     saveCardDatabaseToCache(freshIndex).catch(() => {});
     return cardIndex.length;
@@ -333,6 +336,10 @@ export async function initCardDatabase() {
   return 0;
 }
 
+export function isCardDatabaseReady() {
+  return cardDbReady;
+}
+
 export async function refreshCardDatabase() {
   const freshIndex = await fetchAndBuildIndex();
   if (freshIndex && freshIndex.length > 0) {

lib/data/analytics.js

@@ -68,28 +68,47 @@ export async function getAnalytics({ days = 7, limit = 1000 } = {}) {
 
     const byTier = {};
     const byPath = {};
+    const byStatus = {};
     const queries = {};
+    const dailyMap = {};
+    const users = new Set();
     let totalLatency = 0;
 
     for (const d of docs) {
       byTier[d.tier] = (byTier[d.tier] || 0) + 1;
       byPath[d.path] = (byPath[d.path] || 0) + 1;
+      const sc = d.status ? String(d.status).charAt(0) + "xx" : "unknown";
+      byStatus[sc] = (byStatus[sc] || 0) + 1;
       if (d.query) queries[d.query] = (queries[d.query] || 0) + 1;
+      if (d.userId) users.add(d.userId);
       totalLatency += d.latencyMs || 0;
+      const day = d.ts?.split("T")[0];
+      if (day) {
+        if (!dailyMap[day]) dailyMap[day] = { count: 0, latency: 0 };
+        dailyMap[day].count++;
+        dailyMap[day].latency += d.latencyMs || 0;
+      }
     }
 
     const topQueries = Object.entries(queries)
       .sort((a, b) => b[1] - a[1])
       .slice(0, 10)
       .map(([query, count]) => ({ query, count }));
 
+    const daily = Object.entries(dailyMap)
+      .sort()
+      .map(([date, v]) => ({ date, count: v.count, avgLatencyMs: v.count > 0 ? Math.round(v.latency / v.count) : 0 }));
+
     return {
       total,
       days,
       byTier,
       byPath,
+      byStatus,
       topQueries,
       avgLatencyMs: total > 0 ? Math.round(totalLatency / total) : 0,
+      uniqueUsers: users.size,
+      daily,
     };
   } catch {
     return { total: 0, byTier: {}, byPath: {}, topQueries: [], avgLatencyMs: 0 };

lib/data/firestore.js

@@ -265,6 +265,7 @@ export async function saveErrorLog(record) {
     await fs.collection("error-logs").add({
       ...record,
       createdAt: Firestore.FieldValue.serverTimestamp(),
+      expiresAt: new Date(Date.now() + 30 * 86400000),
     });
   } catch {}
 }
@@ -281,17 +282,63 @@ export async function clearErrorLogs() {
   } catch { return 0; }
 }
 
-export async function getErrorLogs({ limit = 20 } = {}) {
+export async function getErrorLogs({ limit = 20, type } = {}) {
   const fs = getDb();
   if (!fs) return [];
   try {
-    const snap = await fs.collection("error-logs").orderBy("createdAt", "desc").limit(limit).get();
+    let q = fs.collection("error-logs");
+    if (type) q = q.where("type", "==", type);
+    const snap = await q.orderBy("createdAt", "desc").limit(limit).get();
     return snap.docs.map(d => ({ id: d.id, ...d.data() }));
   } catch {
     return [];
   }
 }
 
+// ── User milestones (funnel tracking) ──
+
+const MILESTONES_COLLECTION = "user-milestones";
+const MILESTONE_FIELDS = ["signup", "firstSearch", "firstGrade", "firstPortfolioAdd"];
+
+export async function recordMilestone(userId, field) {
+  if (!userId || !MILESTONE_FIELDS.includes(field)) return;
+  const fs = getDb();
+  if (!fs) return;
+  try {
+    const ref = fs.collection(MILESTONES_COLLECTION).doc(userId);
+    const doc = await ref.get();
+    if (doc.exists && doc.data()[field]) return;
+    await ref.set(
+      { userId, [field]: new Date().toISOString(), updatedAt: new Date().toISOString() },
+      { merge: true },
+    );
+  } catch {}
+}
+
+let funnelCache = null;
+let funnelCacheAt = 0;
+
+export async function getFunnelStats() {
+  if (funnelCache && Date.now() - funnelCacheAt < 300000) return funnelCache;
+  const fs = getDb();
+  if (!fs) return { signup: 0, firstSearch: 0, firstGrade: 0, firstPortfolioAdd: 0 };
+  try {
+    const snap = await fs.collection(MILESTONES_COLLECTION).limit(1000).get();
+    const result = { signup: 0, firstSearch: 0, firstGrade: 0, firstPortfolioAdd: 0 };
+    for (const doc of snap.docs) {
+      const d = doc.data();
+      for (const f of MILESTONE_FIELDS) {
+        if (d[f]) result[f]++;
+      }
+    }
+    funnelCache = result;
+    funnelCacheAt = Date.now();
+    return result;
+  } catch {
+    return { signup: 0, firstSearch: 0, firstGrade: 0, firstPortfolioAdd: 0 };
+  }
+}
+
 // ── Cache (replaces file-based JSON caches) ──
 
 export async function cacheGet(collection, key) {