feature: install duckdb, insert and query people (#314)

* feature: install duckdb, insert and query people

* feature: add flog to indicate the DuckDb is connected

* feature: move the mapper out of dbduck hook

* refactor: clean duckdb hook

* feature: duplicate overview page

* feature: create new route with duckdb

* feature: add component to toggle console

* feature: insert snapshot data in duckdb

* feature: add code editor

* feature: create a duckdb provider

* feature: ensure data is provisionned once

* feature: add a button to run the query

* feature: fix table provisioning

* feature: naive implementation of dual table mapper

* feature: get dualtable from query in main component

* feature: add logs in table provisionning

* feature: add error container on the right

frontend/package.json

@@ -18,8 +18,13 @@
     "build-spa-snapshot": "vite build src/spas/snapshot --outDir ../../../../tools/driftdb/driftdb/spa/snapshot"
   },
   "dependencies": {
+    "@duckdb/duckdb-wasm": "^1.28.1-dev106.0",
+    "@duckdb/react-duckdb": "^1.28.1-dev106.0",
+    "@emotion/react": "11.11.3",
     "@emotion/styled": "^11.11.0",
     "@tanstack/react-query": "^5.17.9",
+    "@uiw/react-textarea-code-editor": "^3.0.2",
+    "apache-arrow": "^15.0.0",
     "axios": "^1.4.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",

frontend/src/components/DuckDb/DuckDbProvider.tsx

@@ -0,0 +1,67 @@
+import {
+  createContext,
+  useContext,
+  useState,
+  useEffect,
+  ReactNode,
+} from "react";
+import * as duckdb from "@duckdb/duckdb-wasm";
+import duckdb_wasm from "@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url";
+import mvp_worker from "@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url";
+import duckdb_wasm_eh from "@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url";
+import eh_worker from "@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js?url";
+
+interface DuckDbProviderProps {
+  children: ReactNode; // Typing children using ReactNode
+}
+
+const MANUAL_BUNDLES = {
+  mvp: {
+    mainModule: duckdb_wasm,
+    mainWorker: mvp_worker,
+  },
+  eh: {
+    mainModule: duckdb_wasm_eh,
+    mainWorker: eh_worker,
+  },
+} as const;
+
+// Create a Context
+const DuckDbContext = createContext<duckdb.AsyncDuckDBConnection | null>(null);
+
+const useDuckDb = () => useContext(DuckDbContext);
+
+const DuckDbProvider = ({ children }: DuckDbProviderProps) => {
+  const [db, setDb] = useState<duckdb.AsyncDuckDBConnection | null>(null);
+
+  useEffect(() => {
+    // Initialize your DuckDb instance here.
+    // This is a placeholder for actual DuckDb initialization logic.
+    // You might need to asynchronously load the database or its schema.
+    const initializeDuckDb = async () => {
+      try {
+        const bundle = await duckdb.selectBundle(MANUAL_BUNDLES);
+        if (!bundle.mainWorker) {
+          throw new Error("No worker found in the selected bundle");
+        }
+        const worker = new Worker(bundle.mainWorker);
+        const logger = new duckdb.ConsoleLogger();
+        const dbInstance = new duckdb.AsyncDuckDB(logger, worker);
+        await dbInstance.instantiate(bundle.mainModule, bundle.pthreadWorker);
+        const connection = await dbInstance.connect();
+
+        setDb(connection);
+      } catch (error) {
+        console.error("Failed to initialize DuckDB:", error);
+      }
+    };
+
+    void initializeDuckDb();
+  }, []);
+
+  return <DuckDbContext.Provider value={db}>{children}</DuckDbContext.Provider>;
+};
+
+DuckDbProvider.useDuckDb = useDuckDb;
+
+export default DuckDbProvider;

frontend/src/pages/Overview/OverviewWithDuckDb.tsx

@@ -0,0 +1,234 @@
+import Lineage from "../../components/Lineage/Lineage";
+import { useCallback, useMemo, useState } from "react";
+
+import {
+  Container,
+  DiffTableContainer,
+  LineageContainer,
+  StyledCollapsibleContent,
+  StyledCollapsibleTitle,
+  StyledDate,
+  StyledDateButton,
+  StyledHeader,
+  StyledSelect,
+} from "./components";
+import {
+  fetchCommitPatchQuery,
+  fetchCommitsQuery,
+  loader,
+  localStrategyLoader,
+  useOverviewLoaderData,
+} from "./loader";
+import { getNodesFromConfig } from "./flow-nodes";
+import { DiffTable } from "../DisplayCommit/DiffTable";
+import Loader from "../../components/Common/Loader";
+import StarUs from "../../components/Common/StarUs";
+import { useQuery } from "@tanstack/react-query";
+import SqlEditor from "./SqlEditor";
+import DuckDbProvider from "../../components/DuckDb/DuckDbProvider";
+import { DualTableProps } from "../../components/Table/DualTable";
+
+const OverviewWithDb = () => {
+  const db = DuckDbProvider.useDuckDb();
+
+  const [dualTableFromSqlQuery, setdualTableFromSqlQuery] =
+    useState<DualTableProps | null>(null);
+
+  const loaderData = useOverviewLoaderData();
+  const searchParams = new URLSearchParams(window.location.search);
+
+  const tableName = searchParams.get("tableName") || "";
+  const initialSelectedMetricNumber = Number(searchParams.get("metric")) || 0;
+
+  const initialSelectedMetric = useMemo(() => {
+    if (tableName.length > 0) {
+      const metric = loaderData.config.metrics.find((metric) =>
+        tableName.includes(metric.filepath.replace(".csv", ""))
+      );
+      return (
+        metric || {
+          filepath: tableName,
+          dateColumnName: "",
+          KPIColumnName: "",
+          metricName: tableName,
+          timeGrains: [],
+          dimensions: [],
+        }
+      );
+    } else {
+      return loaderData.config.metrics[initialSelectedMetricNumber];
+    }
+  }, [tableName, loaderData.config.metrics, initialSelectedMetricNumber]);
+
+  const [selectedMetric, setSelectedMetric] = useState(initialSelectedMetric);
+  const handleSetSelectedMetric = useCallback(
+    (newMetricIndex: number) => {
+      const searchParams = new URLSearchParams(window.location.search);
+      searchParams.set("metric", newMetricIndex.toString());
+      const newUrl = `${window.location.pathname}?${searchParams.toString()}`;
+      window.history.pushState({ path: newUrl }, "", newUrl);
+      setSelectedMetric(loaderData.config.metrics[newMetricIndex]);
+    },
+    [loaderData.config.metrics]
+  );
+
+  const initialSnapshotDate = searchParams.get("snapshotDate")
+    ? new Date(searchParams.get("snapshotDate") as string)
+    : new Date();
+  const [currentDate, setCurrentDate] = useState(initialSnapshotDate);
+
+  const initialCommitSha = searchParams.get("commitSha");
+  const [selectedCommit, setSelectedCommit] = useState(initialCommitSha);
+  const handleSetSelectedCommit = useCallback((newCommitSha: string) => {
+    const searchParams = new URLSearchParams(window.location.search);
+    searchParams.set("commitSha", newCommitSha);
+    const newUrl = `${window.location.pathname}?${searchParams.toString()}`;
+    window.history.pushState({ path: newUrl }, "", newUrl);
+    setSelectedCommit(newCommitSha);
+  }, []);
+
+  const dualTableData = useQuery(
+    fetchCommitPatchQuery(loaderData, selectedCommit)
+  );
+
+  const commitListData = useQuery(fetchCommitsQuery(loaderData, currentDate));
+
+  const { nodes, edges } = getNodesFromConfig(
+    selectedMetric,
+    commitListData.data || [],
+    handleSetSelectedCommit,
+    commitListData.isLoading
+  );
+
+  const topContainerValues = ["lineage", "query", null] as const;
+
+  const [topContainer, setTopContainer] = useState<
+    (typeof topContainerValues)[number]
+  >(topContainerValues[0]);
+
+  const handleTopContainerClick = () => {
+    const currentIndex = topContainerValues.findIndex(
+      (value) => value === topContainer
+    );
+    const nextIndex = (currentIndex + 1) % topContainerValues.length;
+    setTopContainer(topContainerValues[nextIndex]);
+  };
+
+  const handleSetCurrentDate = useCallback(
+    (newDate: Date) => {
+      const searchParams = new URLSearchParams(window.location.search);
+      searchParams.set(
+        "snapshotDate",
+        currentDate.toISOString().substring(0, 10)
+      );
+      const newUrl = `${window.location.pathname}?${searchParams.toString()}`;
+      window.history.pushState({ path: newUrl }, "", newUrl);
+
+      setCurrentDate(newDate);
+      handleSetSelectedCommit("");
+    },
+    [currentDate, handleSetSelectedCommit]
+  );
+
+  const incrementDate = useCallback(() => {
+    const newDate = new Date(currentDate.setDate(currentDate.getDate() + 1));
+    handleSetCurrentDate(newDate);
+  }, [handleSetCurrentDate, currentDate]);
+
+  const decrementDate = useCallback(() => {
+    const newDate = new Date(currentDate.setDate(currentDate.getDate() - 1));
+    handleSetCurrentDate(newDate);
+  }, [handleSetCurrentDate, currentDate]);
+
+  return (
+    <Container>
+      <StyledHeader>
+        <StyledCollapsibleTitle onClick={() => handleTopContainerClick()}>
+          {topContainer == "lineage"
+            ? "Show DuckDb Console"
+            : topContainer == "query"
+            ? "Hide DuckDb Console"
+            : "Show Lineage"}
+        </StyledCollapsibleTitle>
+        {db ? (
+          <div>Connected to DuckDB ✅</div>
+        ) : (
+          <div>Connecting to DuckDB</div>
+        )}
+        <StyledDate>
+          <StyledDateButton onClick={decrementDate}>{"<"}</StyledDateButton>
+          {currentDate.toLocaleDateString()}
+          <StyledDateButton onClick={incrementDate}>{">"}</StyledDateButton>
+        </StyledDate>
+
+        {loaderData.config.metrics.length > 0 && (
+          <StyledSelect
+            value={selectedMetric.filepath}
+            onChange={(e) => {
+              const selectedMetric = loaderData.config.metrics.findIndex(
+                (metric) => metric.filepath === e.target.value
+              );
+              if (
+                typeof selectedMetric === "number" &&
+                !isNaN(selectedMetric)
+              ) {
+                handleSetSelectedMetric(selectedMetric);
+              }
+            }}
+          >
+            {loaderData.config.metrics
+              .reduce((unique, metric) => {
+                return unique.some((item) => item.filepath === metric.filepath)
+                  ? unique
+                  : [...unique, metric];
+              }, [] as typeof loaderData.config.metrics)
+              .map((metric) => (
+                <option key={metric.filepath} value={metric.filepath}>
+                  {metric.filepath}
+                </option>
+              ))}
+          </StyledSelect>
+        )}
+        <StarUs />
+      </StyledHeader>
+
+      <LineageContainer>
+        <StyledCollapsibleContent isCollapsed={!topContainer}>
+          {topContainer == "lineage" ? (
+            <Lineage nodes={nodes} edges={edges} />
+          ) : topContainer == "query" ? (
+            db &&
+            dualTableData.data && (
+              <SqlEditor
+                db={db}
+                dualTable={dualTableData.data}
+                setQueryResult={setdualTableFromSqlQuery}
+              />
+            )
+          ) : null}
+        </StyledCollapsibleContent>
+      </LineageContainer>
+
+      {selectedCommit ? (
+        <DiffTableContainer>
+          {dualTableFromSqlQuery ? (
+            <DiffTable dualTableProps={dualTableFromSqlQuery} />
+          ) : dualTableData.isLoading ? (
+            <Loader />
+          ) : (
+            dualTableData.data && (
+              <DiffTable dualTableProps={dualTableData.data} />
+            )
+          )}
+        </DiffTableContainer>
+      ) : (
+        "No drift selected"
+      )}
+    </Container>
+  );
+};
+
+OverviewWithDb.loader = loader;
+OverviewWithDb.localStrategyLoader = localStrategyLoader;
+
+export default OverviewWithDb;