From 39e8c7fcd02b77a5af1ff17cb2c157008ba74419 Mon Sep 17 00:00:00 2001 From: Danilo Cristian da Silva Sousa <73763541+danilo-css@users.noreply.github.com> Date: Wed, 19 Feb 2025 09:42:37 -0300 Subject: [PATCH 1/4] Date handling prototype --- components/FieldSelection.tsx | 62 ++++++++++++++++++++++++++++++++++- components/Main.tsx | 57 +++++++++++++++++++++----------- stores/usePivotStore.ts | 28 +++++++++++----- 3 files changed, 119 insertions(+), 28 deletions(-) diff --git a/components/FieldSelection.tsx b/components/FieldSelection.tsx index f7f99a7..365e012 100644 --- a/components/FieldSelection.tsx +++ b/components/FieldSelection.tsx @@ -11,10 +11,66 @@ import { import { Badge } from "./ui/badge"; import { TbNumber123 } from "react-icons/tb"; import { PiTextAaFill } from "react-icons/pi"; -import { Columns3, Database, Rows3, SquareSigma } from "lucide-react"; +import { Clock, Columns3, Database, Rows3, SquareSigma } from "lucide-react"; import { useFileStore } from "@/stores/useFileStore"; import { usePivotStore } from "@/stores/usePivotStore"; import FilterDialog from "./FilterDialog"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "./ui/dialog"; + +function DateOptionsDialog({ table, field }: { table: string; field: string }) { + const { addRow, addColumn } = usePivotStore(); + + const options = [ + { label: "Year", extract: "YEAR" }, + { label: "Month", extract: "MONTH" }, + { label: "Quarter", extract: "QUARTER" }, + ] as const; + + return ( + + + + + + + Date parsing + + Date parse {field} from {table} + + +
+

Date Field Options

+
+ {options.map((opt) => ( +
+ {opt.label} +
+ addRow(table, field, opt.extract)} + /> + addColumn(table, field, opt.extract)} + /> +
+
+ ))} +
+
+ + + ); +} export default function FieldSelection() { const { queryFields, setQueryFields, isLoadingFields } = useTableStore(); @@ -101,6 +157,10 @@ export default function FieldSelection() { title="Current format: Number. Click to change to text." className="cursor-pointer hover:text-black" /> + 0 || columns.length > 0) ) { - const row_list = rows.map((row) => row.name); - const column_list = columns.map((column) => column.name); - const all_fields = [...new Set([...row_list, ...column_list])]; + const generateFieldExpression = ( + field: (typeof rows)[0] | (typeof columns)[0] + ) => { + if (field.dateExtract) { + // Extract the original field name by removing the dateExtract prefix + const originalField = field.name + .replace(`${field.dateExtract}(`, "") + .replace(")", ""); + return `CAST(EXTRACT(${field.dateExtract} FROM CAST("${originalField}" AS DATE)) AS VARCHAR) AS "${field.name}"`; + } + return `CAST("${field.name}" AS ${ + getTypeForColumn(queryFields, field.table, field.name) === "Utf8" + ? "VARCHAR" + : "DOUBLE" + }) AS "${field.name}"`; + }; + + const generateGroupByExpression = ( + field: (typeof rows)[0] | (typeof columns)[0] + ) => { + if (field.dateExtract) { + // Extract the original field name by removing the dateExtract prefix + const originalField = field.name + .replace(`${field.dateExtract}(`, "") + .replace(")", ""); + return `EXTRACT(${field.dateExtract} FROM CAST("${originalField}" AS DATE))`; + } + return `CAST("${field.name}" AS ${ + getTypeForColumn(queryFields, field.table, field.name) === "Utf8" + ? "VARCHAR" + : "DOUBLE" + })`; + }; + + const all_fields = [...rows, ...columns]; const all_fields_string = all_fields - .map( - (field) => - `CAST("${field}" AS ${ - getTypeForColumn(queryFields, files[0].name, field) === "Utf8" - ? "VARCHAR" - : "DOUBLE" - }) AS "${field}"` - ) + .map((field) => generateFieldExpression(field)) .join(", "); + const all_fields_string_groupby = all_fields - .map( - (field) => - `CAST("${field}" AS ${ - getTypeForColumn(queryFields, files[0].name, field) === "Utf8" - ? "VARCHAR" - : "DOUBLE" - })` - ) + .map((field) => generateGroupByExpression(field)) .join(", "); return ` diff --git a/stores/usePivotStore.ts b/stores/usePivotStore.ts index c6240f7..d227931 100644 --- a/stores/usePivotStore.ts +++ b/stores/usePivotStore.ts @@ -3,11 +3,13 @@ import { create } from "zustand"; type rowType = { name: string; table: string; + dateExtract?: "YEAR" | "MONTH" | "QUARTER"; }; type columnType = { name: string; table: string; + dateExtract?: "YEAR" | "MONTH" | "QUARTER"; }; type aggregationType = { @@ -25,13 +27,21 @@ export type filterType = { export type PivotState = { rows: rowType[]; setRows: (table: string, rows: string[]) => void; - addRow: (table: string, row: string) => void; + addRow: ( + table: string, + row: string, + dateExtract?: "YEAR" | "MONTH" | "QUARTER" + ) => void; clearRow: (table: string, row: string) => void; clearRows: () => void; clearFileRows: (table?: string) => void; columns: columnType[]; setColumns: (table: string, columns: string[]) => void; - addColumn: (table: string, column: string) => void; + addColumn: ( + table: string, + column: string, + dateExtract?: "YEAR" | "MONTH" | "QUARTER" + ) => void; clearColumn: (table: string, column: string) => void; clearColumns: () => void; clearFileColumns: (table?: string) => void; @@ -64,13 +74,14 @@ export const usePivotStore = create((set) => ({ .map((row) => ({ name: row, table })), ], })), - addRow: (table, row) => { + addRow: (table, row, dateExtract) => { set((state) => { - if (state.rows.some((r) => r.table === table && r.name === row)) { + const fieldId = dateExtract ? `${dateExtract}(${row})` : row; + if (state.rows.some((r) => r.table === table && r.name === fieldId)) { return { rows: state.rows }; } return { - rows: [...state.rows, { name: row, table: table }], + rows: [...state.rows, { name: fieldId, table, dateExtract }], }; }); }, @@ -97,13 +108,14 @@ export const usePivotStore = create((set) => ({ .map((column) => ({ name: column, table })), ], })), - addColumn: (table, column) => { + addColumn: (table, column, dateExtract) => { set((state) => { - if (state.columns.some((c) => c.table === table && c.name === column)) { + const fieldId = dateExtract ? `${dateExtract}(${column})` : column; + if (state.columns.some((c) => c.table === table && c.name === fieldId)) { return { columns: state.columns }; } return { - columns: [...state.columns, { name: column, table: table }], + columns: [...state.columns, { name: fieldId, table, dateExtract }], }; }); }, From 831bf4a381306724de5e91517c6010498329281c Mon Sep 17 00:00:00 2001 From: Danilo Cristian da Silva Sousa <73763541+danilo-css@users.noreply.github.com> Date: Wed, 19 Feb 2025 09:52:57 -0300 Subject: [PATCH 2/4] Date handling prototype multi-file --- components/Main.tsx | 69 ++++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/components/Main.tsx b/components/Main.tsx index 487a629..746416c 100644 --- a/components/Main.tsx +++ b/components/Main.tsx @@ -169,35 +169,58 @@ export default function Main() { aggregation.name && (rows.length > 0 || columns.length > 0) ) { - const fields = [...rows, ...columns]; + const generateFieldExpression = ( + field: (typeof rows)[0] | (typeof columns)[0] + ) => { + if (field.dateExtract) { + const originalField = field.name + .replace(`${field.dateExtract}(`, "") + .replace(")", ""); + return `CAST(EXTRACT(${ + field.dateExtract + } FROM CAST(TABLE${files.findIndex( + (file) => file.name === field.table + )}."${originalField}" AS DATE)) AS VARCHAR) AS "${field.name}"`; + } + return `CAST(TABLE${files.findIndex( + (file) => file.name === field.table + )}."${field.name}" AS ${ + getTypeForColumn(queryFields, field.table, field.name) === "Utf8" + ? "VARCHAR" + : "DOUBLE" + }) AS "${field.name}"`; + }; + const generateGroupByExpression = ( + field: (typeof rows)[0] | (typeof columns)[0] + ) => { + if (field.dateExtract) { + const originalField = field.name + .replace(`${field.dateExtract}(`, "") + .replace(")", ""); + return `EXTRACT(${ + field.dateExtract + } FROM CAST(TABLE${files.findIndex( + (file) => file.name === field.table + )}."${originalField}" AS DATE))`; + } + return `CAST(TABLE${files.findIndex( + (file) => file.name === field.table + )}."${field.name}" AS ${ + getTypeForColumn(queryFields, field.table, field.name) === "Utf8" + ? "VARCHAR" + : "DOUBLE" + })`; + }; + + const fields = [...rows, ...columns]; const uniqueFields = [ ...new Set(fields.map((field) => JSON.stringify(field))), ].map((str) => JSON.parse(str)); - const all_fields_string = uniqueFields.map( - (field) => - `CAST(TABLE${files.findIndex( - (file) => file.name === field.table - )}."${field.name}" AS ${ - getTypeForColumn(queryFields, field.table, field.name) === "Utf8" - ? "VARCHAR" - : "DOUBLE" - }) AS "${field.name}"` - ); - + const all_fields_string = uniqueFields.map(generateFieldExpression); const all_fields_string_groupby = uniqueFields - .map( - (field) => - `CAST(TABLE${files.findIndex( - (file) => file.name === field.table - )}."${field.name}" AS ${ - getTypeForColumn(queryFields, field.table, field.name) === - "Utf8" - ? "VARCHAR" - : "DOUBLE" - })` - ) + .map(generateGroupByExpression) .join(", "); const relationship_list = relationships.map((relationship) => { From 1352efa86dd91b1722da8fd0811fdfc28ce2618d Mon Sep 17 00:00:00 2001 From: Danilo Cristian da Silva Sousa <73763541+danilo-css@users.noreply.github.com> Date: Wed, 19 Feb 2025 10:42:26 -0300 Subject: [PATCH 3/4] Date handling prototype --- components/FieldSelection.tsx | 12 ++++- components/FilterDialog.tsx | 85 ++++++++++++++++++++++------------- components/Main.tsx | 42 ++++++++++++----- stores/usePivotStore.ts | 18 ++++++-- 4 files changed, 109 insertions(+), 48 deletions(-) diff --git a/components/FieldSelection.tsx b/components/FieldSelection.tsx index 365e012..ea4d4ed 100644 --- a/components/FieldSelection.tsx +++ b/components/FieldSelection.tsx @@ -49,7 +49,10 @@ function DateOptionsDialog({ table, field }: { table: string; field: string }) {

Date Field Options

{options.map((opt) => ( -
+
{opt.label}
addColumn(table, field, opt.extract)} /> +
))} @@ -130,7 +138,7 @@ export default function FieldSelection() { handleTypeChange(parentKey.name, index) } className="cursor-pointer hover:text-black" - title="Current format: Text. Click to change to number." + title="Current format: Text. Click to change to number or date." /> { - if (db) { - setLoading(true); - const result: Arrow = await runQuery( - db, - ` - SELECT DISTINCT "${field}" - FROM '${table}' - ORDER BY "${field}" ASC` - ); + if (!db) return; + setLoading(true); + + try { + let query; + // Extract the original field name if it's already a date-extracted field + const originalField = field.match(/^(YEAR|MONTH|QUARTER)\((.*?)\)$/); + const actualField = originalField ? originalField[2] : field; + const actualExtract = originalField ? originalField[1] : dateExtract; - const cleanedData = result.toArray().map((row) => { + if (actualExtract) { + query = `SELECT DISTINCT REPLACE(CAST(EXTRACT(${actualExtract} FROM CAST("${actualField}" AS DATE)) AS VARCHAR), '"', '') as value FROM '${table}' WHERE "${actualField}" IS NOT NULL ORDER BY value`; + } else { + query = `SELECT DISTINCT REPLACE("${actualField}", '"', '') as value FROM '${table}' WHERE "${actualField}" IS NOT NULL ORDER BY value`; + } + const result = await runQuery(db, query); + + const values = result + .toArray() // eslint-disable-next-line @typescript-eslint/no-explicit-any - const cleanedRow: any = {}; - for (const [key, value] of Object.entries(row)) { - cleanedRow[key] = - typeof value === "string" ? value.replace(/"/g, "") : value; - } - return cleanedRow; + .map((row: { value: { toString: () => any } }) => row.value.toString()); + setValues(values); + } catch (error) { + console.error("Error fetching filter options:", error); + toast({ + title: "Error", + description: "Failed to fetch filter values. Please try again.", + variant: "destructive", }); - - setValues( - cleanedData.map( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (obj: any) => obj[field]?.toString() ?? "(Null)" - ) - ); + } finally { setLoading(false); } }; + // Remove the automatic fetching from useEffect since we now have a manual fetch button + useEffect(() => { + // Reset selected values when dialog opens + if (open) { + // Find the filter checking both regular and date-extracted field names + const existingFilter = filters.find( + (f) => + f.table === table && + (f.field === field || f.field === `${dateExtract}(${field})`) + ); + setSelectedValues(existingFilter?.values || []); + } + }, [open, table, field, dateExtract, filters]); + const filteredValues = values.filter((value) => value.toString().toLowerCase().includes(searchQuery.toLowerCase()) ); @@ -108,7 +129,7 @@ export default function FilterDialog({ }; const handleSubmit = () => { - addFilter(table, field, selectedValues); + addFilter(table, field, selectedValues, dateExtract); toast({ title: "Filter Applied", description: `Successfully applied filter for ${field}`, @@ -138,7 +159,9 @@ export default function FilterDialog({ Add filter - Add filter for {field} + {dateExtract + ? `Filter by ${dateExtract.toLowerCase()}s from "${field}" in "${table}"` + : `Add filter for "${field}" from "${table}"`}
diff --git a/components/Main.tsx b/components/Main.tsx index 746416c..4133547 100644 --- a/components/Main.tsx +++ b/components/Main.tsx @@ -153,12 +153,21 @@ export default function Main() { ${ filters.length > 0 ? `WHERE ${filters - .map( - (filter) => - `"${filter.field}" IN (${filter.values + .map((filter) => { + if (filter.dateExtract) { + const originalField = filter.field + .replace(`${filter.dateExtract}(`, "") + .replace(")", ""); + return `CAST(EXTRACT(${ + filter.dateExtract + } FROM CAST("${originalField}" AS DATE)) AS VARCHAR) IN (${filter.values .map((value) => `'${value}'`) - .join(", ")})` - ) + .join(", ")})`; + } + return `"${filter.field}" IN (${filter.values + .map((value) => `'${value}'`) + .join(", ")})`; + }) .join(" AND ")}` : "" } @@ -282,14 +291,25 @@ export default function Main() { ${ filters.length > 0 ? `WHERE ${filters - .map( - (filter) => - `TABLE${files.findIndex( + .map((filter) => { + if (filter.dateExtract) { + const originalField = filter.field + .replace(`${filter.dateExtract}(`, "") + .replace(")", ""); + return `CAST(EXTRACT(${ + filter.dateExtract + } FROM CAST(TABLE${files.findIndex( (file) => file.name === filter.table - )}."${filter.field}" IN (${filter.values + )}."${originalField}" AS DATE)) AS VARCHAR) IN (${filter.values .map((value) => `'${value}'`) - .join(", ")})` - ) + .join(", ")})`; + } + return `TABLE${files.findIndex( + (file) => file.name === filter.table + )}."${filter.field}" IN (${filter.values + .map((value) => `'${value}'`) + .join(", ")})`; + }) .join(" AND ")}` : "" } diff --git a/stores/usePivotStore.ts b/stores/usePivotStore.ts index d227931..4d09726 100644 --- a/stores/usePivotStore.ts +++ b/stores/usePivotStore.ts @@ -22,6 +22,7 @@ export type filterType = { table: string; field: string; values: string[]; + dateExtract?: "YEAR" | "MONTH" | "QUARTER"; }; export type PivotState = { @@ -54,7 +55,12 @@ export type PivotState = { clearAggregation: () => void; clearFileAggregation: (table?: string) => void; filters: filterType[]; - addFilter: (table: string, field: string, values: string[]) => void; + addFilter: ( + table: string, + field: string, + values: string[], + dateExtract?: "YEAR" | "MONTH" | "QUARTER" + ) => void; clearFilter: (table: string, field: string) => void; clearFilters: () => void; clearFileFilters: (table?: string) => void; @@ -148,13 +154,17 @@ export const usePivotStore = create((set) => ({ table && state.aggregation.table === table ? {} : state.aggregation, })), filters: [], - addFilter: (table, field, values) => { + addFilter: (table, field, values, dateExtract) => { set((state) => { + const fieldId = dateExtract ? `${dateExtract}(${field})` : field; const filteredFilters = state.filters.filter( - (f) => !(f.table === table && f.field === field) + (f) => !(f.table === table && f.field === fieldId) ); return { - filters: [...filteredFilters, { table, field, values }], + filters: [ + ...filteredFilters, + { table, field: fieldId, values, dateExtract }, + ], }; }); }, From f83ce2486794bde434933233596772d4b3b3b6c0 Mon Sep 17 00:00:00 2001 From: Danilo Cristian da Silva Sousa <73763541+danilo-css@users.noreply.github.com> Date: Wed, 19 Feb 2025 11:22:14 -0300 Subject: [PATCH 4/4] Update FilterDialog.tsx --- components/FilterDialog.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/FilterDialog.tsx b/components/FilterDialog.tsx index 449e8cc..d63b2d6 100644 --- a/components/FilterDialog.tsx +++ b/components/FilterDialog.tsx @@ -89,7 +89,8 @@ export default function FilterDialog({ console.error("Error fetching filter options:", error); toast({ title: "Error", - description: "Failed to fetch filter values. Please try again.", + description: + "Failed to fetch filter values. Please make sure this is a proper date or text field.", variant: "destructive", }); } finally {