Prepare for the next update: Data Anchor

README.md

@@ -22,6 +22,11 @@ Transform data and create rich visualizations iteratively with AI 🪄. Try Data
 
 ## News 🔥🔥🔥
 
+- [03-20-2025] Data Formulator 0.1.7: Anchoring ⚓︎
+  - Anchor an intermediate dataset, so that followup data analysis are built on top of the anchored data, not the original one.
+  - It is handy when: clean a data and work with only the cleaned data; create a subset from the original data or join multiple data, and then focus your analysis from there. The AI agent will be less likely to get confused and work faster. ⚡️⚡️
+  - Don't forget to update Data Formulator to test it out!
+
 - [02-20-2025] Data Formulator 0.1.6 released! 
   - Now supports working with multiple datasets at once! Tell Data Formulator which data tables you would like to use in the encoding shelf, and it will figure out how to join the tables to create a visualization to answer your question. 🪄
   - Checkout the demo at [[https://github.com/microsoft/data-formulator/releases/tag/0.1.6]](https://github.com/microsoft/data-formulator/releases/tag/0.1.6).

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "data_formulator"
-version = "0.1.6.2"
+version = "0.1.7"
 
 requires-python = ">=3.9"
 authors = [

src/app/dfSlice.tsx

@@ -142,9 +142,11 @@ let deleteChartsRoutine = (state: DataFormulatorState, chartIds: string[]) => {
     state.activeThreadChartId = activeThreadChartId;
 
     let unrefedDerivedTableIds = getUnrefedDerivedTableIds(state);
-    state.tables = state.tables.filter(t => !unrefedDerivedTableIds.includes(t.id));
+    let tableIdsToDelete = state.tables.filter(t => !t.anchored && unrefedDerivedTableIds.includes(t.id)).map(t => t.id);
+
+    state.tables = state.tables.filter(t => !tableIdsToDelete.includes(t.id));
     // remove intermediate charts that lead to this table
-    state.charts = state.charts.filter(c => !(c.intermediate && unrefedDerivedTableIds.includes(c.intermediate.resultTableId)));
+    state.charts = state.charts.filter(c => !(c.intermediate && tableIdsToDelete.includes(c.intermediate.resultTableId)));
 }
 
 export const fetchFieldSemanticType = createAsyncThunk(
@@ -335,6 +337,16 @@ export const dataFormulatorSlice = createSlice({
             // separate this, so that we only delete on tier of table a time
             state.charts = state.charts.filter(c => !(c.intermediate && c.intermediate.resultTableId == tableId));
         },
+        updateTableAnchored: (state, action: PayloadAction<{tableId: string, anchored: boolean}>) => {
+            let tableId = action.payload.tableId;
+            let anchored = action.payload.anchored;
+            state.tables = state.tables.map(t => t.id == tableId ? {...t, anchored} : t);
+        },
+        updateTableDisplayId: (state, action: PayloadAction<{tableId: string, displayId: string}>) => {
+            let tableId = action.payload.tableId;
+            let displayId = action.payload.displayId;
+            state.tables = state.tables.map(t => t.id == tableId ? {...t, displayId} : t);
+        },
         addChallenges: (state, action: PayloadAction<{tableId: string, challenges: { text: string; difficulty: 'easy' | 'medium' | 'hard'; }[]}>) => {
             state.activeChallenges = [...state.activeChallenges, action.payload];
         },
@@ -371,18 +383,6 @@ export const dataFormulatorSlice = createSlice({
                 }
             })
         },
-        updateChartScaleFactor: (state, action: PayloadAction<{chartId: string, scaleFactor: number}>) => {
-            let chartId = action.payload.chartId;
-            let scaleFactor = action.payload.scaleFactor;
-
-            state.charts = state.charts.map(chart => {
-                if (chart.id == chartId) {
-                    return { ...chart, scaleFactor: scaleFactor };
-                } else {
-                    return chart;
-                }
-            })
-        },
         deleteChartById: (state, action: PayloadAction<string>) => {
             let chartId = action.payload;
             deleteChartsRoutine(state, [chartId]);

src/app/utils.tsx

@@ -549,10 +549,21 @@ export const resolveChartFields = (chart: Chart, currentConcepts: FieldItem[], r
 }
 
 export let getTriggers = (leafTable: DictTable, tables: DictTable[]) => {
-    // recursively find triggers that ends in leafTable
+    // recursively find triggers that ends in leafTable (if the leaf table is anchored, we will find till the previous table is anchored)
     let triggers : Trigger[] = [];
     let t = leafTable;
-    while(t.derive != undefined) {
+    while(true) {
+
+        // this is when we find an original table
+        if (t.derive == undefined) {
+            break;
+        }
+
+        // this is when we find an anchored table (which is not the leaf table)
+        if (t !== leafTable && t.anchored) {
+            break;
+        }
+
         let trigger = t.derive.trigger as Trigger;
         triggers = [trigger, ...triggers];
         let parentTable = tables.find(x => x.id == trigger.tableId);

src/components/ComponentType.tsx

@@ -20,9 +20,10 @@ export interface FieldItem {
     type: Type;
     source: FieldSource;
     domain: any[];
+    tableRef: string; // which table it belongs to, it matters when it's an original field or a derived field
+
     transform?: ConceptTransformation;
-    tableRef?: string; // which table it comes from, it matters when it's an original field
-    temporary?: true;
+    temporary?: true; // the field is temporary, and it will be deleted unless it's saved
     levels?: {values: any[], reason: string}; // the order in which values in this field would be sorted
     semanticType?: string; // the semantic type of the object, inferred by the model
 }
@@ -56,6 +57,7 @@ export interface Trigger {
 
 export interface DictTable {
     id: string; // name/id of the table
+    displayId: string; // display id of the table 
     names: string[]; // column names
     types: Type[]; // column types
     rows: any[]; // table content, each entry is a row
@@ -69,22 +71,26 @@ export interface DictTable {
         // source specifies how the deriviation is done from the source tables, they may be the same, but not necessarily
         // in fact, right now dict tables are all triggered from charts
         trigger: Trigger,
-    }
+    };
+    anchored: boolean; // whether this table is anchored as a persistent table used to derive other tables
 }
 
 export function createDictTable(
     id: string, rows: any[], 
     derive: {code: string, codeExpl: string, source: string[], dialog: any[], 
-             trigger: Trigger} | undefined = undefined) : DictTable {
+             trigger: Trigger} | undefined = undefined,
+    anchored: boolean = false) : DictTable {
 
     let names = Object.keys(rows[0])
 
     return {
         id,
+        displayId: `${id}`,
         names, 
         rows,
         types: names.map(name => inferTypeFromValueArray(rows.map(r => r[name]))),
         derive,
+        anchored
     }
 }
 
@@ -94,7 +100,6 @@ export type Chart = {
     encodingMap: EncodingMap, 
     tableRef: string, 
     saved: boolean,
-    scaleFactor?: number,
     intermediate?: Trigger // whether this chart is only an intermediate chart (e.g., only used as a spec for transforming tables)
 }
 

src/data/utils.ts

@@ -18,7 +18,7 @@ export const loadTextDataWrapper = (title: string, text: string, fileType: strin
     if (fileType == "text/csv" || fileType == "text/tab-separated-values") {
         table = createTableFromText(tableName, text);
     } else if (fileType == "application/json") {
-        table = createTableFromFromObjectArray(tableName, JSON.parse(text));
+        table = createTableFromFromObjectArray(tableName, JSON.parse(text), true);
     } 
     return table;
 };
@@ -52,10 +52,10 @@ export const createTableFromText = (title: string, text: string): DictTable | un
             return row;
           });
 
-    return createTableFromFromObjectArray(title, values);
+    return createTableFromFromObjectArray(title, values, true);
 };
 
-export const createTableFromFromObjectArray = (title: string, values: any[], derive?: any): DictTable => {
+export const createTableFromFromObjectArray = (title: string, values: any[], anchored: boolean, derive?: any): DictTable => {
     const len = values.length;
     let names: string[] = [];
     let cleanNames: string[] = [];
@@ -95,10 +95,12 @@ export const createTableFromFromObjectArray = (title: string, values: any[], der
 
     return  {
         id: title,
+        displayId: `${title}`,
         names: columnTable.names(),
         types: columnTable.names().map(name => (columnTable.column(name) as Column).type),
         rows: columnTable.objects(),
-        derive: derive
+        derive: derive,
+        anchored: anchored
     }
 };
 
@@ -171,7 +173,7 @@ export const loadBinaryDataWrapper = (title: string, arrayBuffer: ArrayBuffer):
             const jsonData = XLSX.utils.sheet_to_json(worksheet);
 
             // Create a table from the JSON data with sheet name included in the title
-            const sheetTable = createTableFromFromObjectArray(`${title}-${sheetName}`, jsonData);
+            const sheetTable = createTableFromFromObjectArray(`${title}-${sheetName}`, jsonData, true);
             tables.push(sheetTable);
         }
 

src/scss/App.scss

@@ -91,4 +91,15 @@ h2.view-title {
 
 .Resizer.disabled:hover {
     border-color: transparent;
-}
+}
+
+.GroupHeader {
+    position: sticky;
+    padding: 4px 8px;
+    color: rgba(0, 0, 0, 0.6);
+    font-size: 12px;
+}
+
+.GroupItems {
+    padding: 0;
+}

src/scss/EncodingShelf.scss

@@ -23,7 +23,7 @@
 
 .encoding-shelf-trigger-card {
     margin: 6px 2px;
-    min-width: 160px;
+    min-width: 80px;
 }
 
 .encoding-shelf-compact {

src/scss/VisualizationView.scss

@@ -152,12 +152,6 @@ $accelerate-ease: cubic-bezier(0.4, 0.0, 1, 1);
         animation: appear 0.5s ease-out;
     }
 
-    .focused-vega-thumbnail {
-        //background-color: blue;
-        box-shadow: 0 0 5px rgba(33,33,33,.3);
-        z-index: 1;
-    }
-
     .vega-thumbnail:hover {
         transform: translateY(1px);
         box-shadow: 0 0 3px rgba(33,33,33,.2);