Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions hightable/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,23 @@
"typecheck": "tsc"
},
"dependencies": {
"hightable": "0.19.3",
"hightable": "0.19.5",
"react": "19.1.1",
"react-dom": "19.1.1",
"react-router": "7.8.2"
"react-router": "7.9.1"
},
"devDependencies": {
"@types/react": "19.1.12",
"@types/react": "19.1.13",
"@types/react-dom": "19.1.9",
"@vitejs/plugin-react": "5.0.2",
"@vitejs/plugin-react": "5.0.3",
"@vitest/coverage-v8": "3.2.4",
"eslint": "9.35.0",
"eslint-plugin-react": "7.37.5",
"eslint-plugin-react-hooks": "5.2.0",
"eslint-plugin-react-refresh": "0.4.20",
"globals": "16.4.0",
"typescript": "5.8.3",
"typescript-eslint": "8.43.0",
"typescript-eslint": "8.44.0",
"vite": "7.1.5",
"vitest": "3.2.4"
}
Expand Down
24 changes: 12 additions & 12 deletions hyparquet/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,26 +14,26 @@
"typecheck": "tsc"
},
"dependencies": {
"hightable": "0.18.4",
"hyparquet": "1.17.2",
"hightable": "0.19.5",
"hyparquet": "1.18.0",
"hyparquet-compressors": "1.1.1",
"hyperparam": "0.3.12",
"react": "18.3.1",
"react-dom": "18.3.1"
"hyperparam": "0.3.18",
"react": "19.1.1",
"react-dom": "19.1.1"
},
"devDependencies": {
"@types/react": "19.1.10",
"@types/react-dom": "19.1.7",
"@vitejs/plugin-react": "5.0.0",
"@types/react": "19.1.13",
"@types/react-dom": "19.1.9",
"@vitejs/plugin-react": "5.0.3",
"@vitest/coverage-v8": "3.2.4",
"eslint": "9.33.0",
"eslint": "9.35.0",
"eslint-plugin-react": "7.37.5",
"eslint-plugin-react-hooks": "5.2.0",
"eslint-plugin-react-refresh": "0.4.20",
"globals": "16.3.0",
"globals": "16.4.0",
"typescript": "5.8.3",
"typescript-eslint": "8.39.1",
"vite": "7.1.2",
"typescript-eslint": "8.44.0",
"vite": "7.1.5",
"vitest": "3.2.4"
}
}
3 changes: 2 additions & 1 deletion hyparquet/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { ReactNode } from 'react'
import Page, { PageProps } from './Page.js'
import Welcome from './Welcome.js'

import { sortableDataFrame } from 'hightable'
import { byteLengthFromUrl, parquetMetadataAsync } from 'hyparquet'
import { AsyncBufferFrom, asyncBufferFrom, parquetDataFrame } from 'hyperparam'
import { useCallback, useEffect, useState } from 'react'
Expand All @@ -22,7 +23,7 @@ export default function App(): ReactNode {
const setAsyncBuffer = useCallback(async function setAsyncBuffer(name: string, from: AsyncBufferFrom) {
const asyncBuffer = await asyncBufferFrom(from)
const metadata = await parquetMetadataAsync(asyncBuffer)
const df = parquetDataFrame(from, metadata)
const df = sortableDataFrame(parquetDataFrame(from, metadata))
setPageProps({ metadata, df, name, byteLength: from.byteLength, setError: setUnknownError })
}, [setUnknownError])

Expand Down
4 changes: 4 additions & 0 deletions hyparquet/src/index.css
Original file line number Diff line number Diff line change
Expand Up @@ -316,3 +316,7 @@ input[type="file"] {
background-color: #22222b;
padding-left: 20px;
}

[role="menu"] > button[role="menuitem"] {
color: #444;
}
22 changes: 11 additions & 11 deletions icebird/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,25 +14,25 @@
"typecheck": "tsc"
},
"dependencies": {
"hightable": "0.18.4",
"hyperparam": "0.3.12",
"hightable": "0.19.5",
"hyperparam": "0.3.18",
"icebird": "0.3.0",
"react": "18.3.1",
"react-dom": "18.3.1"
"react": "19.1.1",
"react-dom": "19.1.1"
},
"devDependencies": {
"@types/react": "19.1.10",
"@types/react-dom": "19.1.7",
"@vitejs/plugin-react": "5.0.0",
"@types/react": "19.1.13",
"@types/react-dom": "19.1.9",
"@vitejs/plugin-react": "5.0.3",
"@vitest/coverage-v8": "3.2.4",
"eslint": "9.33.0",
"eslint": "9.35.0",
"eslint-plugin-react": "7.37.5",
"eslint-plugin-react-hooks": "5.2.0",
"eslint-plugin-react-refresh": "0.4.20",
"globals": "16.3.0",
"globals": "16.4.0",
"typescript": "5.8.3",
"typescript-eslint": "8.39.1",
"vite": "7.1.2",
"typescript-eslint": "8.44.0",
"vite": "7.1.5",
"vitest": "3.2.4"
}
}
49 changes: 12 additions & 37 deletions icebird/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@ import { ReactNode } from 'react'
import Page, { PageProps } from './Page.js'
import Welcome from './Welcome.js'

import type { DataFrame, DataFrameEvents, ResolvedValue, UnsortableDataFrame } from 'hightable'
import { createEventTarget, sortableDataFrame } from 'hightable'
import type { DataFrame, DataFrameEvents, ResolvedValue } from 'hightable'
import { checkSignal, createEventTarget, sortableDataFrame, validateFetchParams, validateGetCellParams, validateGetRowNumberParams } from 'hightable'
import { icebergListVersions, icebergMetadata, icebergRead } from 'icebird'
import type { Snapshot, TableMetadata } from 'icebird/src/types.js'
import { useCallback, useEffect, useState } from 'react'
import Layout from './Layout.js'

const empty: DataFrame = {
header: [],
columnDescriptors: [],
numRows: 0,
eventTarget: createEventTarget<DataFrameEvents>(),
getRowNumber: () => undefined,
Expand Down Expand Up @@ -89,7 +89,7 @@ function icebergDataFrame(tableUrl: string, metadataFileName: string, metadata:
const currentSchemaId = metadata['current-schema-id']
const schema = metadata.schemas.find(s => s['schema-id'] === currentSchemaId)
if (!schema) throw new Error('Current schema not found in metadata')
const header = schema.fields.map(f => f.name)
const columnDescriptors = schema.fields.map(({ name }) => ({ name }))
const eventTarget = createEventTarget<DataFrameEvents>()

type CachedValue<T> = {
Expand All @@ -101,16 +101,15 @@ function icebergDataFrame(tableUrl: string, metadataFileName: string, metadata:

const rowNumberCache: CachedValue<number>[] = []
const cellCache = new Map<string, CachedValue<unknown>[]>()
header.forEach(column => cellCache.set(column, []))
columnDescriptors.forEach(({ name }) => cellCache.set(name, []))

function getRowNumber({ row }: {row: number}): ResolvedValue<number> | undefined {
validateRow({ row, data: { numRows } })
function getRowNumber({ row }: { row: number }): ResolvedValue<number> | undefined {
validateGetRowNumberParams({ row, data: { numRows, columnDescriptors } })
const cachedValue = rowNumberCache[row]
return cachedValue?.kind === 'fetched' ? cachedValue.value : undefined
}
function getCell({ row, column }: {row: number, column: string}): ResolvedValue<unknown> | undefined {
validateRow({ row, data: { numRows } })
validateColumn({ column, data: { header } })
function getCell({ row, column }: { row: number, column: string }): ResolvedValue<unknown> | undefined {
validateGetCellParams({ row, column, data: { numRows, columnDescriptors } })
const cachedValue = cellCache.get(column)?.[row]
return cachedValue?.kind === 'fetched' ? cachedValue.value : undefined
}
Expand All @@ -120,14 +119,14 @@ function icebergDataFrame(tableUrl: string, metadataFileName: string, metadata:

// TODO: fetch by row groups, to avoid fetching row by row when we scroll

const unsortableDataFrame: UnsortableDataFrame = {
header,
const unsortableDataFrame: DataFrame = {
columnDescriptors,
numRows,
eventTarget,
getRowNumber,
getCell,
async fetch({ rowStart, rowEnd, columns, signal }) {
validateFetchParams({ rowStart, rowEnd, columns, data: { numRows, header } })
validateFetchParams({ rowStart, rowEnd, columns, data: { numRows, columnDescriptors } })
checkSignal(signal)

const ranges = []
Expand Down Expand Up @@ -200,27 +199,3 @@ function icebergDataFrame(tableUrl: string, metadataFileName: string, metadata:
return sortableDataFrame(unsortableDataFrame)

}

function validateFetchParams({ rowStart, rowEnd, columns, data: { numRows, header } }: {rowStart: number, rowEnd: number, columns?: string[], data: Pick<DataFrame, 'numRows' | 'header'>}): void {
if (rowStart < 0 || rowEnd > numRows || !Number.isInteger(rowStart) || !Number.isInteger(rowEnd) || rowStart > rowEnd) {
throw new Error(`Invalid row range: ${rowStart} - ${rowEnd}, numRows: ${numRows}`)
}
if (columns?.some(column => !header.includes(column))) {
throw new Error(`Invalid columns: ${columns.join(', ')}. Available columns: ${header.join(', ')}`)
}
}
function validateRow({ row, data: { numRows } }: {row: number, data: Pick<DataFrame, 'numRows'>}): void {
if (row < 0 || row >= numRows || !Number.isInteger(row)) {
throw new Error(`Invalid row index: ${row}, numRows: ${numRows}`)
}
}
function validateColumn({ column, data: { header } }: {column: string, data: Pick<DataFrame, 'header'>}): void {
if (!header.includes(column)) {
throw new Error(`Invalid column: ${column}. Available columns: ${header.join(', ')}`)
}
}
function checkSignal(signal?: AbortSignal): void {
if (signal?.aborted) {
throw new DOMException('The operation was aborted.', 'AbortError')
}
}
4 changes: 4 additions & 0 deletions icebird/src/index.css
Original file line number Diff line number Diff line change
Expand Up @@ -257,3 +257,7 @@ main {
align-items: center;
gap: 8px;
}

[role="menu"] > button[role="menuitem"] {
color: #444;
}