remove the concept of filesystem, we don't need it

packages/components/demo/App.tsx

@@ -1,7 +1,7 @@
 import React from 'react'
 import { Page } from '../src/index.js'
 
-import { Source, createHttpFileSystem, getSource } from '../src/index.ts'
+import { getHttpSource } from '../src/index.ts'
 export interface Navigation {
   col?: number
   row?: number
@@ -15,10 +15,6 @@ function getNumberParam(search: URLSearchParams, key: string): number | undefine
   return number
 }
 
-const fileSystems = [
-  createHttpFileSystem(),
-]
-
 export default function App() {
   const search = new URLSearchParams(location.search)
   const url = search.get('url')
@@ -31,20 +27,13 @@ export default function App() {
   const row = getNumberParam(search, 'row')
   const col = getNumberParam(search, 'col')
 
-  let source: Source | undefined = undefined
-  for (const fileSystem of fileSystems) {
-    const fsSource = getSource(url, fileSystem)
-    if (fsSource){
-      source = fsSource
-      break
-    }
-  }
+  const source = getHttpSource(url)
 
   if (!source) {
     return <div>Could not load a data source. You have to pass a valid source in the url, eg: <a href={defaultUrl}>{defaultUrl}</a>.</div>
   }
   return <Page source={source} navigation={{ row, col }} config={{
-    slidePanel: { minWidth: 250, maxWidth: 750 },
+    slidePanel: { minWidth: 250 },
     routes: {
       getSourceRouteUrl: ({ sourceId }) => `/?url=${sourceId}`,
       getCellRouteUrl: ({ sourceId, col, row }) => `/?url=${sourceId}&col=${col}&row=${row}`,

packages/components/src/components/Breadcrumb.tsx

@@ -1,5 +1,5 @@
 import { RoutesConfig } from '../lib/routes.js'
-import { Source } from '../lib/source.js'
+import { Source } from '../lib/sources/types.js'
 
 export type BreadcrumbConfig = RoutesConfig
 interface BreadcrumbProps {

packages/components/src/components/Cell.tsx

@@ -1,7 +1,7 @@
 import { asyncRows } from 'hightable'
 import { asyncBufferFromUrl, parquetMetadataAsync } from 'hyparquet'
 import { useEffect, useState } from 'react'
-import { FileSource } from '../lib/source.js'
+import { FileSource } from '../lib/sources/types.js'
 import { parquetDataFrame } from '../lib/tableProvider.js'
 import Breadcrumb, { BreadcrumbConfig } from './Breadcrumb.js'
 import Layout from './Layout.js'

packages/components/src/components/File.tsx

@@ -1,5 +1,5 @@
 import { useState } from 'react'
-import { FileSource } from '../lib/source.js'
+import { FileSource } from '../lib/sources/types.js'
 import Breadcrumb, { BreadcrumbConfig } from './Breadcrumb.js'
 import Layout from './Layout.js'
 import Viewer, { ViewerConfig } from './viewers/Viewer.js'

packages/components/src/components/Folder.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useState } from 'react'
-import type { DirSource, FileMetadata } from '../lib/source.js'
+import type { DirSource, FileMetadata } from '../lib/sources/types.js'
 import { cn, formatFileSize, getFileDate, getFileDateShort } from '../lib/utils.js'
 import Breadcrumb, { BreadcrumbConfig } from './Breadcrumb.js'
 import Layout, { Spinner } from './Layout.js'

packages/components/src/components/Page.tsx

@@ -1,4 +1,4 @@
-import { Source } from '../lib/source.js'
+import { Source } from '../lib/sources/types.js'
 import { BreadcrumbConfig } from './Breadcrumb.js'
 import Cell from './Cell.js'
 import File, { FileConfig } from './File.js'

packages/components/src/components/index.ts

@@ -2,9 +2,9 @@ import Breadcrumb, { BreadcrumbConfig } from './Breadcrumb.js'
 import Cell, { CellConfig } from './Cell.js'
 import File, { FileConfig } from './File.js'
 import Folder, { FolderConfig } from './Folder.js'
-import Layout from './Layout.js'
+import Layout, { Spinner } from './Layout.js'
 import Markdown from './Markdown.js'
 import Page, { PageConfig } from './Page.js'
 export * from './viewers/index.js'
-export { Breadcrumb, Cell, File, Folder, Layout, Markdown, Page }
+export { Breadcrumb, Cell, File, Folder, Layout, Markdown, Page, Spinner }
 export type { BreadcrumbConfig, CellConfig, FileConfig, FolderConfig, PageConfig }

packages/components/src/components/viewers/ImageView.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useState } from 'react'
-import { FileSource } from '../../lib/source.js'
+import { FileSource } from '../../lib/sources/types.js'
 import { contentTypes, parseFileSize } from '../../lib/utils.js'
 import { Spinner } from '../Layout.js'
 import ContentHeader from './ContentHeader.js'

packages/components/src/components/viewers/MarkdownView.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useState } from 'react'
-import { FileSource } from '../../lib/source.js'
+import { FileSource } from '../../lib/sources/types.js'
 import { parseFileSize } from '../../lib/utils.js'
 import { Spinner } from '../Layout.js'
 import Markdown from '../Markdown.js'

packages/components/src/components/viewers/ParquetView.tsx

@@ -2,7 +2,7 @@ import HighTable, { DataFrame, rowCache } from 'hightable'
 import { asyncBufferFromUrl, parquetMetadataAsync } from 'hyparquet'
 import React, { useCallback, useEffect, useState } from 'react'
 import { RoutesConfig, appendSearchParams } from '../../lib/routes.js'
-import { FileSource } from '../../lib/source.js'
+import { FileSource } from '../../lib/sources/types.js'
 import { parquetDataFrame } from '../../lib/tableProvider.js'
 import { Spinner } from '../Layout.js'
 import CellPanel from './CellPanel.js'

packages/components/src/components/viewers/TextView.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useState } from 'react'
-import { FileSource } from '../../lib/source.js'
+import { FileSource } from '../../lib/sources/types.js'
 import { parseFileSize } from '../../lib/utils.js'
 import { Spinner } from '../Layout.js'
 import ContentHeader, { TextContent } from './ContentHeader.js'

packages/components/src/components/viewers/Viewer.tsx

@@ -1,4 +1,4 @@
-import { FileSource } from '../../lib/source.js'
+import { FileSource } from '../../lib/sources/types.js'
 import { imageTypes } from '../../lib/utils.js'
 import ImageView from './ImageView.js'
 import MarkdownView from './MarkdownView.js'

packages/components/src/lib/index.ts

@@ -1,9 +1,7 @@
-export { createHttpFileSystem, createHyperparamFileSystem, getSource } from './filesystem.js'
-export type { FileSystem } from './filesystem.js'
 export { appendSearchParams, replaceSearchParams } from './routes.js'
 export type { RoutesConfig } from './routes.js'
-export type { DirSource, FileKind, FileMetadata, FileSource, Source, SourcePart } from './source.js'
+export * from './sources/index.js'
 export { parquetDataFrame } from './tableProvider.js'
-export { asyncBufferFrom, cn, contentTypes, formatFileSize, getFileDate, getFileDateShort, getFileName, imageTypes, parseFileSize } from './utils.js'
+export { asyncBufferFrom, cn, contentTypes, formatFileSize, getFileDate, getFileDateShort, imageTypes, parseFileSize } from './utils.js'
 export { parquetQueryWorker } from './workers/parquetWorkerClient.js'
 export type { AsyncBufferFrom, Row } from './workers/types.js'

packages/components/src/lib/sources/httpSource.ts

@@ -0,0 +1,16 @@
+import { FileSource } from './types.js'
+import { getFileName } from './utils.js'
+
+export function getHttpSource(sourceId: string, options?: {requestInit?: RequestInit}): FileSource | undefined {
+  if (!URL.canParse(sourceId)) {
+    return undefined
+  }
+  return {
+    kind: 'file',
+    sourceId,
+    sourceParts: [{ text: sourceId, sourceId }],
+    fileName: getFileName(sourceId),
+    resolveUrl: sourceId,
+    requestInit: options?.requestInit,
+  }
+}

packages/components/src/lib/sources/hyperparamSource.ts

@@ -0,0 +1,93 @@
+import type { DirSource, FileKind, FileMetadata, FileSource, SourcePart } from './types.js'
+import { getFileName } from './utils.js'
+
+export interface HyperparamFileMetadata {
+  key: string
+  eTag?: string
+  fileSize?: number
+  lastModified: string
+}
+
+function canParse(sourceId: string): boolean {
+  /// we expect relative paths, such as path/to/file or path/to/dir/
+  /// let's just check that it is empty or starts with a "word" character
+  return sourceId === '' || /^[\w]/.test(sourceId)
+}
+
+function getSourceParts(sourceId: string): SourcePart[] {
+  const parts = sourceId.split('/')
+  return [
+    { 'text': '/', 'sourceId': '' },
+    ...parts.map((part, depth) => {
+      const slashSuffix = depth === parts.length - 1 ? '' : '/'
+      return {
+        text: part + slashSuffix,
+        sourceId: parts.slice(0, depth + 1).join('/') + slashSuffix,
+      }
+    }),
+  ]
+}
+
+function getKind(sourceId: string): FileKind {
+  return sourceId === '' || sourceId.endsWith('/') ? 'directory' : 'file'
+}
+
+function getResolveUrl(sourceId: string, { endpoint }: {endpoint: string}): string {
+  const url = new URL('/api/store/get', endpoint)
+  url.searchParams.append('key', sourceId)
+  return url.toString()
+}
+
+function getPrefix(sourceId: string): string {
+  return sourceId.replace(/\/$/, '')
+}
+
+async function listFiles(prefix: string, { endpoint, requestInit }: {endpoint: string, requestInit?: RequestInit}): Promise<FileMetadata[]> {
+  const url = new URL('/api/store/list', endpoint)
+  url.searchParams.append('prefix', prefix)
+  const res = await fetch(url, requestInit)
+  if (res.ok) {
+    const files = await res.json() as HyperparamFileMetadata[]
+    return files.map(file => ({
+      name: file.key,
+      eTag: file.eTag,
+      size: file.fileSize,
+      lastModified: file.lastModified,
+      sourceId: (prefix === '' ? '' : prefix + '/') + file.key,
+      kind: getKind(file.key),
+    }))
+  } else {
+    throw new Error(await res.text())
+  }
+}
+
+export function getHyperparamSource(sourceId: string, { endpoint, requestInit }: {endpoint: string, requestInit?: RequestInit}): FileSource | DirSource | undefined {
+  if (!URL.canParse(endpoint)) {
+    throw new Error('Invalid endpoint')
+  }
+  if (!canParse(sourceId)) {
+    return undefined
+  }
+  const sourceParts = getSourceParts(sourceId)
+  if (getKind(sourceId) === 'file') {
+    return {
+      kind: 'file',
+      sourceId,
+      sourceParts,
+      fileName: getFileName(sourceId),
+      resolveUrl: getResolveUrl(sourceId, { endpoint }),
+      requestInit,
+    }
+  } else {
+    const prefix = getPrefix(sourceId)
+    return {
+      kind: 'directory',
+      sourceId,
+      sourceParts,
+      prefix,
+      listFiles: () => listFiles(prefix, { endpoint, requestInit }),
+    }
+  }
+}
+
+

packages/components/src/lib/sources/index.ts

@@ -0,0 +1,5 @@
+export { getHttpSource } from './httpSource.js'
+export { getHyperparamSource } from './hyperparamSource.js'
+export type { HyperparamFileMetadata } from './hyperparamSource.js'
+export type { DirSource, FileKind, FileMetadata, FileSource, Source, SourcePart } from './types.js'
+export { getFileName } from './utils.js'

packages/components/src/lib/source.ts → packages/components/src/lib/sources/types.ts

@@ -33,4 +33,3 @@ export interface DirSource extends BaseSource {
 }
 
 export type Source = FileSource | DirSource
-

packages/components/src/lib/sources/utils.ts

@@ -0,0 +1,8 @@
+export function getFileName(source: string): string {
+  const fileName = source
+    .replace(/\?.*$/, '') // remove query string
+    .split('/')
+    .at(-1)
+  if (!fileName) throw new Error('Cannot extract a filename')
+  return fileName
+}