Merge pull request simwrapper#115 from simwrapper/109-implement-chrom…

…e-local-files

feat: Implement chrome local-files mode

package.json

@@ -47,6 +47,7 @@
     "fflate": "^0.7.2",
     "font-awesome": "^4.7.0",
     "geojson": "^0.5.0",
+    "idb-keyval": "^6.1.0",
     "javascript-natural-sort": "^0.7.1",
     "js-coroutines": "^2.4.5",
     "maplibre-gl": ">=1.14.0",

src/App.vue

@@ -32,8 +32,10 @@ const i18n = {
 import maplibregl from 'maplibre-gl'
 import Buefy from 'buefy'
 import { Vue, Component, Prop, Watch } from 'vue-property-decorator'
+import { get, set, clear } from 'idb-keyval'
 
 import globalStore from '@/store'
+import fileSystems, { addLocalFilesystem } from '@/fileSystemConfig'
 
 import { ColorScheme, MAPBOX_TOKEN, MAP_STYLES_OFFLINE } from '@/Globals'
 import LoginPanel from '@/components/LoginPanel.vue'
@@ -44,11 +46,13 @@ import LoginPanel from '@/components/LoginPanel.vue'
 const writableMapBox: any = maplibregl
 writableMapBox.accessToken = MAPBOX_TOKEN
 
+let doThisOnceForLocalFiles = true
+
 @Component({ i18n, components: { LoginPanel } })
 class App extends Vue {
   private state = globalStore.state
 
-  private mounted() {
+  private async mounted() {
     // theme
     const theme = localStorage.getItem('colorscheme')
       ? localStorage.getItem('colorscheme')
@@ -61,6 +65,22 @@ class App extends Vue {
 
     this.toggleFullScreen(true)
     this.setOnlineOrOfflineMode()
+
+    // local files
+    if (doThisOnceForLocalFiles) await this.setupLocalFiles()
+  }
+
+  // ------ Find Chrome Local File System roots ----
+  private async setupLocalFiles() {
+    console.log(12341235125)
+    if (globalStore.state.localFileHandles.length) return
+
+    const lfsh = (await get('fs')) as { key: string; handle: any }[]
+    if (lfsh && lfsh.length) {
+      for (const entry of lfsh) {
+        addLocalFilesystem(entry.handle, entry.key)
+      }
+    }
   }
 
   /**

src/Globals.ts

@@ -98,6 +98,7 @@ export interface FileSystemConfig {
   skipList?: string[]
   dashboardFolder?: string
   hidden?: boolean
+  handle?: FileSystemAPIHandle
 }
 
 export interface VisualizationPlugin {
@@ -111,6 +112,7 @@ export interface VisualizationPlugin {
 export interface DirectoryEntry {
   files: string[]
   dirs: string[]
+  handles: { [name: string]: FileSystemAPIHandle }
 }
 
 export enum ColorScheme {
@@ -202,6 +204,13 @@ export interface RunYaml {
   }[]
 }
 
+export interface FileSystemAPIHandle {
+  name: string
+  values: any // returns { kind: string; name: string; getFile: any }[]
+  getFile: any
+  requestPermission: any
+}
+
 export const LIGHT_MODE: ColorSet = {
   text: '#000',
   background: '#ccccc4',

src/components/FileSystemProjects.vue

@@ -23,7 +23,9 @@ export default class VueComponent extends Vue {
   private sources: FileSystemConfig[] = []
 
   private mounted() {
-    this.sources = globalStore.state.svnProjects.filter((source) => !source.hidden)
+    this.sources = globalStore.state.svnProjects.filter(
+      source => !source.hidden && !source.slug.startsWith('fs')
+    )
   }
 
   private openProjectPage(source: FileSystemConfig) {

src/fileSystemConfig.ts

@@ -1,10 +1,35 @@
-import { FileSystemConfig } from '@/Globals'
+import { get, set, clear } from 'idb-keyval'
+import { FileSystemConfig, FileSystemAPIHandle } from '@/Globals'
+import globalStore from '@/store'
 
 // The URL contains the websiteLiveHost, calculated at runtime
 const loc = window.location
 const webLiveHostname = loc.hostname
 const websiteLiveHost = `${loc.protocol}//${webLiveHostname}`
 
+export function addLocalFilesystem(handle: FileSystemAPIHandle, key: string | null) {
+  const slug = key || 'fs' + (1 + Object.keys(globalStore.state.localFileHandles).length)
+
+  const system: FileSystemConfig = {
+    name: handle.name,
+    slug: slug,
+    description: 'Local folder',
+    handle: handle,
+    baseURL: '',
+  }
+
+  fileSystems.unshift(system)
+
+  // commit to app state
+  globalStore.commit('addLocalFileSystem', { key: system.slug, handle: handle })
+  // console.log(globalStore.state.localFileHandles)
+
+  // write it out to indexed-db so we have it on next startup
+  console.log('WRITING ALL FILE HANDLES')
+  set('fs', globalStore.state.localFileHandles)
+  return system.slug
+}
+
 const fileSystems: FileSystemConfig[] = [
   {
     name: webLiveHostname + ' live folders',

src/js/HTTPFileSystem.ts

@@ -1,5 +1,5 @@
 import micromatch from 'micromatch'
-import { DirectoryEntry, FileSystemConfig, YamlConfigs } from '@/Globals'
+import { DirectoryEntry, FileSystemAPIHandle, FileSystemConfig, YamlConfigs } from '@/Globals'
 
 const YAML_FOLDER = 'simwrapper'
 
@@ -12,10 +12,12 @@ class SVNFileSystem {
   private baseUrl: string
   private urlId: string
   private needsAuth: boolean
+  private fsHandle: FileSystemAPIHandle | null
 
   constructor(project: FileSystemConfig) {
     this.urlId = project.slug
     this.needsAuth = !!project.needPassword
+    this.fsHandle = project.handle || null
 
     this.baseUrl = project.baseURL
     if (!project.baseURL.endsWith('/')) this.baseUrl += '/'
@@ -45,6 +47,14 @@ class SVNFileSystem {
   }
 
   private async _getFileResponse(scaryPath: string): Promise<Response> {
+    if (this.fsHandle) {
+      return this._getFileFromChromeFileSystem(scaryPath)
+    } else {
+      return this._getFileFetchResponse(scaryPath)
+    }
+  }
+
+  private async _getFileFetchResponse(scaryPath: string): Promise<Response> {
     const path = this.cleanURL(scaryPath)
 
     const headers: any = {}
@@ -66,6 +76,44 @@ class SVNFileSystem {
     return response
   }
 
+  private async _getFileFromChromeFileSystem(scaryPath: string): Promise<Response> {
+    // Chrome File System Access API doesn't handle nested paths, annoying.
+    // We need to first fetch the directory to get the file handle, and then
+    // get the file contents.
+
+    let path = scaryPath.replace(/^0-9a-zA-Z_\-\/:+/i, '')
+    path = path.replaceAll('//', '/')
+    path = new URL(`http://local/${path}`).href
+    path = path.substring(13)
+
+    const slash = path.lastIndexOf('/')
+    const folder = path.substring(0, slash)
+    const filename = path.substring(slash + 1)
+
+    const dirContents = await this.getDirectory(folder)
+    const fileHandle = dirContents.handles[filename]
+
+    const file = (await fileHandle.getFile()) as any
+
+    file.json = () => {
+      return new Promise(async (resolve, reject) => {
+        const text = await file.text()
+        const json = JSON.parse(text)
+        resolve(json)
+      })
+    }
+
+    file.blob = () => {
+      return new Promise(async (resolve, reject) => {
+        resolve(file)
+      })
+    }
+
+    return new Promise((resolve, reject) => {
+      resolve(file)
+    })
+  }
+
   async getFileText(scaryPath: string): Promise<string> {
     // This can throw lots of errors; we are not going to catch them
     // here so the code further up can deal with errors properly.
@@ -101,15 +149,70 @@ class SVNFileSystem {
 
     // Use cached version if we have it
     const cachedEntry = CACHE[this.urlId][stillScaryPath]
-    if (cachedEntry) {
-      return cachedEntry
-    }
+    if (cachedEntry) return cachedEntry
+
     // Generate and cache the listing
+    const dirEntry = this.fsHandle
+      ? await this.getDirectoryFromHandle(stillScaryPath)
+      : await this.getDirectoryFromURL(stillScaryPath)
+
+    CACHE[this.urlId][stillScaryPath] = dirEntry
+    return dirEntry
+  }
+
+  async getDirectoryFromHandle(stillScaryPath: string) {
+    // File System API has no concept of nested paths, which of course
+    // is how every filesystem from the past 60 years is actually laid out.
+    // Caching each level should lessen the pain of this weird workaround.
+
+    // Bounce thru each level until we reach this one.
+    // (Do this top down in order instead of recursive.)
+
+    // I want: /data/project/folder
+    // get / and find data
+    // get /data and find project
+    // get project and find folder
+    // get folder --> that's our answer
+
+    const contents: DirectoryEntry = { files: [], dirs: [], handles: {} }
+    if (!this.fsHandle) return contents
+
+    let parts = stillScaryPath.split('/').filter(p => !!p) // split and remove blanks
+
+    let currentDir = this.fsHandle as any
+
+    // iterate thru the tree, top-down:
+    if (parts.length) {
+      for (const subfolder of parts) {
+        console.log('searching for:', subfolder)
+        let found = false
+        for await (let [name, handle] of currentDir) {
+          if (name === subfolder) {
+            currentDir = handle
+            found = true
+            break
+          }
+        }
+        if (!found) throw Error('Could not find ' + subfolder)
+      }
+    }
+
+    // haven't crashed yet? Get the listing details!
+    for await (let entry of currentDir.values()) {
+      if (contents.handles) contents.handles[entry.name] = entry
+      if (entry.kind === 'file') {
+        contents.files.push(entry.name)
+      } else {
+        contents.dirs.push(entry.name)
+      }
+    }
+    return contents
+  }
+
+  async getDirectoryFromURL(stillScaryPath: string) {
     const response = await this._getFileResponse(stillScaryPath).then()
     const htmlListing = await response.text()
     const dirEntry = this.buildListFromHtml(htmlListing)
-    CACHE[this.urlId][stillScaryPath] = dirEntry
-
     return dirEntry
   }
 
@@ -183,7 +286,7 @@ class SVNFileSystem {
     if (data.indexOf('<ul>') > -1) return this.buildListFromSVN(data)
     if (data.indexOf('<table>') > -1) return this.buildListFromApache24(data)
 
-    return { dirs: [], files: [] }
+    return { dirs: [], files: [], handles: {} }
   }
 
   private buildListFromSimpleWebServer(data: string): DirectoryEntry {
@@ -204,7 +307,7 @@ class SVNFileSystem {
       if (name.endsWith('/')) dirs.push(name.substring(0, name.length - 1))
       else files.push(name)
     }
-    return { dirs, files }
+    return { dirs, files, handles: {} }
   }
 
   private buildListFromSVN(data: string): DirectoryEntry {
@@ -229,7 +332,7 @@ class SVNFileSystem {
       if (name.endsWith('/')) dirs.push(name.substring(0, name.length - 1))
       else files.push(name)
     }
-    return { dirs, files }
+    return { dirs, files, handles: {} }
   }
 
   private buildListFromApache24(data: string): DirectoryEntry {
@@ -259,7 +362,7 @@ class SVNFileSystem {
       if (name.endsWith('/')) dirs.push(name.substring(0, name.length - 1))
       else files.push(name)
     }
-    return { dirs, files }
+    return { dirs, files, handles: {} }
   }
 }