Support plaintext fasta on desktop by dynamically creating a FAI file on the fly

.eslintignore

@@ -11,3 +11,4 @@ products/jbrowse-web/scripts/**
 /config/
 /products/jbrowse-web/config/
 products/jbrowse-desktop/public/electron.js
+products/jbrowse-desktop/public/generateFastaIndex.js

package.json

@@ -75,6 +75,7 @@
     "@types/object.fromentries": "^2.0.0",
     "@types/pako": "^1.0.1",
     "@types/pluralize": "^0.0.29",
+    "@types/pump": "^1.1.1",
     "@types/range-parser": "^1.2.3",
     "@types/rbush": "^3.0.0",
     "@types/react": "^17.0.0",
@@ -84,6 +85,7 @@
     "@types/react-virtualized-auto-sizer": "^1.0.0",
     "@types/set-value": "^2.0.0",
     "@types/shortid": "^0.0.29",
+    "@types/split2": "^3.2.1",
     "@types/string-template": "^1.0.2",
     "@types/tmp": "^0.2.1",
     "@types/unzipper": "^0.10.3",

products/jbrowse-desktop/.gitignore

@@ -16,3 +16,4 @@ yarn-debug.log*
 yarn-error.log*
 coverage/
 public/electron.js
+public/generateFastaIndex.js

products/jbrowse-desktop/package.json

@@ -76,6 +76,7 @@
     "electron-is-dev": "^1.1.0",
     "electron-updater": "^4.3.9",
     "electron-window-state": "^5.0.3",
+    "follow-redirects": "^1.14.1",
     "fontsource-roboto": "3.0.3",
     "json-stable-stringify": "^1.0.1",
     "librpc-web-mod": "^1.1.2",
@@ -84,12 +85,14 @@
     "mobx-state-tree": "3.14.1",
     "node-fetch": "^2.6.0",
     "prop-types": "^15.7.2",
+    "pump": "^3.0.0",
     "raf": "^3.4.1",
     "react": "^17.0.0",
     "react-dom": "^17.0.0",
     "react-error-boundary": "^3.0.0",
     "react-sizeme": "^2.6.7",
     "rxjs": "^6.5.2",
+    "split2": "^4.1.0",
     "timeago.js": "^4.0.2",
     "use-query-params": "1.1.3"
   },

products/jbrowse-desktop/public/electron.ts

@@ -5,8 +5,10 @@ import fs from 'fs'
 import path from 'path'
 import url from 'url'
 import windowStateKeeper from 'electron-window-state'
-import { autoUpdater } from 'electron-updater'
 import fetch from 'node-fetch'
+import { getFileStream, generateFastaIndex } from './generateFastaIndex'
+
+import { autoUpdater } from 'electron-updater'
 
 const { unlink, readFile, copyFile, readdir, writeFile } = fs.promises
 
@@ -50,6 +52,7 @@ const userData = app.getPath('userData')
 const recentSessionsPath = path.join(userData, 'recent_sessions.json')
 const quickstartDir = path.join(userData, 'quickstart')
 const thumbnailDir = path.join(userData, 'thumbnails')
+const faiDir = path.join(userData, 'fai')
 const autosaveDir = path.join(userData, 'autosaved')
 const jbrowseDocDir = path.join(app.getPath('documents'), 'JBrowse')
 const defaultSavePath = path.join(jbrowseDocDir, 'untitled.jbrowse')
@@ -71,6 +74,10 @@ function getAutosavePath(sessionName: string, ext = 'json') {
   return path.join(autosaveDir, `${encodeURIComponent(sessionName)}.${ext}`)
 }
 
+function getFaiPath(name: string) {
+  return path.join(faiDir, `${encodeURIComponent(name)}.fai`)
+}
+
 if (!fs.existsSync(recentSessionsPath)) {
   fs.writeFileSync(recentSessionsPath, stringify([]), 'utf8')
 }
@@ -79,6 +86,10 @@ if (!fs.existsSync(quickstartDir)) {
   fs.mkdirSync(quickstartDir, { recursive: true })
 }
 
+if (!fs.existsSync(faiDir)) {
+  fs.mkdirSync(faiDir, { recursive: true })
+}
+
 if (!fs.existsSync(thumbnailDir)) {
   fs.mkdirSync(thumbnailDir, { recursive: true })
 }
@@ -301,6 +312,17 @@ ipcMain.handle('quit', () => {
   app.quit()
 })
 
+ipcMain.handle(
+  'indexFasta',
+  async (event: unknown, location: { uri: string } | { localPath: string }) => {
+    const filename = 'localPath' in location ? location.localPath : location.uri
+    const faiPath = getFaiPath(path.basename(filename) + Date.now() + '.fai')
+    const stream = await getFileStream(location)
+    await generateFastaIndex(faiPath, stream)
+    return faiPath
+  },
+)
+
 ipcMain.handle(
   'listSessions',
   async (_event: unknown, showAutosaves: boolean) => {

products/jbrowse-desktop/public/generateFastaIndex.ts

@@ -0,0 +1,141 @@
+import fs from 'fs'
+import split2 from 'split2'
+import pump from 'pump'
+import { Readable, Transform } from 'stream'
+import { IncomingMessage } from 'http'
+import { http, https, FollowResponse } from 'follow-redirects'
+
+export async function createRemoteStream(urlIn: string) {
+  const newUrl = new URL(urlIn)
+  const fetcher = newUrl.protocol === 'https:' ? https : http
+
+  return new Promise<IncomingMessage & FollowResponse>((resolve, reject) =>
+    fetcher.get(urlIn, resolve).on('error', reject),
+  )
+}
+
+export async function getFileStream(
+  location: { uri: string } | { localPath: string },
+) {
+  let fileDataStream: Readable
+  // marked as unsed, could be used for progress bar though
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  let totalBytes = 0
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  let receivedBytes = 0
+  let filename: string
+
+  if ('localPath' in location) {
+    filename = location.localPath
+    totalBytes = fs.statSync(filename).size
+    fileDataStream = fs.createReadStream(filename)
+  } else if ('uri' in location) {
+    filename = location.uri
+    const temp = await createRemoteStream(filename)
+    totalBytes = +(temp.headers['content-length'] || 0)
+    fileDataStream = temp
+  } else {
+    throw new Error(`Unknown file handle type ${JSON.stringify(location)}`)
+  }
+
+  fileDataStream.on('data', chunk => {
+    receivedBytes += chunk.length
+  })
+  return fileDataStream
+}
+
+// creates an FAI file from a FASTA file streaming in
+class FastaIndexTransform extends Transform {
+  foundAny = false
+  possibleBadLine = undefined as [number, string] | undefined
+  refName: string | undefined
+  currOffset = 0
+  refSeqLen = 0
+  lineBytes = 0
+  lineBases = 0
+  refOffset = 0
+  lineNum = 0
+
+  _transform(chunk: Buffer, encoding: unknown, done: (error?: Error) => void) {
+    const line = chunk.toString()
+    // line length in bytes including the \n that we split on
+    const currentLineBytes = chunk.length + 1
+    // chop off \r if exists
+    const currentLineBases = line.trim().length
+    if (line[0] === '>') {
+      this.foundAny = true
+      if (
+        this.possibleBadLine &&
+        this.possibleBadLine[0] !== this.lineNum - 1
+      ) {
+        done(new Error(this.possibleBadLine[1]))
+        return
+      }
+      if (this.lineNum > 0) {
+        this.push(
+          `${this.refName}\t${this.refSeqLen}\t${this.refOffset}\t${this.lineBases}\t${this.lineBytes}\n`,
+        )
+      }
+      // reset
+      this.lineBytes = 0
+      this.refSeqLen = 0
+      this.lineBases = 0
+      this.refName = line.trim().slice(1).split(/\s+/)[0]
+      this.currOffset += currentLineBytes
+      this.refOffset = this.currOffset
+      this.possibleBadLine = undefined
+    } else {
+      if (this.lineBases && currentLineBases !== this.lineBases) {
+        this.possibleBadLine = [
+          this.lineNum,
+          `Not all lines in file have same width, please check your FASTA file line ${this.lineNum}`,
+        ]
+      }
+      this.lineBytes = currentLineBytes
+      this.lineBases = currentLineBases
+      this.currOffset += currentLineBytes
+      this.refSeqLen += currentLineBases
+    }
+
+    this.lineNum++
+    done()
+  }
+
+  _flush(done: (error?: Error) => void) {
+    if (!this.foundAny) {
+      done(
+        new Error(
+          'No sequences found in file. Ensure that this is a valid FASTA file',
+        ),
+      )
+    } else {
+      if (this.lineNum > 0) {
+        this.push(
+          `${this.refName}\t${this.refSeqLen}\t${this.refOffset}\t${this.lineBases}\t${this.lineBytes}\n`,
+        )
+      }
+      done()
+    }
+  }
+}
+
+export async function generateFastaIndex(
+  faiPath: string,
+  fileDataStream: Readable,
+) {
+  return new Promise((resolve, reject) => {
+    pump(
+      fileDataStream,
+      split2(/\n/),
+      new FastaIndexTransform(),
+      fs.createWriteStream(faiPath),
+      function (err) {
+        if (err) {
+          reject(err)
+        } else {
+          resolve('success')
+        }
+      },
+    )
+  })
+}