Support plaintext fasta

products/jbrowse-desktop/.gitignore

@@ -16,3 +16,4 @@ yarn-debug.log*
 yarn-error.log*
 coverage/
 public/electron.js
+public/generateFastaIndex.js

products/jbrowse-desktop/package.json

@@ -76,6 +76,7 @@
     "electron-is-dev": "^1.1.0",
     "electron-updater": "^4.3.9",
     "electron-window-state": "^5.0.3",
+    "follow-redirects": "^1.14.1",
     "fontsource-roboto": "3.0.3",
     "json-stable-stringify": "^1.0.1",
     "librpc-web-mod": "^1.1.2",

products/jbrowse-desktop/public/electron.ts

@@ -5,8 +5,10 @@ import fs from 'fs'
 import path from 'path'
 import url from 'url'
 import windowStateKeeper from 'electron-window-state'
-import { autoUpdater } from 'electron-updater'
 import fetch from 'node-fetch'
+import { getFileStream, generateFastaIndex } from './generateFastaIndex'
+
+import { autoUpdater } from 'electron-updater'
 
 const { unlink, readFile, copyFile, readdir, writeFile } = fs.promises
 
@@ -50,6 +52,7 @@ const userData = app.getPath('userData')
 const recentSessionsPath = path.join(userData, 'recent_sessions.json')
 const quickstartDir = path.join(userData, 'quickstart')
 const thumbnailDir = path.join(userData, 'thumbnails')
+const faiDir = path.join(userData, 'fai')
 const autosaveDir = path.join(userData, 'autosaved')
 const jbrowseDocDir = path.join(app.getPath('documents'), 'JBrowse')
 const defaultSavePath = path.join(jbrowseDocDir, 'untitled.jbrowse')
@@ -71,6 +74,10 @@ function getAutosavePath(sessionName: string, ext = 'json') {
   return path.join(autosaveDir, `${encodeURIComponent(sessionName)}.${ext}`)
 }
 
+function getFaiPath(name: string) {
+  return path.join(faiDir, `${encodeURIComponent(name)}.fai`)
+}
+
 if (!fs.existsSync(recentSessionsPath)) {
   fs.writeFileSync(recentSessionsPath, stringify([]), 'utf8')
 }
@@ -79,6 +86,10 @@ if (!fs.existsSync(quickstartDir)) {
   fs.mkdirSync(quickstartDir, { recursive: true })
 }
 
+if (!fs.existsSync(faiDir)) {
+  fs.mkdirSync(faiDir, { recursive: true })
+}
+
 if (!fs.existsSync(thumbnailDir)) {
   fs.mkdirSync(thumbnailDir, { recursive: true })
 }
@@ -102,22 +113,26 @@ interface SessionSnap {
 let mainWindow: electron.BrowserWindow | null
 
 async function createWindow() {
-  const response = await fetch('https://jbrowse.org/genomes/sessions.json')
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status} ${response.statusText}`)
-  }
-  const data = await response.json()
-  Object.entries(data).forEach(([key, value]) => {
-    // if there is not a 'gravestone' (.deleted file), then repopulate it on
-    // startup, this allows the user to delete even defaults if they want to
-    if (!fs.existsSync(getQuickstartPath(key) + '.deleted')) {
-      fs.writeFileSync(
-        getQuickstartPath(key),
-        JSON.stringify(value, null, 2),
-        'utf8',
-      )
+  try {
+    const response = await fetch('https://jbrowse.org/genomes/sessions.json')
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status} ${response.statusText}`)
     }
-  })
+    const data = await response.json()
+    Object.entries(data).forEach(([key, value]) => {
+      // if there is not a 'gravestone' (.deleted file), then repopulate it on
+      // startup, this allows the user to delete even defaults if they want to
+      if (!fs.existsSync(getQuickstartPath(key) + '.deleted')) {
+        fs.writeFileSync(
+          getQuickstartPath(key),
+          JSON.stringify(value, null, 2),
+          'utf8',
+        )
+      }
+    })
+  } catch (e) {
+    console.error('Failed to fetch sessions.json')
+  }
 
   const mainWindowState = windowStateKeeper({
     defaultWidth: 1400,
@@ -295,6 +310,18 @@ ipcMain.handle('quit', () => {
   app.quit()
 })
 
+ipcMain.handle(
+  'indexFasta',
+  async (event: unknown, location: { uri: string } | { localPath: string }) => {
+    const filename = 'localPath' in location ? location.localPath : location.uri
+    const faiPath = getFaiPath(path.basename(filename) + Date.now() + '.fai')
+    const stream = await getFileStream(location)
+    const faiData = await generateFastaIndex(stream)
+    await writeFile(faiPath, faiData, 'utf8')
+    return faiPath
+  },
+)
+
 ipcMain.handle(
   'listSessions',
   async (_event: unknown, showAutosaves: boolean) => {

products/jbrowse-desktop/public/generateFastaIndex.ts

@@ -0,0 +1,106 @@
+import readline from 'readline'
+import fs from 'fs'
+import { Readable } from 'stream'
+import { IncomingMessage } from 'http'
+import { http, https, FollowResponse } from 'follow-redirects'
+
+// Method for handing off the parsing of a gff3 file URL.
+// Calls the proper parser depending on if it is gzipped or not.
+// Returns a @gmod/gff stream.
+export async function createRemoteStream(urlIn: string) {
+  const newUrl = new URL(urlIn)
+  const fetcher = newUrl.protocol === 'https:' ? https : http
+
+  return new Promise<IncomingMessage & FollowResponse>((resolve, reject) =>
+    fetcher.get(urlIn, resolve).on('error', reject),
+  )
+}
+
+export async function getFileStream(
+  location: { uri: string } | { localPath: string },
+) {
+  let fileDataStream: Readable
+  // marked as unsed, could be used for progress bar though
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  let totalBytes = 0
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  let receivedBytes = 0
+  let filename: string
+
+  if ('localPath' in location) {
+    filename = location.localPath
+    totalBytes = fs.statSync(filename).size
+    fileDataStream = fs.createReadStream(filename)
+  } else if ('uri' in location) {
+    filename = location.uri
+    const temp = await createRemoteStream(filename)
+    totalBytes = +(temp.headers['content-length'] || 0)
+    fileDataStream = temp
+  } else {
+    throw new Error(`Unknown file handle type ${JSON.stringify(location)}`)
+  }
+
+  fileDataStream.on('data', chunk => {
+    receivedBytes += chunk.length
+  })
+  return fileDataStream
+}
+
+export async function generateFastaIndex(fileDataStream: Readable) {
+  const rl = readline.createInterface({
+    input: fileDataStream,
+  })
+
+  let refName: string | undefined
+  let currOffset = 0
+  let refSeqLen = 0
+  let lineBytes = 0
+  let lineBases = 0
+  let refOffset = currOffset
+  const entries = []
+  let possibleBadLine = undefined as [number, string] | undefined
+  let i = 0
+
+  for await (const line of rl) {
+    // assumes unix formatted files with only one '\n' newline
+    const currentLineBytes = line.length + 1
+    const currentLineBases = line.length
+    if (line[0] === '>') {
+      if (possibleBadLine && possibleBadLine[0] !== i - 1) {
+        throw new Error(possibleBadLine[1])
+      }
+      if (i > 0) {
+        entries.push(
+          `${refName}\t${refSeqLen}\t${refOffset}\t${lineBases}\t${lineBytes}\n`,
+        )
+      }
+      // reset
+      lineBytes = 0
+      refSeqLen = 0
+      lineBases = 0
+      refName = line.trim().slice(1)
+      currOffset += currentLineBytes
+      refOffset = currOffset
+    } else {
+      if (lineBases && currentLineBases !== lineBases) {
+        possibleBadLine = [
+          i,
+          `Not all lines in file have same width, please check your FASTA file line ${i}: ${lineBases} ${currentLineBases}`,
+        ] as [number, string]
+      }
+      lineBytes = currentLineBytes
+      lineBases = currentLineBases
+      currOffset += currentLineBytes
+      refSeqLen += currentLineBases
+    }
+
+    i++
+  }
+
+  if (i > 0) {
+    entries.push(
+      `${refName}\t${refSeqLen}\t${refOffset}\t${lineBases}\t${lineBytes}`,
+    )
+  }
+  return entries.join('\n')
+}