diff --git a/.eslintignore b/.eslintignore index 2f40aadf73..0df2d38d08 100644 --- a/.eslintignore +++ b/.eslintignore @@ -11,3 +11,4 @@ products/jbrowse-web/scripts/** /config/ /products/jbrowse-web/config/ products/jbrowse-desktop/public/electron.js +products/jbrowse-desktop/public/generateFastaIndex.js diff --git a/package.json b/package.json index 8013085c9f..68a030ddc4 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "@types/object.fromentries": "^2.0.0", "@types/pako": "^1.0.1", "@types/pluralize": "^0.0.29", + "@types/pump": "^1.1.1", "@types/range-parser": "^1.2.3", "@types/rbush": "^3.0.0", "@types/react": "^17.0.0", @@ -84,6 +85,7 @@ "@types/react-virtualized-auto-sizer": "^1.0.0", "@types/set-value": "^2.0.0", "@types/shortid": "^0.0.29", + "@types/split2": "^3.2.1", "@types/string-template": "^1.0.2", "@types/tmp": "^0.2.1", "@types/unzipper": "^0.10.3", diff --git a/products/jbrowse-desktop/.gitignore b/products/jbrowse-desktop/.gitignore index 6d19a10857..08c10fd6c0 100644 --- a/products/jbrowse-desktop/.gitignore +++ b/products/jbrowse-desktop/.gitignore @@ -16,3 +16,4 @@ yarn-debug.log* yarn-error.log* coverage/ public/electron.js +public/generateFastaIndex.js diff --git a/products/jbrowse-desktop/package.json b/products/jbrowse-desktop/package.json index 6ae9da9263..fdb887716b 100644 --- a/products/jbrowse-desktop/package.json +++ b/products/jbrowse-desktop/package.json @@ -76,6 +76,7 @@ "electron-is-dev": "^1.1.0", "electron-updater": "^4.3.9", "electron-window-state": "^5.0.3", + "follow-redirects": "^1.14.1", "fontsource-roboto": "3.0.3", "json-stable-stringify": "^1.0.1", "librpc-web-mod": "^1.1.2", @@ -84,12 +85,14 @@ "mobx-state-tree": "3.14.1", "node-fetch": "^2.6.0", "prop-types": "^15.7.2", + "pump": "^3.0.0", "raf": "^3.4.1", "react": "^17.0.0", "react-dom": "^17.0.0", "react-error-boundary": "^3.0.0", "react-sizeme": "^2.6.7", "rxjs": "^6.5.2", + "split2": "^4.1.0", "timeago.js": "^4.0.2", "use-query-params": "1.1.3" }, diff --git a/products/jbrowse-desktop/public/electron.ts b/products/jbrowse-desktop/public/electron.ts index 9a5a676c0d..95ddf84714 100644 --- a/products/jbrowse-desktop/public/electron.ts +++ b/products/jbrowse-desktop/public/electron.ts @@ -5,8 +5,10 @@ import fs from 'fs' import path from 'path' import url from 'url' import windowStateKeeper from 'electron-window-state' -import { autoUpdater } from 'electron-updater' import fetch from 'node-fetch' +import { getFileStream, generateFastaIndex } from './generateFastaIndex' + +import { autoUpdater } from 'electron-updater' const { unlink, readFile, copyFile, readdir, writeFile } = fs.promises @@ -50,6 +52,7 @@ const userData = app.getPath('userData') const recentSessionsPath = path.join(userData, 'recent_sessions.json') const quickstartDir = path.join(userData, 'quickstart') const thumbnailDir = path.join(userData, 'thumbnails') +const faiDir = path.join(userData, 'fai') const autosaveDir = path.join(userData, 'autosaved') const jbrowseDocDir = path.join(app.getPath('documents'), 'JBrowse') const defaultSavePath = path.join(jbrowseDocDir, 'untitled.jbrowse') @@ -71,6 +74,10 @@ function getAutosavePath(sessionName: string, ext = 'json') { return path.join(autosaveDir, `${encodeURIComponent(sessionName)}.${ext}`) } +function getFaiPath(name: string) { + return path.join(faiDir, `${encodeURIComponent(name)}.fai`) +} + if (!fs.existsSync(recentSessionsPath)) { fs.writeFileSync(recentSessionsPath, stringify([]), 'utf8') } @@ -79,6 +86,10 @@ if (!fs.existsSync(quickstartDir)) { fs.mkdirSync(quickstartDir, { recursive: true }) } +if (!fs.existsSync(faiDir)) { + fs.mkdirSync(faiDir, { recursive: true }) +} + if (!fs.existsSync(thumbnailDir)) { fs.mkdirSync(thumbnailDir, { recursive: true }) } @@ -301,6 +312,17 @@ ipcMain.handle('quit', () => { app.quit() }) +ipcMain.handle( + 'indexFasta', + async (event: unknown, location: { uri: string } | { localPath: string }) => { + const filename = 'localPath' in location ? location.localPath : location.uri + const faiPath = getFaiPath(path.basename(filename) + Date.now() + '.fai') + const stream = await getFileStream(location) + await generateFastaIndex(faiPath, stream) + return faiPath + }, +) + ipcMain.handle( 'listSessions', async (_event: unknown, showAutosaves: boolean) => { diff --git a/products/jbrowse-desktop/public/generateFastaIndex.ts b/products/jbrowse-desktop/public/generateFastaIndex.ts new file mode 100644 index 0000000000..fc4a8438f5 --- /dev/null +++ b/products/jbrowse-desktop/public/generateFastaIndex.ts @@ -0,0 +1,141 @@ +import fs from 'fs' +import split2 from 'split2' +import pump from 'pump' +import { Readable, Transform } from 'stream' +import { IncomingMessage } from 'http' +import { http, https, FollowResponse } from 'follow-redirects' + +export async function createRemoteStream(urlIn: string) { + const newUrl = new URL(urlIn) + const fetcher = newUrl.protocol === 'https:' ? https : http + + return new Promise((resolve, reject) => + fetcher.get(urlIn, resolve).on('error', reject), + ) +} + +export async function getFileStream( + location: { uri: string } | { localPath: string }, +) { + let fileDataStream: Readable + // marked as unsed, could be used for progress bar though + // eslint-disable-next-line @typescript-eslint/no-unused-vars + let totalBytes = 0 + // eslint-disable-next-line @typescript-eslint/no-unused-vars + let receivedBytes = 0 + let filename: string + + if ('localPath' in location) { + filename = location.localPath + totalBytes = fs.statSync(filename).size + fileDataStream = fs.createReadStream(filename) + } else if ('uri' in location) { + filename = location.uri + const temp = await createRemoteStream(filename) + totalBytes = +(temp.headers['content-length'] || 0) + fileDataStream = temp + } else { + throw new Error(`Unknown file handle type ${JSON.stringify(location)}`) + } + + fileDataStream.on('data', chunk => { + receivedBytes += chunk.length + }) + return fileDataStream +} + +// creates an FAI file from a FASTA file streaming in +class FastaIndexTransform extends Transform { + foundAny = false + possibleBadLine = undefined as [number, string] | undefined + refName: string | undefined + currOffset = 0 + refSeqLen = 0 + lineBytes = 0 + lineBases = 0 + refOffset = 0 + lineNum = 0 + + _transform(chunk: Buffer, encoding: unknown, done: (error?: Error) => void) { + const line = chunk.toString() + // line length in bytes including the \n that we split on + const currentLineBytes = chunk.length + 1 + // chop off \r if exists + const currentLineBases = line.trim().length + if (line[0] === '>') { + this.foundAny = true + if ( + this.possibleBadLine && + this.possibleBadLine[0] !== this.lineNum - 1 + ) { + done(new Error(this.possibleBadLine[1])) + return + } + if (this.lineNum > 0) { + this.push( + `${this.refName}\t${this.refSeqLen}\t${this.refOffset}\t${this.lineBases}\t${this.lineBytes}\n`, + ) + } + // reset + this.lineBytes = 0 + this.refSeqLen = 0 + this.lineBases = 0 + this.refName = line.trim().slice(1).split(/\s+/)[0] + this.currOffset += currentLineBytes + this.refOffset = this.currOffset + this.possibleBadLine = undefined + } else { + if (this.lineBases && currentLineBases !== this.lineBases) { + this.possibleBadLine = [ + this.lineNum, + `Not all lines in file have same width, please check your FASTA file line ${this.lineNum}`, + ] + } + this.lineBytes = currentLineBytes + this.lineBases = currentLineBases + this.currOffset += currentLineBytes + this.refSeqLen += currentLineBases + } + + this.lineNum++ + done() + } + + _flush(done: (error?: Error) => void) { + if (!this.foundAny) { + done( + new Error( + 'No sequences found in file. Ensure that this is a valid FASTA file', + ), + ) + } else { + if (this.lineNum > 0) { + this.push( + `${this.refName}\t${this.refSeqLen}\t${this.refOffset}\t${this.lineBases}\t${this.lineBytes}\n`, + ) + } + done() + } + } +} + +export async function generateFastaIndex( + faiPath: string, + fileDataStream: Readable, +) { + return new Promise((resolve, reject) => { + pump( + fileDataStream, + split2(/\n/), + new FastaIndexTransform(), + fs.createWriteStream(faiPath), + function (err) { + if (err) { + reject(err) + } else { + resolve('success') + } + }, + ) + }) +} diff --git a/products/jbrowse-desktop/src/OpenSequenceDialog.tsx b/products/jbrowse-desktop/src/OpenSequenceDialog.tsx index b16352ea01..9d96f46bcc 100644 --- a/products/jbrowse-desktop/src/OpenSequenceDialog.tsx +++ b/products/jbrowse-desktop/src/OpenSequenceDialog.tsx @@ -15,58 +15,20 @@ import { import FileSelector from '@jbrowse/core/ui/FileSelector' import ErrorMessage from '@jbrowse/core/ui/ErrorMessage' import { FileLocation } from '@jbrowse/core/util/types' +import { ipcRenderer } from 'electron' const useStyles = makeStyles(theme => ({ - root: { - flexGrow: 1, - overflow: 'hidden', - padding: theme.spacing(0, 3), + message: { + background: '#ddd', + margin: theme.spacing(2), + padding: theme.spacing(2), }, paper: { - margin: `${theme.spacing(1)}px auto`, padding: theme.spacing(2), - }, - createButton: { - marginTop: '1em', - justifyContent: 'center', - }, - paperContent: { - flex: 'auto', - margin: `${theme.spacing(1)}px auto`, - padding: theme.spacing(1), - overflow: 'auto', + margin: theme.spacing(2), }, })) -function AdapterSelector({ - adapterSelection, - setAdapterSelection, - adapterTypes, -}: { - adapterSelection: string - setAdapterSelection: Function - adapterTypes: string[] -}) { - return ( - { - setAdapterSelection(event.target.value) - }} - > - {adapterTypes.map(str => ( - - {str} - - ))} - - ) -} - function AdapterInput({ adapterSelection, fastaLocation, @@ -92,10 +54,7 @@ function AdapterInput({ setTwoBitLocation: Function setChromSizesLocation: Function }) { - if ( - adapterSelection === 'IndexedFastaAdapter' || - adapterSelection === 'BgzipFastaAdapter' - ) { + if (adapterSelection === 'IndexedFastaAdapter') { return ( @@ -112,15 +71,33 @@ function AdapterInput({ setLocation={loc => setFaiLocation(loc)} /> - {adapterSelection === 'BgzipFastaAdapter' ? ( - - setGziLocation(loc)} - /> - - ) : null} + + ) + } + if (adapterSelection === 'BgzipFastaAdapter') { + return ( + + + setFastaLocation(loc)} + /> + + + setFaiLocation(loc)} + /> + + + setGziLocation(loc)} + /> + ) } @@ -146,11 +123,29 @@ function AdapterInput({ ) } + if (adapterSelection === 'FastaAdapter') { + return ( + + + setFastaLocation(loc)} + /> + + + ) + } + return null } const blank = { uri: '' } as FileLocation +function isBlank(location: FileLocation) { + return 'uri' in location && location.uri === '' +} + const OpenSequenceDialog = ({ onClose, }: { @@ -161,11 +156,13 @@ const OpenSequenceDialog = ({ const adapterTypes = [ 'IndexedFastaAdapter', 'BgzipFastaAdapter', + 'FastaAdapter', 'TwoBitAdapter', ] const [error, setError] = useState() const [assemblyName, setAssemblyName] = useState('') const [assemblyDisplayName, setAssemblyDisplayName] = useState('') + const [loading, setLoading] = useState() const [adapterSelection, setAdapterSelection] = useState(adapterTypes[0]) const [fastaLocation, setFastaLocation] = useState(blank) const [faiLocation, setFaiLocation] = useState(blank) @@ -173,8 +170,26 @@ const OpenSequenceDialog = ({ const [twoBitLocation, setTwoBitLocation] = useState(blank) const [chromSizesLocation, setChromSizesLocation] = useState(blank) - function createAssemblyConfig() { + async function createAssemblyConfig() { + if (adapterSelection === 'FastaAdapter') { + setLoading('Creating .fai file for FASTA') + const faiLocation = await ipcRenderer.invoke('indexFasta', fastaLocation) + return { + name: assemblyName, + displayName: assemblyDisplayName, + sequence: { + adapter: { + type: 'IndexedFastaAdapter', + fastaLocation, + faiLocation: { localPath: faiLocation }, + }, + }, + } + } if (adapterSelection === 'IndexedFastaAdapter') { + if (isBlank(fastaLocation) || isBlank(faiLocation)) { + throw new Error('Need both fastaLocation and faiLocation') + } return { name: assemblyName, displayName: assemblyDisplayName, @@ -187,6 +202,15 @@ const OpenSequenceDialog = ({ }, } } else if (adapterSelection === 'BgzipFastaAdapter') { + if ( + isBlank(fastaLocation) || + isBlank(faiLocation) || + isBlank(gziLocation) + ) { + throw new Error( + 'Need both fastaLocation and faiLocation and gziLocation', + ) + } return { name: assemblyName, displayName: assemblyDisplayName, @@ -200,6 +224,9 @@ const OpenSequenceDialog = ({ }, } } else if (adapterSelection === 'TwoBitAdapter') { + if (isBlank(twoBitLocation)) { + throw new Error('Need twoBitLocation') + } return { name: assemblyName, displayName: assemblyDisplayName, @@ -218,53 +245,70 @@ const OpenSequenceDialog = ({ onClose()}> Open sequence + + Use this dialog to open a new indexed FASTA file, bgzipped+indexed + FASTA file, or .2bit file of a genome assembly or other sequence + + {loading ? ( + {loading} + ) : null} + {error ? : null} -
- - Use this dialog to open a new indexed FASTA file, bgzipped+indexed - FASTA file, or .2bit file of a genome assembly or other sequence - - - setAssemblyName(event.target.value)} - /> - setAssemblyDisplayName(event.target.value)} - /> - -
- -
- -
+ + + setAssemblyName(event.target.value)} + /> + + setAssemblyDisplayName(event.target.value)} + /> + + { + setAdapterSelection(event.target.value) + }} + > + {adapterTypes.map(str => ( + + {str} + + ))} + + + +