Skip to content

Commit

Permalink
Misc
Browse files Browse the repository at this point in the history
  • Loading branch information
cmdcolin committed Nov 21, 2023
1 parent 026f1d3 commit cb9364a
Show file tree
Hide file tree
Showing 8 changed files with 224 additions and 127 deletions.
138 changes: 138 additions & 0 deletions plugins/data-management/src/ucsc-trackhub/doConnect.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
import { HubFile, SingleFileHub } from '@gmod/ucsc-hub'
import {
generateTracks,
fetchGenomesFile,
fetchTrackDbFile,
} from './ucscTrackHub'

import { getConf } from '@jbrowse/core/configuration'
import { FileLocation, getSession } from '@jbrowse/core/util'
import { openLocation } from '@jbrowse/core/util/io'
import { nanoid } from '@jbrowse/core/util/nanoid'

function resolve(uri: string, baseUri: string) {
return new URL(uri, baseUri).href
}

// eslint-disable-next-line @typescript-eslint/no-explicit-any
export async function doConnect(self: any) {
const session = getSession(self)
const notLoadedAssemblies = [] as string[]
try {
const hubFileLocation = getConf(self, 'hubTxtLocation') as FileLocation
const hubFileText = await openLocation(hubFileLocation).readFile('utf8')
const hubUri = resolve(hubFileLocation.uri, hubFileLocation.baseUri)
const { assemblyManager } = session
if (hubFileText.includes('useOneFile on')) {
const hub = new SingleFileHub(hubFileText)
const { genome, tracks } = hub
const genomeName = genome.name!

const asm = assemblyManager.get(genomeName)
if (!asm) {
console.log({ genome }, genome.data.twoBitPath)

Check warning on line 33 in plugins/data-management/src/ucsc-trackhub/doConnect.ts

View workflow job for this annotation

GitHub Actions / Lint on node 20 and ubuntu-latest

Unexpected console statement
// @ts-expect-error
session.addAssemblyConf({
name: genomeName,
sequence: {
type: 'ReferenceSequenceTrack',
trackId: `${genomeName}-${nanoid()}`,
adapter: {
type: 'TwoBitAdapter',
twoBitLocation: {
uri: resolve(genome.data.twoBitPath, hubUri),
},
chromSizesLocation: {
uri: resolve(genome.data.chromSizes, hubUri),
},
},
},
})
}
const asm2 = assemblyManager.get(genomeName)
const sequenceAdapter = getConf(asm2!, ['sequence', 'adapter'])
const tracksNew = generateTracks({
trackDb: tracks,
trackDbLoc: hubFileLocation,
assemblyName: genomeName,
sequenceAdapter,
})
self.addTrackConfs(tracksNew)
} else {
const hubFile = new HubFile(hubFileText)
const genomeFile = hubFile.data.genomesFile
if (!genomeFile) {
throw new Error('genomesFile not found on hub')
}

// @ts-expect-error
const hubUri = resolve(hubFileLocation.uri, hubFileLocation.baseUri)
const genomesFileLocation = hubUri
? {
uri: resolve(genomeFile, hubUri),
locationType: 'UriLocation' as const,
}
: {
localPath: genomeFile,
locationType: 'LocalPathLocation' as const,
}
const genomesFile = await fetchGenomesFile(genomesFileLocation)
const map = {} as Record<string, number>
for (const [genomeName, genome] of Object.entries(genomesFile.data)) {
const assemblyNames = getConf(self, 'assemblyNames')
if (assemblyNames.length > 0 && !assemblyNames.includes(genomeName)) {
continue
}

const asm = assemblyManager.get(genomeName)
if (!asm) {
notLoadedAssemblies.push(genomeName)
continue
}

// @ts-expect-error
const db = genome.data.trackDb
if (!db) {
throw new Error('genomesFile not found on hub')
}
const base = new URL(genomeFile, hubUri)
const loc = hubUri
? {
uri: new URL(db, base).href,
locationType: 'UriLocation' as const,
}
: {
localPath: db,
locationType: 'LocalPathLocation' as const,
}
const trackDb = await fetchTrackDbFile(loc)
const sequenceAdapter = getConf(asm, ['sequence', 'adapter'])
const tracks = generateTracks({
trackDb,
trackDbLoc: loc,
assemblyName: genomeName,
sequenceAdapter,
})
self.addTrackConfs(tracks)
map[genomeName] = tracks.length
}

const loadedAssemblies = Object.entries(map)
const str1 = loadedAssemblies.length
? `Loaded data from these assemblies: ${loadedAssemblies
.map(([key, val]) => `${key} (${val} tracks)`)
.join(', ')}`
: ''
const str2 = notLoadedAssemblies.length
? `Skipped data from these assemblies: ${notLoadedAssemblies.join(
', ',
)}`
: ''
session.notify([str1, str2].filter(f => !!f).join('. '), 'success')
}
} catch (e) {
console.error(e)
session.notify(`${getConf(self, 'name')}: "${e}"`, 'error')
session.breakConnection?.(self.configuration)
}
}
95 changes: 5 additions & 90 deletions plugins/data-management/src/ucsc-trackhub/model.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,11 @@
import PluginManager from '@jbrowse/core/PluginManager'
import { BaseConnectionModelFactory } from '@jbrowse/core/pluggableElementTypes/models'
import {
ConfigurationReference,
readConfObject,
getConf,
} from '@jbrowse/core/configuration'
import { getSession } from '@jbrowse/core/util'
import { ConfigurationReference } from '@jbrowse/core/configuration'
import { types } from 'mobx-state-tree'

// locals
import configSchema from './configSchema'
import { doConnect } from './doConnect'

export default function UCSCTrackHubConnection(pluginManager: PluginManager) {
return types
Expand All @@ -22,90 +18,9 @@ export default function UCSCTrackHubConnection(pluginManager: PluginManager) {
}),
)
.actions(self => ({
async connect() {
const session = getSession(self)
const notLoadedAssemblies = [] as string[]
try {
const hubFileLocation = getConf(self, 'hubTxtLocation')
const {
generateTracks,
fetchGenomesFile,
fetchTrackDbFile,
fetchHubFile,
} = await import('./ucscTrackHub')
const hubFile = await fetchHubFile(hubFileLocation)
const genomeFile = hubFile.data.genomesFile
if (!genomeFile) {
throw new Error('genomesFile not found on hub')
}

const hubUri = new URL(hubFileLocation.uri, hubFileLocation.baseUri)
const genomesFileLocation = hubUri
? {
uri: new URL(genomeFile, hubUri).href,
locationType: 'UriLocation' as const,
}
: {
localPath: genomeFile,
locationType: 'LocalPathLocation' as const,
}
const genomesFile = await fetchGenomesFile(genomesFileLocation)
const map = {} as Record<string, number>
for (const [genomeName, genome] of Object.entries(genomesFile)) {
const assemblyNames = getConf(self, 'assemblyNames')
if (
assemblyNames.length > 0 &&
!assemblyNames.includes(genomeName)
) {
continue
}

const conf = session.assemblyManager.get(genomeName)?.configuration
if (!conf) {
notLoadedAssemblies.push(genomeName)
continue
}
const db = genome.get('trackDb')
if (!db) {
throw new Error('genomesFile not found on hub')
}
const base = new URL(genomeFile, hubUri)
const loc = hubUri
? {
uri: new URL(db, base).href,
locationType: 'UriLocation' as const,
}
: {
localPath: db,
locationType: 'LocalPathLocation' as const,
}
const trackDb = await fetchTrackDbFile(loc)
const seqAdapter = readConfObject(conf, ['sequence', 'adapter'])
const tracks = generateTracks(trackDb, loc, genomeName, seqAdapter)
self.addTrackConfs(tracks)
map[genomeName] = tracks.length
}

const loadedAssemblies = Object.entries(map)
const str1 = loadedAssemblies.length
? `Loaded data from these assemblies: ${loadedAssemblies
.map(([key, val]) => `${key} (${val} tracks)`)
.join(', ')}`
: ''
const str2 = notLoadedAssemblies.length
? `Skipped data from these assemblies: ${notLoadedAssemblies.join(
', ',
)}`
: ''
session.notify([str1, str2].filter(f => !!f).join('. '), 'success')
} catch (e) {
console.error(e)
session.notify(
`There was a problem connecting to the UCSC Track Hub "${self.configuration.name}". Please make sure you have entered a valid hub.txt file. The error that was thrown is: "${e}"`,
'error',
)
session.breakConnection?.(self.configuration)
}
connect() {
// eslint-disable-next-line @typescript-eslint/no-floating-promises
doConnect(self)
},
}))
}
64 changes: 33 additions & 31 deletions plugins/data-management/src/ucsc-trackhub/ucscTrackHub.ts
Original file line number Diff line number Diff line change
@@ -1,37 +1,28 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { RaStanza, TrackDbFile } from '@gmod/ucsc-hub'
import { FileLocation, isUriLocation, objectHash } from '@jbrowse/core/util'
import { openLocation } from '@jbrowse/core/util/io'
import {
generateUnsupportedTrackConf,
generateUnknownTrackConf,
} from '@jbrowse/core/util/tracks'

export async function fetchHubFile(hubLocation: FileLocation) {
try {
const hubFileText = await openLocation(hubLocation).readFile('utf8')
const { HubFile } = await import('@gmod/ucsc-hub')
return new HubFile(hubFileText)
} catch (error) {
throw new Error(`Not a valid hub.txt file, got error: '${error}'`)
}
}
import { RaStanza, GenomesFile, TrackDbFile } from '@gmod/ucsc-hub'

export async function fetchGenomesFile(genomesLoc: FileLocation) {
const genomesFileText = await openLocation(genomesLoc).readFile('utf8')
const { GenomesFile } = await import('@gmod/ucsc-hub')
return new GenomesFile(genomesFileText)
}

export async function fetchTrackDbFile(trackDbLoc: FileLocation) {
const text = await openLocation(trackDbLoc).readFile('utf8')
const { TrackDbFile } = await import('@gmod/ucsc-hub')
return new TrackDbFile(text)
}

export function makeLoc(first: string, base: { uri: string }) {
export function makeLoc(
first: string,
base: { uri: string; baseUri?: string },
) {
return {
uri: new URL(first, base.uri).href,
uri: new URL(first, new URL(base.uri, base.baseUri)).href,
locationType: 'UriLocation',
}
}
Expand All @@ -52,12 +43,17 @@ export function makeLoc2(first: string, alt?: string) {
}
}

export function generateTracks(
trackDb: TrackDbFile,
trackDbLoc: FileLocation,
assemblyName: string,
sequenceAdapter: any,
) {
export function generateTracks({
trackDb,
trackDbLoc,
assemblyName,
sequenceAdapter,
}: {
trackDb: TrackDbFile
trackDbLoc: FileLocation
assemblyName: string
sequenceAdapter: any
}) {
const tracks: any = []

for (const [trackName, track] of Object.entries(trackDb.data)) {
Expand All @@ -84,13 +80,13 @@ export function generateTracks(
const categories = parentTracks
.map(p => p?.data.shortLabel)
.filter((f): f is string => !!f)
const res = makeTrackConfig(
track!,
const res = makeTrackConfig({
track: track!,

Check failure on line 84 in plugins/data-management/src/ucsc-trackhub/ucscTrackHub.ts

View workflow job for this annotation

GitHub Actions / Lint on node 20 and ubuntu-latest

This assertion is unnecessary since it does not change the type of the expression
categories,
trackDbLoc,
trackDb,
sequenceAdapter,
)
})
tracks.push({
...res,
trackId: `ucsc-trackhub-${objectHash(res)}`,
Expand All @@ -101,13 +97,19 @@ export function generateTracks(
return tracks
}

function makeTrackConfig(
track: RaStanza,
categories: string[],
trackDbLoc: FileLocation,
trackDb: TrackDbFile,
sequenceAdapter: any,
) {
function makeTrackConfig({
track,
categories,
trackDbLoc,
trackDb,
sequenceAdapter,
}: {
track: RaStanza
categories: string[]
trackDbLoc: FileLocation
trackDb: TrackDbFile
sequenceAdapter: any
}) {
const trackType =
track.data.type || trackDb.data[track.data.parent || '']?.data.type || ''
const name = track.data.shortLabel || ''
Expand Down
Binary file not shown.
Binary file added test_data/volvoxhub/bbi/volvox.bb
Binary file not shown.
13 changes: 7 additions & 6 deletions test_data/volvoxhub/config.json
Original file line number Diff line number Diff line change
Expand Up @@ -66,21 +66,22 @@
"locationType": "UriLocation"
}
},

{
"type": "UCSCTrackHubConnection",
"connectionId": "UCSCTrackHubConnection-volvox-hub3",
"name": "UCSCTrackHubConnection-volvox-hub3",
"connectionId": "UCSCTrackHubConnection-volvox-hub2",
"name": "UCSCTrackHubConnection-volvox-hub2",
"hubTxtLocation": {
"uri": "singlefile.hub.txt",
"uri": "hub2/hub.txt",
"locationType": "UriLocation"
}
},
{
"type": "UCSCTrackHubConnection",
"connectionId": "UCSCTrackHubConnection-volvox-hub2",
"name": "UCSCTrackHubConnection-volvox-hub2",
"connectionId": "UCSCTrackHubConnection-volvox-singlefilehub",
"name": "UCSCTrackHubConnection-volvox-singlefilehub",
"hubTxtLocation": {
"uri": "hub2/hub.txt",
"uri": "singlefile.volvox.hub.txt",
"locationType": "UriLocation"
}
},
Expand Down
7 changes: 7 additions & 0 deletions test_data/volvoxhub/html/volvox.description.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<!doctype html>
<html>
<body>
<p>Hello world!</p>
<p>Volvox single file trackhub</p>
</body>
</html>
Loading

0 comments on commit cb9364a

Please sign in to comment.