diff --git a/packages/core/data_adapters/BaseAdapter.ts b/packages/core/data_adapters/BaseAdapter.ts index 86aa130c15..6a8508a5a9 100644 --- a/packages/core/data_adapters/BaseAdapter.ts +++ b/packages/core/data_adapters/BaseAdapter.ts @@ -1,5 +1,5 @@ import { Observable, merge } from 'rxjs' -import { takeUntil } from 'rxjs/operators' +import { filter, takeUntil, toArray } from 'rxjs/operators' import { isStateTreeNode, getSnapshot } from 'mobx-state-tree' import { ObservableCreate } from '../util/rxjs' import { checkAbortSignal, observeAbortSignal } from '../util' @@ -10,7 +10,12 @@ import { } from '../configuration/configurationSchema' import { getSubAdapterType } from './dataAdapterCache' import { Region, NoAssemblyRegion } from '../util/types' -import { blankStats, rectifyStats, scoresToStats } from '../util/stats' +import { + blankStats, + rectifyStats, + scoresToStats, + BaseFeatureStats, +} from '../util/stats' import BaseResult from '../TextSearch/BaseResults' import idMaker from '../util/idMaker' import PluginManager from '../PluginManager' @@ -71,6 +76,7 @@ export abstract class BaseAdapter { this.config = config this.getSubAdapter = getSubAdapter this.pluginManager = pluginManager + // note: we use switch on jest here for more simple feature IDs // in test environment if (typeof jest === 'undefined') { @@ -94,6 +100,7 @@ export abstract class BaseAdapter { * subclasses must implement. */ export abstract class BaseFeatureDataAdapter extends BaseAdapter { + private estimateStatsCache: BaseFeatureStats | undefined /** * Get all reference sequence names used in the data source * @@ -258,6 +265,88 @@ export abstract class BaseFeatureDataAdapter extends BaseAdapter { scoreSum, }) } + + public async getGlobalStats( + regionToStart: Region, + opts?: BaseOptions, + ): Promise { + // Estimates once, then cache stats for future calls + return this.estimateStatsCache + ? this.estimateStatsCache + : this.estimateGlobalStats(regionToStart, opts) + } + + public async estimateGlobalStats( + region: Region, + opts?: BaseOptions, + ): Promise { + const { statusCallback = () => {} } = opts || {} + const statsFromInterval = async (length: number, expansionTime: number) => { + const sampleCenter = region.start * 0.75 + region.end * 0.25 + const start = Math.max(0, Math.round(sampleCenter - length / 2)) + const end = Math.min(Math.round(sampleCenter + length / 2), region.end) + + const feats = this.getFeatures(region, opts) + let features + try { + features = await feats + .pipe( + filter( + (f: Feature) => + typeof f.get === 'function' && + f.get('start') >= start && + f.get('end') <= end, + ), + toArray(), + ) + .toPromise() + } catch (e) { + if (`${e}`.match(/HTTP 404/)) { + throw new Error(`${e}`) + } + console.warn('Skipping feature density calculation: ', e) + return { featureDensity: Infinity } + } + + // if no features in range or adapter has no f.get, cancel feature density calculation + if (features.length === 0) { + return { featureDensity: 0 } + } + const featureDensity = features.length / length + return maybeRecordStats( + length, + { + featureDensity: featureDensity, + }, + features.length, + expansionTime, + ) + } + + const maybeRecordStats = async ( + interval: number, + stats: BaseFeatureStats, + statsSampleFeatures: number, + expansionTime: number, + ): Promise => { + const refLen = region.end - region.start + if (statsSampleFeatures >= 300 || interval * 2 > refLen) { + statusCallback('') + this.estimateStatsCache = stats + } else if (expansionTime <= 4) { + expansionTime++ + return statsFromInterval(interval * 2, expansionTime) + } else { + statusCallback('') + console.error('Stats estimation reached timeout') + this.estimateStatsCache = { featureDensity: Infinity } + } + return this.estimateStatsCache as BaseFeatureStats + } + + statusCallback('Calculating stats') + return statsFromInterval(100, 0) + } } export interface RegionsAdapter extends BaseAdapter { diff --git a/packages/core/rpc/coreRpcMethods.ts b/packages/core/rpc/coreRpcMethods.ts index 7e5053b670..fd6ccff7f6 100644 --- a/packages/core/rpc/coreRpcMethods.ts +++ b/packages/core/rpc/coreRpcMethods.ts @@ -18,6 +18,7 @@ import { import { Region } from '../util/types' import { checkAbortSignal, renameRegionsIfNeeded } from '../util' import SimpleFeature, { SimpleFeatureSerialized } from '../util/simpleFeature' +import { BaseFeatureStats } from '../util/stats' export class CoreGetRefNames extends RpcMethodType { name = 'CoreGetRefNames' @@ -40,6 +41,7 @@ export class CoreGetRefNames extends RpcMethodType { sessionId, adapterConfig, ) + if (dataAdapter instanceof BaseFeatureDataAdapter) { return dataAdapter.getRefNames(deserializedArgs) } @@ -188,6 +190,55 @@ export interface RenderArgsSerialized extends ServerSideRenderArgsSerialized { rendererType: string } +export class CoreGetGlobalStats extends RpcMethodType { + name = 'CoreGetGlobalStats' + + async serializeArguments( + args: RenderArgs & { signal?: AbortSignal; statusCallback?: Function }, + rpcDriverClassName: string, + ) { + const assemblyManager = + this.pluginManager.rootModel?.session?.assemblyManager + if (!assemblyManager) { + return args + } + const renamedArgs = await renameRegionsIfNeeded(assemblyManager, { + ...args, + filters: args.filters && args.filters.toJSON().filters, + }) + + return super.serializeArguments(renamedArgs, rpcDriverClassName) + } + + async execute( + args: { + adapterConfig: {} + regions: Region[] + signal?: RemoteAbortSignal + headers?: Record + sessionId: string + }, + rpcDriverClassName: string, + ): Promise { + const deserializedArgs = await this.deserializeArguments( + args, + rpcDriverClassName, + ) + + const { adapterConfig, sessionId, regions } = deserializedArgs + const { dataAdapter } = await getAdapter( + this.pluginManager, + sessionId, + adapterConfig, + ) + + if (dataAdapter instanceof BaseFeatureDataAdapter) { + return dataAdapter.getGlobalStats(regions[0], deserializedArgs) + } + throw new Error('Data adapter not found') + } +} + /** * fetches features from an adapter and call a renderer with them */ diff --git a/packages/core/util/stats.ts b/packages/core/util/stats.ts index 7aa390e896..efb5bb4c07 100644 --- a/packages/core/util/stats.ts +++ b/packages/core/util/stats.ts @@ -11,10 +11,14 @@ export interface UnrectifiedFeatureStats { featureCount: number basesCovered: number } -export interface FeatureStats extends UnrectifiedFeatureStats { +export interface BaseFeatureStats { + featureDensity: number +} +export interface FeatureStats + extends UnrectifiedFeatureStats, + BaseFeatureStats { scoreMean: number scoreStdDev: number - featureDensity: number } /* diff --git a/plugins/alignments/src/BamAdapter/BamAdapter.ts b/plugins/alignments/src/BamAdapter/BamAdapter.ts index 53f1679137..1df9860c6e 100644 --- a/plugins/alignments/src/BamAdapter/BamAdapter.ts +++ b/plugins/alignments/src/BamAdapter/BamAdapter.ts @@ -8,6 +8,7 @@ import { checkAbortSignal } from '@jbrowse/core/util' import { openLocation } from '@jbrowse/core/util/io' import { ObservableCreate } from '@jbrowse/core/util/rxjs' import { Feature } from '@jbrowse/core/util/simpleFeature' +import { BaseFeatureStats } from '@jbrowse/core/util/stats' import { toArray } from 'rxjs/operators' import { readConfObject } from '@jbrowse/core/configuration' import BamSlightlyLazyFeature from './BamSlightlyLazyFeature' @@ -188,6 +189,20 @@ export default class BamAdapter extends BaseFeatureDataAdapter { }, signal) } + async estimateGlobalStats( + region: Region, + opts?: BaseOptions, + ): Promise { + const { bam } = await this.configure() + const featCount = await bam.lineCount(region.refName) + if (featCount < 0) { + return super.estimateGlobalStats(region, opts) + } + + const featureDensity = featCount / (region.end - region.start) + return { featureDensity } + } + freeResources(/* { region } */): void {} // depends on setup being called before the BAM constructor diff --git a/plugins/alignments/src/LinearAlignmentsDisplay/models/model.tsx b/plugins/alignments/src/LinearAlignmentsDisplay/models/model.tsx index 3245bae55d..7d1efa6b00 100644 --- a/plugins/alignments/src/LinearAlignmentsDisplay/models/model.tsx +++ b/plugins/alignments/src/LinearAlignmentsDisplay/models/model.tsx @@ -32,6 +32,7 @@ const stateModelFactory = ( height: 250, showCoverage: true, showPileup: true, + userFeatureScreenDensity: types.maybe(types.number), }), ) .volatile(() => ({ @@ -126,9 +127,9 @@ const stateModelFactory = ( height: self.snpCovHeight, } }, - setUserBpPerPxLimit(limit: number) { - self.PileupDisplay.setUserBpPerPxLimit(limit) - self.SNPCoverageDisplay.setUserBpPerPxLimit(limit) + setUserFeatureScreenDensity(limit: number) { + self.PileupDisplay.setUserFeatureScreenDensity(limit) + self.SNPCoverageDisplay.setUserFeatureScreenDensity(limit) }, setPileupDisplay(displayConfig: AnyConfigurationModel) { self.PileupDisplay = { diff --git a/plugins/alignments/src/LinearPileupDisplay/configSchema.ts b/plugins/alignments/src/LinearPileupDisplay/configSchema.ts index 28a5cc8cc4..a5a8bd878e 100644 --- a/plugins/alignments/src/LinearPileupDisplay/configSchema.ts +++ b/plugins/alignments/src/LinearPileupDisplay/configSchema.ts @@ -23,10 +23,10 @@ function PileupConfigFactory(pluginManager: PluginManager) { SvgFeatureRenderer: SvgFeatureRendererConfigSchema, }), renderer: '', - maxDisplayedBpPerPx: { + maxFeatureScreenDensity: { type: 'number', - description: 'maximum bpPerPx that is displayed in the view', - defaultValue: 100, + description: 'maximum features per pixel that is displayed in the view', + defaultValue: 15, }, colorScheme: { type: 'stringEnum', diff --git a/plugins/alignments/src/LinearPileupDisplay/model.ts b/plugins/alignments/src/LinearPileupDisplay/model.ts index 711ee9d4eb..a1a0454b14 100644 --- a/plugins/alignments/src/LinearPileupDisplay/model.ts +++ b/plugins/alignments/src/LinearPileupDisplay/model.ts @@ -416,6 +416,7 @@ const stateModelFactory = ( colorBy, rpcDriverName, } = self + return { ...superRenderProps(), notReady: !ready || (sortedBy && self.currBpPerPx !== view.bpPerPx), diff --git a/plugins/alignments/src/LinearSNPCoverageDisplay/models/configSchema.ts b/plugins/alignments/src/LinearSNPCoverageDisplay/models/configSchema.ts index 69004c6462..d5e19d2b9c 100644 --- a/plugins/alignments/src/LinearSNPCoverageDisplay/models/configSchema.ts +++ b/plugins/alignments/src/LinearSNPCoverageDisplay/models/configSchema.ts @@ -39,10 +39,10 @@ export default function SNPCoverageConfigFactory(pluginManager: PluginManager) { description: 'draw upside down', defaultValue: false, }, - maxDisplayedBpPerPx: { + maxFeatureScreenDensity: { type: 'number', - description: 'maximum bpPerPx that is displayed in the view', - defaultValue: 100, + description: 'maximum features per pixel that is displayed in the view', + defaultValue: 15, }, headroom: { type: 'number', diff --git a/plugins/alignments/src/SNPCoverageAdapter/SNPCoverageAdapter.ts b/plugins/alignments/src/SNPCoverageAdapter/SNPCoverageAdapter.ts index 1dadcd0bf7..573bd17770 100644 --- a/plugins/alignments/src/SNPCoverageAdapter/SNPCoverageAdapter.ts +++ b/plugins/alignments/src/SNPCoverageAdapter/SNPCoverageAdapter.ts @@ -16,6 +16,7 @@ import { getModificationPositions, Mismatch, } from '../BamAdapter/MismatchParser' +import { BaseFeatureStats } from '@jbrowse/core/util/stats' interface SNPCoverageOptions extends BaseOptions { filters?: SerializableFilterChain @@ -104,6 +105,14 @@ export default class SNPCoverageAdapter extends BaseFeatureDataAdapter { }, opts.signal) } + async estimateGlobalStats( + region: Region, + opts?: BaseOptions, + ): Promise { + const { subadapter } = await this.configure() + return subadapter.estimateGlobalStats(region, opts) + } + async getRefNames(opts: BaseOptions = {}) { const { subadapter } = await this.configure() return subadapter.getRefNames(opts) diff --git a/plugins/bed/src/BedTabixAdapter/BedTabixAdapter.ts b/plugins/bed/src/BedTabixAdapter/BedTabixAdapter.ts index 4a28d884a6..bc76534906 100644 --- a/plugins/bed/src/BedTabixAdapter/BedTabixAdapter.ts +++ b/plugins/bed/src/BedTabixAdapter/BedTabixAdapter.ts @@ -15,6 +15,7 @@ import { ucscProcessedTranscript } from '../util' import MyConfigSchema from './configSchema' import PluginManager from '@jbrowse/core/PluginManager' import { getSubAdapterType } from '@jbrowse/core/data_adapters/dataAdapterCache' +import { BaseFeatureStats } from '@jbrowse/core/util/stats' export default class BedTabixAdapter extends BaseFeatureDataAdapter { private parser: any @@ -154,5 +155,17 @@ export default class BedTabixAdapter extends BaseFeatureDataAdapter { }, opts.signal) } + async estimateGlobalStats( + region: Region, + opts?: BaseOptions, + ): Promise { + const featCount = await this.bed.lineCount(region.refName) + if (featCount === -1) { + return super.estimateGlobalStats(region, opts) + } + const featureDensity = featCount / (region.end - region.start) + return { featureDensity } + } + public freeResources(): void {} } diff --git a/plugins/config/src/ConfigurationEditorWidget/components/__snapshots__/ConfigurationEditor.test.js.snap b/plugins/config/src/ConfigurationEditorWidget/components/__snapshots__/ConfigurationEditor.test.js.snap index 1c01fd41fb..d44f381af8 100644 --- a/plugins/config/src/ConfigurationEditorWidget/components/__snapshots__/ConfigurationEditor.test.js.snap +++ b/plugins/config/src/ConfigurationEditorWidget/components/__snapshots__/ConfigurationEditor.test.js.snap @@ -761,7 +761,7 @@ exports[`ConfigurationEditor widget renders with defaults of the PileupTrack sch class="MuiFormLabel-root MuiInputLabel-root MuiInputLabel-formControl MuiInputLabel-animated MuiInputLabel-shrink MuiFormLabel-filled" data-shrink="true" > - maxDisplayedBpPerPx + maxFeatureScreenDensity

- maximum bpPerPx that is displayed in the view + maximum features per pixel that is displayed in the view

diff --git a/plugins/gff3/src/Gff3TabixAdapter/Gff3TabixAdapter.ts b/plugins/gff3/src/Gff3TabixAdapter/Gff3TabixAdapter.ts index b21780a9c1..80b4eb6b15 100644 --- a/plugins/gff3/src/Gff3TabixAdapter/Gff3TabixAdapter.ts +++ b/plugins/gff3/src/Gff3TabixAdapter/Gff3TabixAdapter.ts @@ -17,7 +17,9 @@ import { readConfObject } from '@jbrowse/core/configuration' import MyConfigSchema from './configSchema' import PluginManager from '@jbrowse/core/PluginManager' import { getSubAdapterType } from '@jbrowse/core/data_adapters/dataAdapterCache' +import { Region } from '@jbrowse/core/util/types' import { FeatureLoc } from '../util' +import { BaseFeatureStats } from '@jbrowse/core/util/stats' interface LineFeature { start: number @@ -245,5 +247,17 @@ export default class extends BaseFeatureDataAdapter { return f } + async estimateGlobalStats( + region: Region, + opts?: BaseOptions, + ): Promise { + const featCount = await this.gff.lineCount(region.refName) + if (featCount === -1) { + return super.estimateGlobalStats(region, opts) + } + const featureDensity = featCount / (region.end - region.start) + return { featureDensity } + } + public freeResources(/* { region } */) {} } diff --git a/plugins/linear-genome-view/src/BaseLinearDisplay/models/BaseLinearDisplayModel.tsx b/plugins/linear-genome-view/src/BaseLinearDisplay/models/BaseLinearDisplayModel.tsx index f2cb0810bc..33f92fc400 100644 --- a/plugins/linear-genome-view/src/BaseLinearDisplay/models/BaseLinearDisplayModel.tsx +++ b/plugins/linear-genome-view/src/BaseLinearDisplay/models/BaseLinearDisplayModel.tsx @@ -3,20 +3,25 @@ import { BaseDisplay } from '@jbrowse/core/pluggableElementTypes/models' import { getConf } from '@jbrowse/core/configuration' import { MenuItem } from '@jbrowse/core/ui' import { + isAbortException, getContainingView, getSession, isSelectionContainer, isSessionModelWithWidgets, } from '@jbrowse/core/util' +import { BaseFeatureStats } from '@jbrowse/core/util/stats' import { BaseBlock } from '@jbrowse/core/util/blockTypes' import { Region } from '@jbrowse/core/util/types' import CompositeMap from '@jbrowse/core/util/compositeMap' import { Feature, isFeature } from '@jbrowse/core/util/simpleFeature' -import { getParentRenderProps } from '@jbrowse/core/util/tracks' +import { + getParentRenderProps, + getRpcSessionId, +} from '@jbrowse/core/util/tracks' import Button from '@material-ui/core/Button' import Typography from '@material-ui/core/Typography' import MenuOpenIcon from '@material-ui/icons/MenuOpen' -import { autorun } from 'mobx' +import { autorun, observable } from 'mobx' import { addDisposer, Instance, isAlive, types } from 'mobx-state-tree' import React from 'react' import { Tooltip } from '../components/BaseLinearDisplay' @@ -24,6 +29,8 @@ import BlockState, { renderBlockData } from './serverSideRenderedBlock' import { LinearGenomeViewModel, ExportSvgOptions } from '../../LinearGenomeView' +type LGV = LinearGenomeViewModel + export interface Layout { minX: number minY: number @@ -49,7 +56,7 @@ export const BaseLinearDisplay = types defaultDisplayHeight, ), blockState: types.map(BlockState), - userBpPerPxLimit: types.maybe(types.number), + userFeatureScreenDensity: types.maybe(types.number), }), ) .volatile(() => ({ @@ -57,6 +64,10 @@ export const BaseLinearDisplay = types featureIdUnderMouse: undefined as undefined | string, contextMenuFeature: undefined as undefined | Feature, scrollTop: 0, + globalStats: observable({ + featureDensity: 0, + } as BaseFeatureStats), + statsStatus: 'none' as 'none' | 'loading' | 'loaded' | 'error', })) .views(self => ({ get blockType(): 'staticBlocks' | 'dynamicBlocks' { @@ -75,8 +86,12 @@ export const BaseLinearDisplay = types /** * set limit to config amount, or user amount if they force load, */ - get maxViewBpPerPx() { - return self.userBpPerPxLimit || getConf(self, 'maxDisplayedBpPerPx') + + get maxFeatureScreenDensity() { + return ( + self.userFeatureScreenDensity || + getConf(self, 'maxFeatureScreenDensity') + ) }, /** @@ -157,6 +172,13 @@ export const BaseLinearDisplay = types } }) .actions(self => ({ + // base display reload does nothing, see specialized displays for details + setMessage(message: string) { + self.message = message + }, + setStatsStatus(state: 'none' | 'loading' | 'loaded' | 'error') { + self.statsStatus = state + }, afterAttach() { // watch the parent's blocks to update our block state when they change const blockWatchDisposer = autorun(() => { @@ -181,6 +203,41 @@ export const BaseLinearDisplay = types addDisposer(self, blockWatchDisposer) }, + async getGlobalStats( + region: Region, + opts: { + headers?: Record + signal?: AbortSignal + filters?: string[] + }, + ): Promise { + const { rpcManager } = getSession(self) + const { adapterConfig } = self + const sessionId = getRpcSessionId(self) + + const params = { + sessionId, + regions: [region], + adapterConfig, + statusCallback: (message: string) => { + if (isAlive(self)) { + this.setMessage(message) + } + }, + ...opts, + } + + this.setStatsStatus('loading') + return rpcManager.call( + sessionId, + 'CoreGetGlobalStats', + params, + ) as Promise + }, + updateGlobalStats(stats: BaseFeatureStats) { + self.globalStats.featureDensity = stats.featureDensity + this.setStatsStatus('loaded') + }, setHeight(displayHeight: number) { if (displayHeight > minDisplayHeight) { self.height = displayHeight @@ -197,13 +254,8 @@ export const BaseLinearDisplay = types setScrollTop(scrollTop: number) { self.scrollTop = scrollTop }, - // sets the new bpPerPxLimit if user chooses to force load - setUserBpPerPxLimit(limit: number) { - self.userBpPerPxLimit = limit - }, - // base display reload does nothing, see specialized displays for details - setMessage(message: string) { - self.message = message + setUserFeatureScreenDensity(limit: number) { + self.userFeatureScreenDensity = limit }, addBlock(key: string, block: BaseBlock) { self.blockState.set( @@ -245,11 +297,99 @@ export const BaseLinearDisplay = types self.contextMenuFeature = feature }, })) + .actions(self => { + const { reload: superReload } = self + + return { + async reload() { + self.setError() + const aborter = new AbortController() + const view = getContainingView(self) as LGV + if (!view.initialized) { + return + } + + let stats + if (view.staticBlocks.contentBlocks[0]) { + try { + stats = await self.getGlobalStats( + view.staticBlocks.contentBlocks[0], + { signal: aborter.signal }, + ) + + if (isAlive(self)) { + self.updateGlobalStats(stats) + superReload() + } else { + return + } + } catch (e) { + self.setError(e) + self.setStatsStatus('error') + } + } + }, + afterAttach() { + addDisposer( + self, + autorun( + async () => { + try { + const aborter = new AbortController() + const view = getContainingView(self) as LGV + const currentFeatureScreenDensity = + self.globalStats?.featureDensity * view?.bpPerPx + + if (!view.initialized) { + return + } + + if ( + view && + self.globalStats && + currentFeatureScreenDensity > self.maxFeatureScreenDensity + ) { + return + } + + if (view.staticBlocks.contentBlocks[0]) { + const stats = await self.getGlobalStats( + view.staticBlocks.contentBlocks[0], + { + signal: aborter.signal, + }, + ) + + if (isAlive(self)) { + self.updateGlobalStats(stats) + } else { + return + } + } + } catch (e) { + if (!isAbortException(e) && isAlive(self)) { + console.error(e) + self.setError(e) + self.setStatsStatus('error') + } + } + }, + { delay: 1000 }, + ), + ) + }, + } + }) .views(self => ({ regionCannotBeRenderedText(_region: Region) { const view = getContainingView(self) as LinearGenomeViewModel - if (view && view.bpPerPx > self.maxViewBpPerPx) { - return 'Zoom in to see features' + const currentFeatureScreenDensity = + self.globalStats?.featureDensity * view?.bpPerPx + if (self.statsStatus === 'error') { + return 'Force load to see features' + } + if (view && currentFeatureScreenDensity > self.maxFeatureScreenDensity) { + return 'Force load to see features' } return '' }, @@ -263,17 +403,25 @@ export const BaseLinearDisplay = types */ regionCannotBeRendered(_region: Region) { const view = getContainingView(self) as LinearGenomeViewModel - if (view && view.bpPerPx > self.maxViewBpPerPx) { + + const currentFeatureScreenDensity = + self.globalStats?.featureDensity * view?.bpPerPx + if ( + view && + self.globalStats.featureDensity !== undefined && + currentFeatureScreenDensity > self.maxFeatureScreenDensity + ) { return ( <> Zoom in to see features or{' '}