Make a comparative-adapters plugin that can be used outside of dotplo…

…t view

Add delta file support

Add delta track to yeast synteny

Add delta adapter support

plugins/comparative-adapters/.babelrc

@@ -0,0 +1,9 @@
+{
+  "presets": [
+    // need this to be able to use spread operator on Set and Map
+    // see https://github.com/formium/tsdx/issues/376#issuecomment-566750042
+    ["@babel/preset-env", { "loose": false }],
+    // can remove this if all .js files are converted to .ts
+    "@babel/preset-react"
+  ]
+}

plugins/comparative-adapters/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "@jbrowse/plugin-comparative-adapters",
+  "version": "1.6.4",
+  "description": "JBrowse 2 comparative adapters",
+  "keywords": [
+    "jbrowse",
+    "jbrowse2"
+  ],
+  "license": "Apache-2.0",
+  "homepage": "https://jbrowse.org",
+  "bugs": "https://github.com/GMOD/jbrowse-components/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/GMOD/jbrowse-components.git",
+    "directory": "plugins/comparative-adapters"
+  },
+  "author": "JBrowse Team",
+  "distMain": "dist/index.js",
+  "srcMain": "src/index.ts",
+  "main": "src/index.ts",
+  "distModule": "dist/plugin-comparative-adapters.esm.js",
+  "module": "",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "scripts": {
+    "start": "tsdx watch --verbose --noClean",
+    "build": "tsdx build",
+    "test": "cd ../..; jest plugins/comparative-adapters",
+    "prepublishOnly": "yarn test",
+    "prepack": "yarn build; yarn useDist",
+    "postpack": "yarn useSrc",
+    "useDist": "node ../../scripts/useDist.js",
+    "useSrc": "node ../../scripts/useSrc.js"
+  },
+  "dependencies": {
+    "@gmod/bgzf-filehandle": "^1.4.2",
+    "@material-ui/icons": "^4.9.1",
+    "abortable-promise-cache": "^1.1.3",
+    "clsx": "^1.0.0",
+    "generic-filehandle": "^2.2.2",
+    "json-stable-stringify": "^1.0.1",
+    "normalize-wheel": "^1.0.1",
+    "react-sizeme": "^3.0.2"
+  },
+  "peerDependencies": {
+    "@jbrowse/core": "^1.0.0",
+    "@jbrowse/plugin-alignments": "^1.0.0",
+    "@jbrowse/plugin-linear-genome-view": "^1.0.0",
+    "@material-ui/core": "^4.12.2",
+    "@material-ui/lab": "^4.0.0-alpha.45",
+    "mobx": "^5.0.0",
+    "mobx-react": "^6.0.0",
+    "mobx-state-tree": "3.14.1",
+    "prop-types": "^15.0.0",
+    "react": ">=16.8.0",
+    "react-dom": ">=16.8.0",
+    "rxjs": "^6.0.0"
+  },
+  "private": true
+}

plugins/comparative-adapters/src/DeltaAdapter/DeltaAdapter.ts

@@ -0,0 +1,286 @@
+import {
+  BaseFeatureDataAdapter,
+  BaseOptions,
+} from '@jbrowse/core/data_adapters/BaseAdapter'
+import { NoAssemblyRegion, Region } from '@jbrowse/core/util/types'
+import { doesIntersect2 } from '@jbrowse/core/util/range'
+import { openLocation } from '@jbrowse/core/util/io'
+import { ObservableCreate } from '@jbrowse/core/util/rxjs'
+import SimpleFeature, { Feature } from '@jbrowse/core/util/simpleFeature'
+import { readConfObject } from '@jbrowse/core/configuration'
+import { unzip } from '@gmod/bgzf-filehandle'
+
+interface PafRecord {
+  records: NoAssemblyRegion[]
+  extra: {
+    blockLen: number
+    mappingQual: number
+    numMatches: number
+    strand: number
+  }
+}
+
+function isGzip(buf: Buffer) {
+  return buf[0] === 31 && buf[1] === 139 && buf[2] === 8
+}
+
+/* paf2delta from paftools.js in the minimap2 repository, license reproduced below
+ *
+ * The MIT License
+ *
+ * Copyright (c) 2018-     Dana-Farber Cancer Institute
+ *               2017-2018 Broad Institute, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+function paf_delta2paf(lines: string[]) {
+  let rname = ''
+  let qname = ''
+  let rlen = 0
+  let qlen = 0
+  let qs = 0
+  let qe = 0
+  let rs = 0
+  let re = 0
+  let strand = 0
+  let NM = 0
+  let cigar = [] as any[]
+  let x = 0
+  let y = 0
+  let seen_gt = false
+
+  const records = []
+  const regex = new RegExp(/^>(\S+)\s+(\S+)\s+(\d+)\s+(\d+)/)
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]
+    const m = regex.exec(line)
+    if (m !== null) {
+      rname = m[1]
+      qname = m[2]
+      rlen = +m[3]
+      qlen = +m[4]
+      seen_gt = true
+      continue
+    }
+    if (!seen_gt) {
+      continue
+    }
+    const t = line.split(' ')
+    if (t.length === 7) {
+      const t0 = +t[0]
+      const t1 = +t[1]
+      const t2 = +t[2]
+      const t3 = +t[3]
+      const t4 = +t[4]
+      strand = (t0 < t1 && t2 < t3) || (t0 > t1 && t2 > t3) ? 1 : -1
+      rs = +(t0 < t1 ? t0 : t1) - 1
+      re = +(t1 > t0 ? t1 : t0)
+      qs = +(t2 < t3 ? t2 : t3) - 1
+      qe = +(t3 > t2 ? t3 : t2)
+      x = y = 0
+      NM = t4
+      cigar = []
+    } else if (t.length === 1) {
+      const d = +t[0]
+      if (d === 0) {
+        let blen = 0
+        const cigar_str = []
+
+        if (re - rs - x !== qe - qs - y) {
+          throw new Error(`inconsistent alignment on line ${i}`)
+        }
+        cigar.push((re - rs - x) << 4)
+        for (let i = 0; i < cigar.length; ++i) {
+          blen += cigar[i] >> 4
+          cigar_str.push((cigar[i] >> 4) + 'MID'.charAt(cigar[i] & 0xf))
+        }
+        records.push([
+          qname,
+          qlen,
+          qs,
+          qe,
+          strand > 0 ? '+' : '-',
+          rname,
+          rlen,
+          rs,
+          re,
+          blen - NM,
+          blen,
+          0,
+          'NM:i:' + NM,
+          'cg:Z:' + cigar_str.join(''),
+        ])
+      } else if (d > 0) {
+        const l = d - 1
+        x += l + 1
+        y += l
+        if (l > 0) {
+          cigar.push(l << 4)
+        }
+        if (cigar.length > 0 && (cigar[cigar.length - 1] & 0xf) === 2) {
+          cigar[cigar.length - 1] += 1 << 4
+        } else {
+          cigar.push((1 << 4) | 2)
+        } // deletion
+      } else {
+        const l = -d - 1
+        x += l
+        y += l + 1
+        if (l > 0) {
+          cigar.push(l << 4)
+        }
+        if (cigar.length > 0 && (cigar[cigar.length - 1] & 0xf) === 1) {
+          cigar[cigar.length - 1] += 1 << 4
+        } else {
+          cigar.push((1 << 4) | 1)
+        } // insertion
+      }
+    }
+  }
+  return records
+}
+
+export default class PAFAdapter extends BaseFeatureDataAdapter {
+  private setupP?: Promise<PafRecord[]>
+
+  public static capabilities = ['getFeatures', 'getRefNames']
+
+  async setup(opts?: BaseOptions) {
+    if (!this.setupP) {
+      this.setupP = this.setupPre(opts).catch(e => {
+        this.setupP = undefined
+        throw e
+      })
+    }
+    return this.setupP
+  }
+
+  async setupPre(opts?: BaseOptions) {
+    const deltaLocation = openLocation(
+      readConfObject(this.config, 'deltaLocation'),
+      this.pluginManager,
+    )
+    const buffer = (await deltaLocation.readFile(opts)) as Buffer
+    const buf = isGzip(buffer) ? await unzip(buffer) : buffer
+    // 512MB  max chrome string length is 512MB
+    if (buf.length > 536_870_888) {
+      throw new Error('Data exceeds maximum string length (512MB)')
+    }
+    const text = new TextDecoder('utf8', { fatal: true }).decode(buf)
+
+    const records = paf_delta2paf(text.split('\n').filter(line => !!line))
+
+    return records.map(record => {
+      const [
+        chr1,
+        ,
+        start1,
+        end1,
+        strand,
+        chr2,
+        ,
+        start2,
+        end2,
+        numMatches,
+        blockLen,
+        mappingQual,
+        ...fields
+      ] = record
+
+      const rest = Object.fromEntries(
+        fields.map(field => {
+          const r = field.indexOf(':')
+          const fieldName = field.slice(0, r)
+          const fieldValue = field.slice(r + 3)
+          return [fieldName, fieldValue]
+        }),
+      )
+
+      return {
+        records: [
+          { refName: chr1, start: +start1, end: +end1 },
+          { refName: chr2, start: +start2, end: +end2 },
+        ],
+        extra: {
+          numMatches: +numMatches,
+          blockLen: +blockLen,
+          strand: strand === '-' ? -1 : 1,
+          mappingQual: +mappingQual,
+          ...rest,
+        },
+      } as PafRecord
+    })
+  }
+
+  async hasDataForRefName() {
+    // determining this properly is basically a call to getFeatures
+    // so is not really that important, and has to be true or else
+    // getFeatures is never called (BaseAdapter filters it out)
+    return true
+  }
+
+  async getRefNames() {
+    // we cannot determine this accurately
+    return []
+  }
+
+  getFeatures(region: Region, opts: BaseOptions = {}) {
+    return ObservableCreate<Feature>(async observer => {
+      const pafRecords = await this.setup(opts)
+      const assemblyNames = readConfObject(this.config, 'assemblyNames')
+
+      // The index of the assembly name in the region list corresponds to
+      // the adapter in the subadapters list
+      const index = assemblyNames.indexOf(region.assemblyName)
+      if (index !== -1) {
+        for (let i = 0; i < pafRecords.length; i++) {
+          const { extra, records } = pafRecords[i]
+          const { start, end, refName } = records[index]
+          if (
+            refName === region.refName &&
+            doesIntersect2(region.start, region.end, start, end)
+          ) {
+            observer.next(
+              new SimpleFeature({
+                uniqueId: `row_${i}`,
+                start,
+                end,
+                refName,
+                syntenyId: i,
+                mate: {
+                  start: records[+!index].start,
+                  end: records[+!index].end,
+                  refName: records[+!index].refName,
+                },
+                ...extra,
+              }),
+            )
+          }
+        }
+      }
+
+      observer.complete()
+    })
+  }
+
+  freeResources(/* { region } */): void {}
+}

plugins/comparative-adapters/src/DeltaAdapter/configSchema.ts

@@ -0,0 +1,16 @@
+import { ConfigurationSchema } from '@jbrowse/core/configuration'
+
+export default ConfigurationSchema(
+  'DeltaAdapter',
+  {
+    assemblyNames: {
+      type: 'stringArray',
+      defaultValue: [],
+    },
+    deltaLocation: {
+      type: 'fileLocation',
+      defaultValue: { uri: '/path/to/file.delta', locationType: 'UriLocation' },
+    },
+  },
+  { explicitlyTyped: true },
+)

plugins/comparative-adapters/src/DeltaAdapter/index.ts

@@ -0,0 +1,22 @@
+import PluginManager from '@jbrowse/core/PluginManager'
+import AdapterType from '@jbrowse/core/pluggableElementTypes/AdapterType'
+
+import AdapterClass from './DeltaAdapter'
+import configSchema from './configSchema'
+
+export default (pluginManager: PluginManager) => {
+  pluginManager.addAdapterType(
+    () =>
+      new AdapterType({
+        name: 'DeltaAdapter',
+        configSchema,
+        adapterMetadata: {
+          category: null,
+          hiddenFromGUI: true,
+          displayName: null,
+          description: null,
+        },
+        AdapterClass,
+      }),
+  )
+}