Add .delta and .chain format adapters, fix ref name aliasing in synte…

…ny/dotplot views, and optimize very long CIGAR string in synteny view (#2746)

* Make a comparative-adapters plugin that can be used outside of dotplot view

Add delta file support

Add delta track to yeast synteny

Add delta adapter support

* Add delta support to linear synteny view import form also

* Use our feature type

* Add chain file adapter

* small refactor

* Refactor chain/delta to extend paf

* Move the refname renaming to the dotplot renderer itself

* Cleanups

* Use generics to type base class

* Fixup typescript

* Update snaps

* T1

* Move around

* Misc

* Test

* Add code to avoid drawing thousands of very tiny entries in cigar for e.g. hg19 vs hg38 chain file

* Fix lint

* Small renamings

* Small fixes

* Small cleanup

* clone dotplot importform to linear synteny import form

* Misc

* Misc

packages/core/pluggableElementTypes/renderers/ComparativeServerSideRendererType.ts

@@ -52,6 +52,11 @@ export default class ComparativeServerSideRenderer extends ServerSideRenderer {
    * @param args - the arguments passed to render
    * @returns the same object
    */
+
+  async renameRegionsIfNeeded(args: RenderArgs) {
+    return args
+  }
+
   serializeArgsInClient(args: RenderArgs) {
     const deserializedArgs = {
       ...args,

packages/core/ui/index.ts

@@ -2,6 +2,7 @@ export * from './theme'
 export { LogoFull, Logomark } from './Logo'
 export { default as App } from './App'
 export { default as ErrorMessage } from './ErrorMessage'
+export { default as AssemblySelector } from './AssemblySelector'
 export { default as FileSelector } from './FileSelector'
 export { default as PrerenderedCanvas } from './PrerenderedCanvas'
 export { default as ResizeHandle } from './ResizeHandle'

plugins/comparative-adapters/.babelrc

@@ -0,0 +1,9 @@
+{
+  "presets": [
+    // need this to be able to use spread operator on Set and Map
+    // see https://github.com/formium/tsdx/issues/376#issuecomment-566750042
+    ["@babel/preset-env", { "loose": false }],
+    // can remove this if all .js files are converted to .ts
+    "@babel/preset-react"
+  ]
+}

plugins/comparative-adapters/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@jbrowse/plugin-comparative-adapters",
+  "version": "1.6.5",
+  "description": "JBrowse 2 comparative adapters",
+  "keywords": [
+    "jbrowse",
+    "jbrowse2"
+  ],
+  "license": "Apache-2.0",
+  "homepage": "https://jbrowse.org",
+  "bugs": "https://github.com/GMOD/jbrowse-components/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/GMOD/jbrowse-components.git",
+    "directory": "plugins/comparative-adapters"
+  },
+  "author": "JBrowse Team",
+  "distMain": "dist/index.js",
+  "srcMain": "src/index.ts",
+  "main": "src/index.ts",
+  "distModule": "dist/plugin-comparative-adapters.esm.js",
+  "module": "",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "scripts": {
+    "start": "tsdx watch --verbose --noClean",
+    "build": "tsdx build",
+    "test": "cd ../..; jest plugins/comparative-adapters",
+    "prepublishOnly": "yarn test",
+    "prepack": "yarn build; yarn useDist",
+    "postpack": "yarn useSrc",
+    "useDist": "node ../../scripts/useDist.js",
+    "useSrc": "node ../../scripts/useSrc.js"
+  },
+  "dependencies": {
+    "@gmod/bgzf-filehandle": "^1.4.2"
+  },
+  "peerDependencies": {
+    "@jbrowse/core": "^1.0.0",
+    "@jbrowse/plugin-alignments": "^1.0.0",
+    "@jbrowse/plugin-linear-genome-view": "^1.0.0",
+    "@material-ui/core": "^4.12.2",
+    "@material-ui/lab": "^4.0.0-alpha.45",
+    "mobx": "^5.0.0",
+    "mobx-react": "^6.0.0",
+    "mobx-state-tree": "3.14.1",
+    "prop-types": "^15.0.0",
+    "react": ">=16.8.0",
+    "react-dom": ">=16.8.0",
+    "rxjs": "^6.0.0"
+  },
+  "private": true
+}

plugins/comparative-adapters/src/ChainAdapter/ChainAdapter.ts

@@ -0,0 +1,195 @@
+import { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter'
+import { NoAssemblyRegion } from '@jbrowse/core/util/types'
+import { openLocation } from '@jbrowse/core/util/io'
+import { readConfObject } from '@jbrowse/core/configuration'
+import { unzip } from '@gmod/bgzf-filehandle'
+import PAFAdapter from '../PAFAdapter/PAFAdapter'
+
+interface PafRecord {
+  records: NoAssemblyRegion[]
+  extra: {
+    blockLen: number
+    mappingQual: number
+    numMatches: number
+    strand: number
+  }
+}
+
+function isGzip(buf: Buffer) {
+  return buf[0] === 31 && buf[1] === 139 && buf[2] === 8
+}
+
+/* adapted from chain2paf by Andrea Guarracino, license reproduced below
+ *
+ * MIT License
+ *
+ * Copyright (c) 2021 Andrea Guarracino
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+function generate_record(
+  q_name: string,
+  q_start: number,
+  q_end: number,
+  q_strand: string,
+  t_name: string,
+  t_start: number,
+  t_end: number,
+  cigar: string,
+  num_matches: number,
+) {
+  return {
+    records: [
+      { refName: q_name, start: q_start, end: q_end },
+      { refName: t_name, start: t_start, end: t_end },
+    ],
+    extra: {
+      numMatches: num_matches,
+      blockLen: Math.max(q_end - q_start, t_end - t_start),
+      strand: q_strand === '-' ? -1 : 1,
+      mappingQual: 0,
+      cg: cigar,
+    },
+  } as PafRecord
+}
+
+function paf_chain2paf(lines: string[]) {
+  let t_name = ''
+  let t_start = 0
+  let t_end = 0
+  let q_name = ''
+  let q_size = ''
+  let q_strand = ''
+  let q_start = 0
+  let q_end = 0
+  let num_matches = 0
+  let cigar = ''
+  const records = []
+  for (let i = 0; i < lines.length; i++) {
+    const l = lines[i]
+    const l_tab = l.replace(/ /g, '\t') // There are CHAIN files with space-separated fields
+    const l_vec = l_tab.split('\t')
+
+    if (l_vec[0] === 'chain') {
+      // Emit previous PAF row, if available
+      if (cigar) {
+        records.push(
+          generate_record(
+            q_name,
+            q_start,
+            q_end,
+            q_strand,
+            t_name,
+            t_start,
+            t_end,
+            cigar,
+            num_matches,
+          ),
+        )
+      }
+
+      // Save query/target information
+      // score -- chain score
+      // tName -- chromosome (reference sequence)
+      // tSize -- chromosome size (reference sequence)
+      // tStrand -- strand (reference sequence)
+      // tStart -- alignment start position (reference sequence)
+      // tEnd -- alignment end position (reference sequence)
+      // qName -- chromosome (query sequence)
+      // qSize -- chromosome size (query sequence)
+      // qStrand -- strand (query sequence)
+      // qStart -- alignment start position (query sequence)
+      // qEnd -- alignment end position (query sequence)
+      // id -- chain ID
+      t_name = l_vec[2]
+      t_start = +l_vec[5]
+      t_end = +l_vec[6]
+      q_name = l_vec[7]
+      q_size = l_vec[8]
+      q_strand = l_vec[9]
+      q_start = +l_vec[10]
+      q_end = +l_vec[11]
+      if (q_strand === '-') {
+        const tmp = q_start
+        q_start = +q_size - q_end
+        q_end = +q_size - tmp
+      }
+
+      // Initialize PAF fields
+      num_matches = 0
+      cigar = ''
+    } else {
+      // size -- the size of the ungapped alignment
+      //
+      // dt -- the difference between the end of this block and the beginning
+      //    of the next block (reference sequence)
+      //
+      // dq -- the difference between the end of this block and the beginning
+      //    of the next block (query sequence)
+      const size_ungapped_alignment = +l_vec[0] || 0
+      const diff_in_target = l_vec.length > 1 ? +l_vec[1] : 0
+      const diff_in_query = l_vec.length > 2 ? +l_vec[2] : 0
+
+      if (size_ungapped_alignment !== 0) {
+        num_matches += +size_ungapped_alignment
+        cigar += size_ungapped_alignment + 'M'
+      }
+      if (diff_in_query !== 0) {
+        cigar += diff_in_query + 'I'
+      }
+      if (diff_in_target !== 0) {
+        cigar += diff_in_target + 'D'
+      }
+    }
+  }
+
+  // Emit last PAF row, if available
+  if (cigar) {
+    generate_record(
+      q_name,
+      q_start,
+      q_end,
+      q_strand,
+      t_name,
+      t_start,
+      t_end,
+      cigar,
+      num_matches,
+    )
+  }
+  return records
+}
+
+export default class ChainAdapter extends PAFAdapter {
+  async setupPre(opts?: BaseOptions) {
+    const chainLocation = openLocation(
+      readConfObject(this.config, 'chainLocation'),
+      this.pluginManager,
+    )
+    const buffer = (await chainLocation.readFile(opts)) as Buffer
+    const buf = isGzip(buffer) ? await unzip(buffer) : buffer
+    // 512MB  max chrome string length is 512MB
+    if (buf.length > 536_870_888) {
+      throw new Error('Data exceeds maximum string length (512MB)')
+    }
+    const text = new TextDecoder('utf8', { fatal: true }).decode(buf)
+    return paf_chain2paf(text.split('\n').filter(line => !!line))
+  }
+}

plugins/comparative-adapters/src/ChainAdapter/configSchema.ts

@@ -0,0 +1,16 @@
+import { ConfigurationSchema } from '@jbrowse/core/configuration'
+
+export default ConfigurationSchema(
+  'ChainAdapter',
+  {
+    assemblyNames: {
+      type: 'stringArray',
+      defaultValue: [],
+    },
+    chainLocation: {
+      type: 'fileLocation',
+      defaultValue: { uri: '/path/to/file.chain', locationType: 'UriLocation' },
+    },
+  },
+  { explicitlyTyped: true },
+)

plugins/comparative-adapters/src/ChainAdapter/index.ts

@@ -0,0 +1,22 @@
+import PluginManager from '@jbrowse/core/PluginManager'
+import AdapterType from '@jbrowse/core/pluggableElementTypes/AdapterType'
+
+import AdapterClass from './ChainAdapter'
+import configSchema from './configSchema'
+
+export default (pluginManager: PluginManager) => {
+  pluginManager.addAdapterType(
+    () =>
+      new AdapterType({
+        name: 'ChainAdapter',
+        configSchema,
+        adapterMetadata: {
+          category: null,
+          hiddenFromGUI: true,
+          displayName: null,
+          description: null,
+        },
+        AdapterClass,
+      }),
+  )
+}