feat(data-fetcher,core): Add support for getting data from URLs that …

…require certain request initializations (#965)

* feat(core): pass fetch options into compiler 

* feat(data-fetch): make datafetchers use the fetch options 

* test(core): test that compiler can create higlass spec with the fetch options 
---------

Co-authored-by: Sehi L'Yi <sehilyi@gmail.com>

src/compiler/compile.test.ts

@@ -246,6 +246,107 @@ describe('Dummy track', () => {
     });
 });
 
+describe('Compiler with UrlToFetchOptions', () => {
+    it('passes UrlToFetchOptions to the data fetcher', () => {
+        const urlToFetchOptions = { 'https://my-csv-url.com': { headers: { Authentication: 'Bearer 1234' } } };
+        const spec: GoslingSpec = {
+            tracks: [
+                {
+                    id: 'track-id',
+                    data: {
+                        type: 'csv',
+                        url: 'https://my-csv-url.com'
+                    },
+                    mark: 'rect',
+                    width: 100,
+                    height: 100
+                }
+            ]
+        };
+        compile(
+            spec,
+            hgSpec => {
+                // @ts-ignore
+                expect(hgSpec.views[0].tracks.center[0].contents[0].data).toMatchInlineSnapshot(`
+                  {
+                    "assembly": "hg38",
+                    "indexUrlFetchOptions": {},
+                    "type": "csv",
+                    "url": "https://my-csv-url.com",
+                    "urlFetchOptions": {
+                      "headers": {
+                        "Authentication": "Bearer 1234",
+                      },
+                    },
+                    "x": undefined,
+                    "x1": undefined,
+                    "x1e": undefined,
+                    "xe": undefined,
+                  }
+                `);
+            },
+            [],
+            getTheme(),
+            {},
+            urlToFetchOptions
+        );
+    });
+
+    it('passes UrlToFetchOptions and IndexUrlFetchOptions to the data fetcher', () => {
+        const urlToFetchOptions = {
+            'https://file.gff': { headers: { Authentication: 'Bearer 1234' } },
+            'https://file.gff.tbi': { headers: { Authentication: 'Bearer 4321' } }
+        };
+        const spec: GoslingSpec = {
+            tracks: [
+                {
+                    id: 'track-id',
+                    data: {
+                        type: 'gff',
+                        url: 'https://file.gff',
+                        indexUrl: 'https://file.gff.tbi'
+                    },
+                    mark: 'rect',
+                    width: 100,
+                    height: 100
+                }
+            ]
+        };
+        compile(
+            spec,
+            hgSpec => {
+                // @ts-ignore
+                expect(hgSpec.views[0].tracks.center[0].contents[0].data).toMatchInlineSnapshot(`
+                  {
+                    "assembly": "hg38",
+                    "indexUrl": "https://file.gff.tbi",
+                    "indexUrlFetchOptions": {
+                      "headers": {
+                        "Authentication": "Bearer 4321",
+                      },
+                    },
+                    "type": "gff",
+                    "url": "https://file.gff",
+                    "urlFetchOptions": {
+                      "headers": {
+                        "Authentication": "Bearer 1234",
+                      },
+                    },
+                    "x": undefined,
+                    "x1": undefined,
+                    "x1e": undefined,
+                    "xe": undefined,
+                  }
+                `);
+            },
+            [],
+            getTheme(),
+            {},
+            urlToFetchOptions
+        );
+    });
+});
+
 describe('Maintain IDs', () => {
     it('Overlaid tracks', () => {
         const twoTracksWithDiffData: SingleView = {

src/compiler/compile.ts

@@ -4,6 +4,7 @@ import { traverseToFixSpecDownstream, overrideDataTemplates } from './spec-prepr
 import { replaceTrackTemplates } from '../core/utils/template';
 import { getRelativeTrackInfo, type Size } from './bounding-box';
 import type { CompleteThemeDeep } from '../core/utils/theme';
+import type { UrlToFetchOptions } from 'src/core/gosling-component';
 import { renderHiGlass as createHiGlassModels } from './create-higlass-models';
 import { manageResponsiveSpecs } from './responsive';
 import type { IdTable } from '../api/track-and-view-ids';
@@ -17,6 +18,7 @@ export type CompileCallback = (
     idTable: IdTable
 ) => void;
 
+
 export function compile(
     spec: GoslingSpec,
     callback: CompileCallback,
@@ -25,7 +27,8 @@ export function compile(
     containerStatus: {
         containerSize?: { width: number; height: number };
         containerParentSize?: { width: number; height: number };
-    }
+    },
+    urlToFetchOptions?: UrlToFetchOptions
 ) {
     // Make sure to keep the original spec as-is
     const specCopy = JSON.parse(JSON.stringify(spec));
@@ -68,5 +71,5 @@ export function compile(
     }
 
     // Make HiGlass models for individual tracks
-    createHiGlassModels(specCopy, trackInfos, callback, theme);
+    createHiGlassModels(specCopy, trackInfos, callback, theme, urlToFetchOptions);
 }

src/compiler/create-higlass-models.ts

@@ -12,6 +12,7 @@ import type {
 } from '@gosling-lang/gosling-schema';
 import type { CompleteThemeDeep } from '../core/utils/theme';
 import type { CompileCallback } from './compile';
+import type { UrlToFetchOptions } from 'src/core/gosling-component';
 import { getViewApiData } from '../api/api-data';
 import { GoslingToHiGlassIdMapper } from '../api/track-and-view-ids';
 import { IsDummyTrack } from '@gosling-lang/gosling-schema';
@@ -20,7 +21,8 @@ export function renderHiGlass(
     spec: GoslingSpec,
     trackInfos: TrackInfo[],
     callback: CompileCallback,
-    theme: CompleteThemeDeep
+    theme: CompleteThemeDeep,
+    urlToFetchOptions?: UrlToFetchOptions
 ) {
     if (trackInfos.length === 0) {
         // no tracks to render
@@ -36,7 +38,7 @@ export function renderHiGlass(
     /* Update the HiGlass model by iterating tracks */
     trackInfos.forEach(tb => {
         const { track, boundingBox: bb, layout } = tb;
-        goslingToHiGlass(hgModel, track, bb, layout, theme, idMapper);
+        goslingToHiGlass(hgModel, track, bb, layout, theme, idMapper, urlToFetchOptions);
     });
 
     /* Add linking information to the HiGlass model */

src/compiler/gosling-to-higlass.ts

@@ -20,6 +20,7 @@ import { DEWFAULT_TITLE_PADDING_ON_TOP_AND_BOTTOM } from './defaults';
 import type { CompleteThemeDeep } from '../core/utils/theme';
 import { DEFAULT_TEXT_STYLE } from '../core/utils/text-style';
 import type { GoslingToHiGlassIdMapper } from '../api/track-and-view-ids';
+import type { UrlToFetchOptions } from 'src/core/gosling-component';
 
 /**
  * Convert a gosling track into a HiGlass view and add it into a higlass model.
@@ -30,7 +31,8 @@ export function goslingToHiGlass(
     bb: BoundingBox,
     layout: RelativePosition,
     theme: Required<CompleteThemeDeep>,
-    idMapper: GoslingToHiGlassIdMapper
+    idMapper: GoslingToHiGlassIdMapper,
+    urlToFetchOptions?: UrlToFetchOptions
 ): HiGlassModel {
     // TODO: check whether there are multiple track.data across superposed tracks
     // ...
@@ -152,10 +154,19 @@ export function goslingToHiGlass(
                 x1: getFieldName('x1'),
                 x1e: getFieldName('x1e')
             } as const;
-            // use gosling's custom data fetchers
+
+            // Check whether there are any URL specific fetch options
+            const urlFetchOptions =
+                ('url' in firstResolvedSpec.data && urlToFetchOptions?.[firstResolvedSpec.data.url]) || {};
+            const indexUrlFetchOptions =
+                ('indexUrl' in firstResolvedSpec.data && urlToFetchOptions?.[firstResolvedSpec.data.indexUrl]) || {};
+
+            // This object will be passed to the data fetchers
             hgTrack.data = {
                 ...firstResolvedSpec.data,
                 ...xFields,
+                urlFetchOptions,
+                indexUrlFetchOptions,
                 // Additionally, add assembly, otherwise, a default genome build is used
                 assembly
                 // TODO: should look all sub tracks' `dataTransform` and apply OR operation.

src/core/gosling-component.tsx

@@ -1,6 +1,7 @@
 /* eslint-disable react/prop-types */
 import { type HiGlassApi, HiGlassComponentWrapper } from './higlass-component-wrapper';
 import type { TemplateTrackDef, VisUnitApiData } from '@gosling-lang/gosling-schema';
+import type { RequestInit } from '@gosling-lang/higlass-schema';
 import React, { useState, useEffect, useMemo, useRef, forwardRef, useCallback, useImperativeHandle } from 'react';
 import ResizeSensor from 'css-element-queries/src/ResizeSensor';
 import * as gosling from '..';
@@ -17,6 +18,10 @@ import type { IdTable } from '../api/track-and-view-ids';
 // If HiGlass is rendered and then the container resizes, the viewport position changes, unmatching `xDomain` specified by users.
 const DELAY_FOR_CONTAINER_RESIZE_BEFORE_RERENDER = 300;
 
+/** Matches URLs to specific fetch options so that datafetchers have access URL specific fetch options */
+export interface UrlToFetchOptions {
+    [url: string]: RequestInit;
+}
 interface GoslingCompProps {
     spec?: gosling.GoslingSpec;
     compiled?: (goslingSpec: gosling.GoslingSpec, higlassSpec: gosling.HiGlassSpec) => void;
@@ -27,6 +32,7 @@ interface GoslingCompProps {
     className?: string;
     theme?: Theme;
     templates?: TemplateTrackDef[];
+    urlToFetchOptions?: UrlToFetchOptions;
     experimental?: {
         reactive?: boolean;
     };
@@ -157,7 +163,8 @@ export const GoslingComponent = forwardRef<GoslingRef, GoslingCompProps>((props,
                 {
                     containerSize: wrapperSize.current,
                     containerParentSize: wrapperParentSize.current
-                }
+                },
+                props.urlToFetchOptions
             );
         }
     }, [props.spec, theme]);

src/core/gosling-embed.ts

@@ -6,6 +6,7 @@ import type { HiGlassSpec } from '@gosling-lang/higlass-schema';
 
 import { validateGoslingSpec } from '@gosling-lang/gosling-schema';
 import { compile } from '../compiler/compile';
+import type { UrlToFetchOptions } from './gosling-component';
 import { getTheme, type Theme } from './utils/theme';
 import { GoslingTemplates } from './utils/template';
 import { type GoslingApi, createApi } from '../api/api';
@@ -20,6 +21,7 @@ export type GoslingEmbedOptions = Omit<HiGlassComponentWrapperProps['options'],
     id?: string;
     className?: string;
     theme?: Theme;
+    urlToFetchOptions?: UrlToFetchOptions;
 };
 
 const MAX_TRIES = 20;
@@ -92,7 +94,8 @@ export function embed(element: HTMLElement, spec: GoslingSpec, opts: GoslingEmbe
             },
             [...GoslingTemplates],
             theme,
-            {} // TODO: properly specify this
+            {}, // TODO: properly specify this
+            opts.urlToFetchOptions
         );
     });
 }

src/data-fetchers/bam/bam-worker.ts

@@ -1,14 +1,23 @@
 // Adopted from https://github.com/higlass/higlass-pileup/blob/master/src/bam-fetcher-worker.js
 import { expose, Transfer } from 'threads/worker';
 import { BamFile as _BamFile } from '@gmod/bam';
+import type { BamRecord } from '@gmod/bam';
 import QuickLRU from 'quick-lru';
-
 import type { TilesetInfo } from '@higlass/types';
-import type { BamRecord } from '@gmod/bam';
 
-import { DataSource, RemoteFile } from '../utils';
 import type { ChromSizes } from '@gosling-lang/gosling-schema';
 
+import { DataSource, RemoteFile } from '../utils';
+
+interface BamFileOptions {
+    loadMates: boolean;
+    maxInsertSize: number;
+    extractJunction: boolean;
+    junctionMinCoverage: number;
+    urlFetchOptions?: RequestInit;
+    indexUrlFetchOptions?: RequestInit;
+}
+
 function parseMD(mdString: string, useCounts: true): { type: string; length: number }[];
 function parseMD(mdString: string, useCounts: false): { pos: number; base: string; length: 1; bamSeqShift: number }[];
 function parseMD(mdString: string, useCounts: boolean) {
@@ -127,14 +136,6 @@ function getSubstitutions(segment: Segment, seq: string) {
             } else {
                 // console.log('skipping:', sub.type);
             }
-            // if (referenceConsuming.has(sub.base)) {
-            //   if (queryConsuming.has(sub.base)) {
-            //     substitutions.push(
-            //     {
-            //       pos:
-            //     })
-            //   }
-            // }
         }
 
         const firstSub = cigarSubs[0];
@@ -235,27 +236,17 @@ class BamFile extends _BamFile {
         super(...args);
         this.headerPromise = this.getHeader();
     }
-    static fromUrl(url: string, indexUrl: string) {
+    static fromUrl(url: string, indexUrl: string, urlFetchOptions?: RequestInit, indexUrlFetchOptions?: RequestInit) {
         return new BamFile({
-            bamFilehandle: new RemoteFile(url),
-            baiFilehandle: new RemoteFile(indexUrl)
-            // fetchSizeLimit: 500000000,
-            // chunkSizeLimit: 100000000,
-            // yieldThreadTime: 1000,
+            bamFilehandle: new RemoteFile(url, { overrides: urlFetchOptions }),
+            baiFilehandle: new RemoteFile(indexUrl, { overrides: indexUrlFetchOptions })
         });
     }
     getChromNames() {
         return this.indexToChr.map((v: { refName: string; length: number }) => v.refName);
     }
 }
 
-interface BamFileOptions {
-    loadMates: boolean;
-    maxInsertSize: number;
-    extractJunction: boolean;
-    junctionMinCoverage: number;
-}
-
 // indexed by dataset uuid
 const dataSources: Map<string, DataSource<BamFile, BamFileOptions>> = new Map();
 // indexed by bam url
@@ -270,7 +261,7 @@ const init = async (
     options: Partial<BamFileOptions> = {}
 ) => {
     if (!bamFileCache.has(bam.url)) {
-        const bamFile = BamFile.fromUrl(bam.url, bam.indexUrl);
+        const bamFile = BamFile.fromUrl(bam.url, bam.indexUrl, options.urlFetchOptions, options.indexUrlFetchOptions);
         await bamFile.getHeader(); // reads bam/bai headers
 
         // Infer the correct chromosome names between 'chr1' and '1'