fix(links): Link widths could show wrong data if multiple datafiles l…

…oaded

Switched from using a link-offset attached to the dataset rows
-> to a dataset offset attached to the link rows.

Otherwise, the lookup indices could get mangled if colors and widths
were accessing differing datasets.

src/Globals.ts

@@ -53,7 +53,7 @@ export interface DataTableColumn {
 export interface LookupDataset {
   dataTable: DataTable
   activeColumn: string
-  joinColumn: string
+  csvRowFromLinkRow: number[]
 }
 
 export interface CSV {

src/components/viz-configurator/Colors.vue

@@ -108,9 +108,17 @@ export default class VueComponent extends Vue {
     this.datasetsAreLoaded()
   }
 
+  private isFirstDataset = true
+
   @Watch('datasets')
   private datasetsAreLoaded() {
     const datasetIds = Object.keys(this.datasets)
+    this.datasetLabels = datasetIds
+
+    // don't change colors if we already set them
+    if (!this.isFirstDataset) return
+
+    if (datasetIds.length) this.isFirstDataset = false
 
     const { dataset, columnName, colorRamp } = this.vizConfiguration.display.color
 
@@ -122,15 +130,14 @@ export default class VueComponent extends Vue {
         this.selectedColor =
           this.colorChoices.find(c => c.ramp.toLowerCase() === colorRamp.ramp.toLowerCase()) ||
           this.colorChoices[0]
-        this.flip = !!colorRamp.reverse ? !!this.selectedColor.reverse : !this.selectedColor.reverse // XOR
+        this.flip = !!colorRamp.reverse ? !this.selectedColor.reverse : !!this.selectedColor.reverse // XOR
         if (colorRamp.steps) this.steps = '' + colorRamp.steps
       }
     } else if (datasetIds.length) {
       const secondColumn = Object.keys(this.datasets[datasetIds[0]])[1]
       console.log(secondColumn)
       if (secondColumn) this.dataColumn = `${datasetIds[0]}/${secondColumn}`
     }
-    this.datasetLabels = datasetIds
   }
 
   @Watch('flip')

src/plugins/links-gl/LinkLayer.tsx

@@ -32,10 +32,11 @@ export default function Component({
 
   const [viewState, setViewState] = useState(globalStore.state.viewState)
 
-  const { dataTable, activeColumn, joinColumn } = build
+  const { dataTable, activeColumn, csvRowFromLinkRow } = build
   const buildColumn: DataTableColumn = dataTable[activeColumn] || { values: [] }
 
   const widthValues = widths.dataTable[widths.activeColumn]
+  const widthRowLookup = widths.csvRowFromLinkRow
 
   // deck.gl colors must be in rgb[] or rgba[] format
   const colorsAsRGB: any = colors.map(hexcolor => {
@@ -71,12 +72,19 @@ export default function Component({
     setViewState(globalStore.state.viewState)
   }
 
+  const numCsvRows = csvRowFromLinkRow.length
+
   // --- LINE COLORS -----------------------------------------------
   const getLineColor = (
     feature: any,
     objectInfo: { index: number; data: any; target: number[] }
   ) => {
-    let value = dataTable[activeColumn].values[objectInfo.index]
+    if (!dataTable[activeColumn]) return colorPaleGrey
+
+    // use the csvRowLookup if we have it; if now then just use the link row number.
+    const csvRow = numCsvRows ? csvRowFromLinkRow[objectInfo.index] : objectInfo.index
+    let value = dataTable[activeColumn].values[csvRow]
+
     if (!value) return colorInvisible
 
     if (colors.length === 1) return colorsAsRGB[0]
@@ -111,7 +119,8 @@ export default function Component({
   ) => {
     if (!widthValues) return 0
 
-    const value = widthValues.values[objectInfo.index]
+    const csvRow = widthRowLookup.length ? widthRowLookup[objectInfo.index] : objectInfo.index
+    const value = widthValues.values[csvRow]
 
     if (showDiffs) {
       const baseValue = base.dataTable[base.activeColumn].values[objectInfo.index]
@@ -194,20 +203,12 @@ export default function Component({
   //@ts-ignore
   const layer = new LineOffsetLayer({
     id: 'linkLayer',
-    data: buildColumn.values,
-    getSourcePosition: (object: any, props: { index: number; data: any; target: any[] }) => {
-      // target is [long,lat]
-      const offset = 2 * dataTable[joinColumn].values[props.index]
-      props.target[0] = links.source[offset]
-      props.target[1] = links.source[offset + 1]
-      return props.target
-    },
-    getTargetPosition: (object: any, props: { index: number; data: any; target: any[] }) => {
-      // target is [long,lat]
-      const offset = 2 * dataTable[joinColumn].values[props.index]
-      props.target[0] = links.dest[offset]
-      props.target[1] = links.dest[offset + 1]
-      return props.target
+    data: {
+      length: links.source.length / 2,
+      attributes: {
+        getSourcePosition: { value: links.source, size: 2 },
+        getTargetPosition: { value: links.dest, size: 2 },
+      },
     },
     getColor: getLineColor,
     getWidth: getLineWidth,

src/plugins/links-gl/LinkVolumes.vue

@@ -177,7 +177,11 @@ class MyPlugin extends Vue {
   private showDiffs = false
   private showTimeRange = false
 
-  private geojsonData = { source: new Float32Array(), dest: new Float32Array() }
+  private geojsonData = {
+    source: new Float32Array(),
+    dest: new Float32Array(),
+    linkIds: [] as any[],
+  }
 
   private generatedColors: string[] = ['#4e79a7']
 
@@ -192,11 +196,23 @@ class MyPlugin extends Vue {
     thumbnail: false,
   }
 
-  private csvData: LookupDataset = { activeColumn: '', joinColumn: '', dataTable: {} }
-  private csvBase: LookupDataset = { activeColumn: '', joinColumn: '', dataTable: {} }
-  private csvWidth: LookupDataset = { activeColumn: '', joinColumn: '', dataTable: {} }
+  private csvData: LookupDataset = {
+    activeColumn: '',
+    dataTable: {},
+    csvRowFromLinkRow: [],
+  }
+  private csvBase: LookupDataset = {
+    activeColumn: '',
+    dataTable: {},
+    csvRowFromLinkRow: [],
+  }
+  private csvWidth: LookupDataset = {
+    activeColumn: '',
+    dataTable: {},
+    csvRowFromLinkRow: [],
+  }
 
-  private linkOffsetLookup: { [id: string]: number } = {}
+  // private linkOffsetLookup: { [id: string]: number } = {}
   private numLinks = 0
 
   private isDarkMode = this.$store.state.colorScheme === ColorScheme.DarkMode
@@ -214,7 +230,6 @@ class MyPlugin extends Vue {
   }
 
   private get colorRampType() {
-    console.log(this.vizDetails)
     const rampType = this.vizDetails.display.color?.colorRamp?.style
     if (rampType === undefined) return -1
     return rampType
@@ -342,7 +357,7 @@ class MyPlugin extends Vue {
     width?: WidthDefinition
     dataset?: DatasetDefinition
   }) {
-    console.log({ props })
+    console.log(props)
 
     if (props['color']) {
       // if (JSON.stringify(props.color) === JSON.stringify(this.vizDetails.display.color)) return
@@ -378,7 +393,6 @@ class MyPlugin extends Vue {
   }
 
   private handleNewWidth(width: WidthDefinition) {
-    console.log({ width })
     const { columnName, dataset, scaleFactor } = width
     if (!columnName) return
 
@@ -399,7 +413,7 @@ class MyPlugin extends Vue {
     this.csvWidth = {
       dataTable: selectedDataset,
       activeColumn: columnName,
-      joinColumn: LOOKUP_COLUMN,
+      csvRowFromLinkRow: dataset ? this.csvRowLookupFromLinkRow[dataset] : [],
     }
   }
 
@@ -416,13 +430,11 @@ class MyPlugin extends Vue {
     const selectedDataset = this.datasets[datasetKey]
     if (!selectedDataset) return
 
-    console.log(datasetKey, columnName)
-
     if (this.csvData.dataTable !== selectedDataset) {
       this.csvData = {
         dataTable: selectedDataset,
         activeColumn: '',
-        joinColumn: LOOKUP_COLUMN,
+        csvRowFromLinkRow: this.csvRowLookupFromLinkRow[datasetKey],
       }
     }
 
@@ -498,7 +510,7 @@ class MyPlugin extends Vue {
 
   private async loadNetwork(): Promise<any> {
     this.myState.statusMessage = 'Loading network...'
-    this.linkOffsetLookup = {}
+    // this.linkOffsetLookup = {}
     this.numLinks = 0
 
     const filename = this.vizDetails.network || this.vizDetails.geojsonFile
@@ -515,7 +527,7 @@ class MyPlugin extends Vue {
           msg: `Error loading: ${networkPath}`,
         })
       } else {
-        this.linkOffsetLookup = buffer.data.linkOffsetLookup
+        // this.linkOffsetLookup = buffer.data.linkOffsetLookup
         this.geojsonData = buffer.data.links
         this.numLinks = this.geojsonData.source.length / 2
 
@@ -534,42 +546,56 @@ class MyPlugin extends Vue {
     this.networkWorker.postMessage({ filePath: networkPath, fileSystem: this.myState.fileSystem })
   }
 
+  private dataLoaderWorkers: Worker[] = []
+
   private beforeDestroy() {
     // MUST delete the React view handle to prevent gigantic memory leak!
     delete REACT_VIEW_HANDLES[this.linkLayerId]
 
     if (this.networkWorker) this.networkWorker.terminate()
+    for (const worker of this.dataLoaderWorkers) worker.terminate()
 
     this.$store.commit('setFullScreen', false)
   }
 
+  private csvRowLookupFromLinkRow: { [datasetId: string]: number[] } = {}
+
   private handleNewDataset(props: DatasetDefinition) {
     console.log('NEW dataset', props)
     const { key, dataTable, filename } = props
 
-    // Create a LOOKUP column which links this CSV data to the network links
-    const joinColumn: DataTableColumn = {
-      name: LOOKUP_COLUMN,
-      type: DataType.LOOKUP,
-      values: [],
-    }
+    // We need a lookup so we can find the CSV row that matches each link row.
+    // A normal hashmap lookup is too slow, so we'll create an array containing
+    // the lookup on load (now); then it should be O(1) fast from that point forward.
 
-    // For now we assume the 1st column always has the link ID
+    // For now we assume the 1st CSV column always has the link ID
     const columnNames = Object.keys(dataTable)
     const assumedLinkIdIsFirstColumn = columnNames[0]
     const linkIdColumn = dataTable[assumedLinkIdIsFirstColumn]
 
-    // do the lookup
-    for (let i = 0; i < linkIdColumn.values.length; i++) {
-      joinColumn.values[i] = this.linkOffsetLookup[linkIdColumn.values[i]]
+    let tempMapLinkIdToCsvRow = {} as any
+    for (let csvRow = 0; csvRow < linkIdColumn.values.length; csvRow++) {
+      tempMapLinkIdToCsvRow[linkIdColumn.values[csvRow]] = csvRow
     }
 
-    // add the join column to the CSV dataset
-    dataTable[LOOKUP_COLUMN] = joinColumn
-    this.datasets = Object.assign({ ...this.datasets }, { [key]: dataTable })
-    this.handleDatasetisLoaded(key)
+    // Create a LOOKUP array which links this CSV data to the network links
+    // loop through all network links, we need the CSV row for each link.
+    const getCsvRowNumberFromLinkRowNumber: number[] = []
+    console.log({ geo: this.geojsonData })
+    for (let linkRow = 0; linkRow < this.geojsonData.linkIds.length; linkRow++) {
+      const linkId = this.geojsonData.linkIds[linkRow]
+      const csvRow = tempMapLinkIdToCsvRow[linkId]
+      if (csvRow !== undefined) getCsvRowNumberFromLinkRowNumber[linkRow] = csvRow
+    }
+
+    // Save the lookup with the dataset.
+    this.csvRowLookupFromLinkRow[key] = getCsvRowNumberFromLinkRowNumber
+    tempMapLinkIdToCsvRow = {} // probably unnecessary but we def want this to be GC'ed
 
+    // all done!
     if (filename) this.vizDetails.datasets[key] = filename
+    this.datasets = Object.assign({ ...this.datasets }, { [key]: dataTable })
+    this.handleDatasetisLoaded(key)
   }
 
   private loadCSVFiles() {
@@ -605,11 +631,11 @@ class MyPlugin extends Vue {
         },
       },
       activeColumn: LOOKUP_COLUMN,
-      joinColumn: LOOKUP_COLUMN,
+      csvRowFromLinkRow: [],
     }
 
     // there is no range(maxValue) in Javascript! :-(
-    const length = Object.keys(this.linkOffsetLookup).length
+    const length = this.geojsonData.source.length / 2 // half because this contains x/y coordinates
     const lookup = [...Array(length).keys()]
     this.csvData.dataTable[LOOKUP_COLUMN].values = lookup
 
@@ -647,7 +673,7 @@ class MyPlugin extends Vue {
         this.csvData = {
           dataTable: this.datasets[datasetId],
           activeColumn: firstColumnName,
-          joinColumn: LOOKUP_COLUMN,
+          csvRowFromLinkRow: this.csvRowLookupFromLinkRow[datasetId],
         }
       }
     }
@@ -657,7 +683,7 @@ class MyPlugin extends Vue {
       this.csvBase = {
         dataTable: this.datasets[datasetId],
         activeColumn: '',
-        joinColumn: LOOKUP_COLUMN,
+        csvRowFromLinkRow: this.csvRowLookupFromLinkRow[datasetId],
       }
       this.showDiffs = true
     }
@@ -677,6 +703,8 @@ class MyPlugin extends Vue {
 
     const thread = new Promise<DataTable>((resolve, reject) => {
       const worker = new DataFetcher() as Worker
+      this.dataLoaderWorkers.push(worker) // so we can terminate them all on destroy
+
       try {
         worker.onmessage = e => {
           worker.terminate()
@@ -695,6 +723,7 @@ class MyPlugin extends Vue {
     })
 
     const dataTable = await thread
+
     this.finishedLoadingCSV(key, dataTable)
   }