Skip to content

Commit

Permalink
fix: firefox on windows hangs on background worker dataset load
Browse files Browse the repository at this point in the history 
Papaparse library seems to hang on dynamic value typing. Work around.
  • Loading branch information
@billyc
billyc committed Feb 9, 2022
1 parent 3d76add commit f1fdb87
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 42 deletions.
12 changes: 11 additions & 1 deletion src/js/DashboardDataManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ export default class DashboardDataManager {
this.fileApi = this.getFileSystem(this.root)
}

public kill() {
for (const worker of this.threads) worker.terminate()
}

public async getFilteredDataset(config: { dataset: string; groupBy?: string; value?: string }) {
const rows = this.datasets[config.dataset].filteredRows
if (!rows) return { filteredRows: null }
Expand Down Expand Up @@ -222,6 +226,7 @@ export default class DashboardDataManager {
}

public clearCache() {
this.kill() // any stragglers must die
this.datasets = {}
}

Expand Down Expand Up @@ -256,15 +261,18 @@ export default class DashboardDataManager {
// private thread!: any
private files: any[] = []

private threads: Worker[] = []

private async fetchDataset(config: { dataset: string }) {
if (!this.files.length) {
const { files } = await new HTTPFileSystem(this.fileApi).getDirectory(this.subfolder)
this.files = files
}

return new Promise<DataTable>((resolve, reject) => {
const thread = new DataFetcherWorker()
this.threads.push(thread)
try {
const thread = new DataFetcherWorker()
thread.postMessage({
fileSystemConfig: this.fileApi,
subfolder: this.subfolder,
Expand All @@ -273,6 +281,7 @@ export default class DashboardDataManager {
})

thread.onmessage = e => {
thread.terminate()
if (e.data.error) {
console.error(e.data.error)
globalStore.commit('error', e.data.error)
Expand All @@ -281,6 +290,7 @@ export default class DashboardDataManager {
resolve(e.data)
}
} catch (err) {
thread.terminate()
console.error(err)
reject(err)
}
Expand Down
4 changes: 3 additions & 1 deletion src/plugins/links-gl/LinkVolumes.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,6 @@ class MyPlugin extends Vue {
private setDataIsLoaded() {
this.isDataLoaded = true
this.$emit('isLoaded', true)
}
public buildFileApi() {
Expand Down Expand Up @@ -585,6 +584,9 @@ class MyPlugin extends Vue {
this.myState.statusMessage = ''
this.$emit('isLoaded', true)
// this.setDataIsLoaded()
// then load CSVs in background
this.loadCSVFiles()
} catch (e) {
Expand Down
6 changes: 3 additions & 3 deletions src/views/DashBoard.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,17 @@


//- card contents
.spinner-box(v-if="getCardComponent(card)" :id="card.id" :class="{'is-loaded': numberOfShownCards >= card.number}")
.spinner-box(v-if="getCardComponent(card)" :id="card.id" :class="{'is-loaded': card.isLoaded}")

component.dash-card(
v-if="numberOfShownCards >= card.number"
:is="getCardComponent(card)"
:fileSystemConfig="fileSystemConfig"
:subfolder="xsubfolder"
:files="fileList"
:yaml="card.props.configFile"
:config="card.props"
:datamanager="datamanager"
:style="{opacity: opacity[card.id]}"
:cardId="card.id"
:cardTitle="card.title"
:allConfigFiles="allConfigFiles"
Expand Down Expand Up @@ -276,7 +276,7 @@ export default class VueComponent extends Vue {
card.number = numCard
// Vue is weird about new properties: use Vue.set() instead
Vue.set(this.opacity, card.id, 0.15)
Vue.set(this.opacity, card.id, 0.2)
Vue.set(this.infoToggle, card.id, false)
numCard++
Expand Down
171 changes: 134 additions & 37 deletions src/workers/DataFetcher.worker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,56 +149,153 @@ function parseCsvFile(fileKey: string, filename: string, text: string) {
// prepare storage object -- figure out records and columns
const dataTable: DataTable = {}

const columnNames: string[] = []
let rowCount = 0

const csv = Papaparse.parse(text, {
// preview: 10000,
delimitersToGuess: ['\t', ';', ','],
comments: '#',
dynamicTyping: true,
header: true,
dynamicTyping: false,
header: false,
skipEmptyLines: true,
step: (results: any, parser) => {
const row = results.data
// console.log(row)
for (const key in row) {
if (!dataTable[key]) {
dataTable[key] = { name: key, values: [], type: DataType.NUMBER } // DONT KNOW TYPE YET
})

// set up arrays
// First we assume everything is a number. Then when we find out otherwise, we switch
// to a regular JS array as necessary
const headers = csv.data[0] as string[]
const numColumns = headers.length

// first column is always string lookup
dataTable[headers[0]] = { name: headers[0], values: [], type: DataType.STRING }
// other columns: start with numbers and switch if we have to
for (let column = 1; column < numColumns; column++) {
const values = new Float32Array(csv.data.length - 1)
values.fill(NaN)
dataTable[headers[column]] = { name: headers[column], values, type: DataType.NUMBER } // DONT KNOW TYPE YET
}

// copy data to column-based arrays
for (let rowCount = 0; rowCount < csv.data.length - 1; rowCount++) {
const row = csv.data[rowCount + 1] as any[]

// if (rowCount % 65536 === 0) console.log(row)

for (let column = 0; column < numColumns; column++) {
const key = headers[column]
const value = row[column] // always string

if (column == 0) {
// column 0 is always string
dataTable[key].values[rowCount] = value
} else {
const float = parseFloat(value) // number or NaN
if (!isNaN(float)) {
// number:
dataTable[key].values[rowCount] = value
} else {
// non-number:
if (Array.isArray(dataTable[key].values)) {
dataTable[key].values[rowCount] = value
} else {
// first, convert the Float32Array to a normal javascript array
if (typeof value == 'string') dataTable[key].type = DataType.STRING
if (typeof value == 'boolean') dataTable[key].type = DataType.BOOLEAN

// convert to regular array
const regularArray = Array.from(dataTable[key].values.slice(0, rowCount))
regularArray[rowCount] = value
// switch the array out
dataTable[key].values = regularArray
}
}
;(dataTable[key].values as any[]).push(row[key])
}
},
complete: results => {
let firstColumnName = ''
for (const columnName in dataTable) {
// first column is special: it contains the linkID
// TODO: This is obviously wrong; we need a way to specify this from User
if (!firstColumnName) {
firstColumnName = columnName
continue
}
}
}

// figure out types
const column = dataTable[columnName]
if (typeof column.values[0] == 'string') column.type = DataType.STRING
if (typeof column.values[0] == 'boolean') column.type = DataType.BOOLEAN
// calculate max for numeric columns
for (const column of Object.values(dataTable)) {
if (column.type !== DataType.NUMBER) continue

// convert numbers to Float32Arrays
if (column.type === DataType.NUMBER) {
const fArray = new Float32Array(column.values)
column.values = fArray
let max = -Infinity
for (const value of column.values) max = Math.max(max, value)
column.max = max
}

// calculate max for numeric columns
let max = -Infinity
for (const value of column.values) max = Math.max(max, value)
column.max = max
}
}
_fileData[fileKey] = dataTable
}

// and save it
_fileData[fileKey] = dataTable
},
function badparseCsvFile(fileKey: string, filename: string, text: string) {
// prepare storage object -- figure out records and columns
// console.log('c1')
const dataTable: DataTable = {}

const csv = Papaparse.parse(text, {
// preview: 10000,
delimitersToGuess: ['\t', ';', ','],
comments: '#',
dynamicTyping: false,
header: false,
skipEmptyLines: true,
})
console.log('c2')

// step: (results: any, parser) => {
// rowCount++
// const row = results.data

// // if (rowCount % 8192 === 0) console.log(row)

// // console.log(row)
// const keys = Object.keys(row)
// for (let column = 0; column < keys.length; column++) {
// const key = keys[column]
// if (!dataTable[key]) {
// dataTable[key] = { name: key, values: [], type: DataType.NUMBER } // DONT KNOW TYPE YET
// }

// const v = row[key]
// const float = parseFloat(v)
// if (column === 0 || isNaN(float)) {
// dataTable[key].values[rowCount] = v
// } else {
// dataTable[key].values[rowCount] = float
// }
// }
// },
// complete: results => {
// console.log('finished parsing', fileKey)
// let firstColumnName = ''

// for (const columnName in dataTable) {
// // first column is special: it contains the linkID
// // TODO: This is obviously wrong; we need a way to specify this from User
// if (!firstColumnName) {
// firstColumnName = columnName
// continue
// }

// // figure out types
// const column = dataTable[columnName]
// if (typeof column.values[0] == 'string') column.type = DataType.STRING
// if (typeof column.values[0] == 'boolean') column.type = DataType.BOOLEAN

// // convert numbers to Float32Arrays
// if (column.type === DataType.NUMBER) {
// const fArray = new Float32Array(column.values)
// column.values = fArray

// // calculate max for numeric columns
// let max = -Infinity
// for (const value of column.values) max = Math.max(max, value)
// column.max = max
// }
// }

// and save it
_fileData[fileKey] = dataTable

// })
}

async function loadFileOrGzipFile(filename: string) {
Expand Down

0 comments on commit f1fdb87

Please sign in to comment.