From 7c1458a5b92821965a5d7e2f7ab8643ab8d3043b Mon Sep 17 00:00:00 2001 From: luke-harrison-personal <84890777+luke-harrison-personal@users.noreply.github.com> Date: Sun, 12 Mar 2023 05:38:38 +1000 Subject: [PATCH] Zoom optimisation for Waves and matching implementation for Spectrograms (#2646) * Used setTimeout() method to require a minimum time period passes before redrawing/recalculating to reduce CPU requirements while zooming in on waveforms. Currently displaying a waveform image in example/zoom which will be used to hide the lack of wave drawing while zooming. * Implemented a backimage mimicking the wave to display zoom level faster TO DO: Hide the real wave while zooming Hide the backimage after zooming Zoom in on current timestamp while zooming on backimage FUTURE: Split or crop backimage for longer waves Re-render backimage to match the colour style of the wave with selected timestamp * Added hiding/showing of elements while zooming Since the coloured section of the wave is also handled by canvas objects within a wave I am going to move the backimage functionality into the canvasentry.js file so that each canvas handles it's own image, which should resolve the issue as well as segment the image into sections. * backimages are stored and drawn for each canvas entry. Current issue is the left offsets of each backimage are not being calculated correctly and images are overlapping. * Zooming now correctly handles progress positions. Creates backimages for progress wave Left offset of canvas elements are handled correctly. * Zooming out now works without any issues Zooming in needs a viewable region boundary implemented so that canvases don't become too large. Progress tracking and regions plugin work without any issues now. * Zoom optimisations have been made Canvases not in frame are cleared and not rendered Canvases that become too large are clipped Progress location is maintained * Added priority drawing for canvases in view Currently, changing zoom level before all canvases have been loaded causes issues, my next goal is to fix this by either improving the handling of missing canvases or preventing zoom change until all canvases have loaded. * Zoom slider locks up if rendering isn't finished I'll use and hopefully instead of locking up I can return to a backup render of the wave. * Allows zooming immediately after release This is done by cancelling loading and showing only loaded sections * Improved backup image that can handle large waves Zooming out on partially loaded waves loads a backup image, this features works regardless of wave size. * Optimisation checks if there is a wrapper With this change the branch now passes the included tests * Fixed issues with back image generation Low zoom level previously caused issue with back image generation, the issue has been resolved. * Added zooming to spectrogram example This will allow testing of the intended multicanvas implementation * Spectrogram successfully split into two canvases Canvases are equally sized Calculation is on the whole spectrogram and then converted to seperate canvases This is a starting point for scaling up to an adaptive number of canvases, similar to the multicanvas drawer used for the main wave. Ideally calculation can be moved such that it is done for each canvas, this would mean speed could be improved by calcuating visible spectrogram canvases. Right now this offers no noticeable speed improvement, but could improve the ability to display zoomable spectrograms without creating images too large for browsers. * Working for arbitrary number of canvases Image still generated first then split, limiting max size Committing a working version before attempting fix * spectrogram sections are generated per-canvas Currently trying to draw all canvases at once causes failure Next I'm going to implement staggering of the canvases to hopefully allowing further zooming even without performance improvements * Spectrogram can now display at unlimited width Fixed an issue where resample() was creating full-size arrays instead of limiting to the canvas size Fixed an issue where render() was being called twice as a result of an incorrect event fire in wavesurfer.js Removed placeholder text in zoom example * Spectrogram only draws viewable canvases After this I will implement delays rendering of low-priority canvases * Rearranged variables to allow for setting delays * Canvases stretch to zoom * Zoom now scales correctly against waveform Fixed issue where progress wasn't accurate during zoom * Improved zooming functionality for wavesurfer Removed functions that were no longer necessary backimages are no longer required, existing canvases are stretched during zoom. backup image is no longer required, canvases can continue rendering during the zooming process. * Implemented priority rendering for spectrogram Removed/changed comments referring to the removed backimage/backupimage * Added timeout clearing to prevent drawing old canvases after zoom level changes * Removed unecessary class variables * Patched in a fix so that original zooming behaviour is preserved Upon first call of the stretchCanvases() function the drawer will switch over to the optimised zoom functionality * Removed unecessary changes to formatting * Removed more unecessary changes --------- Co-authored-by: Luke Harrison --- example/spectrogram/app.js | 32 +++++-- example/spectrogram/index.html | 18 ++-- example/zoom/index.html | 10 ++- example/zoom/main.js | 3 + src/drawer.canvasentry.js | 43 +++++++++ src/drawer.js | 4 - src/drawer.multicanvas.js | 72 ++++++++++++++- src/plugin/spectrogram/index.js | 151 +++++++++++++++++++++++++------- src/wavesurfer.js | 33 +++++++ 9 files changed, 312 insertions(+), 54 deletions(-) diff --git a/example/spectrogram/app.js b/example/spectrogram/app.js index 0b6a58ed0..68037e8bc 100644 --- a/example/spectrogram/app.js +++ b/example/spectrogram/app.js @@ -22,15 +22,6 @@ function initAndLoadSpectrogram(colorMap) { ] }; - if (location.search.match('scroll')) { - options.minPxPerSec = 100; - options.scrollParent = true; - } - - if (location.search.match('normalize')) { - options.normalize = true; - } - wavesurfer = WaveSurfer.create(options); /* Progress bar */ @@ -54,6 +45,29 @@ function initAndLoadSpectrogram(colorMap) { })(); wavesurfer.load('../media/demo.wav'); + + // Zoom slider + let slider = document.querySelector('[data-action="zoom"]'); + + slider.value = wavesurfer.params.minPxPerSec; + slider.min = wavesurfer.params.minPxPerSec; + slider.max = 250; + + + slider.addEventListener('input', function() { + wavesurfer.zooming(slider.value); + }); + slider.addEventListener('mouseup', function() { + wavesurfer.zoom(slider.value); + + let desiredWidth = Math.max(parseInt(wavesurfer.container.offsetWidth * window.devicePixelRatio), + parseInt(wavesurfer.getDuration() * slider.value * window.devicePixelRatio)); + wavesurfer.spectrogram.width = desiredWidth; + wavesurfer.spectrogram.render(); + }); + + // set initial zoom to match slider value + wavesurfer.zoom(slider.value); } document.addEventListener('DOMContentLoaded', function() { diff --git a/example/spectrogram/index.html b/example/spectrogram/index.html index 8a18bbf9c..ec99b89c9 100644 --- a/example/spectrogram/index.html +++ b/example/spectrogram/index.html @@ -31,13 +31,6 @@
- - - -

wavesurfer.js + Spectrogram

@@ -59,6 +52,17 @@

wavesurfer.js Pause +
+ +
+ +
+ +
+ +
+ +

diff --git a/example/zoom/index.html b/example/zoom/index.html index f4f18ad03..f4ab9fc38 100644 --- a/example/zoom/index.html +++ b/example/zoom/index.html @@ -82,9 +82,13 @@

How to Zoom

backend: 'MediaElement' }); -document.querySelector('#slider').oninput = function () { - wavesurfer.zoom(Number(this.value)); -}; +let slider = document.querySelector('[data-action="zoom"]'); +slider.addEventListener('input', function() { + wavesurfer.zooming(slider.value); +}); +slider.addEventListener('mouseup', function() { + wavesurfer.zoom(slider.value); +});

diff --git a/example/zoom/main.js b/example/zoom/main.js index c3163beed..f63a8681c 100644 --- a/example/zoom/main.js +++ b/example/zoom/main.js @@ -49,6 +49,9 @@ document.addEventListener('DOMContentLoaded', function() { slider.max = 1000; slider.addEventListener('input', function() { + wavesurfer.zooming(Number(this.value)); + }); + slider.addEventListener('mouseup', function() { wavesurfer.zoom(Number(this.value)); }); diff --git a/src/drawer.canvasentry.js b/src/drawer.canvasentry.js index 7f1da3884..3866345b2 100644 --- a/src/drawer.canvasentry.js +++ b/src/drawer.canvasentry.js @@ -66,6 +66,11 @@ export default class CanvasEntry { * @type {object} */ this.canvasContextAttributes = {}; + /** + * The Timeout id used to track this canvas entry. + */ + this.drawTimeout = null; + } /** @@ -125,21 +130,27 @@ export default class CanvasEntry { */ clearWave() { // wave + this.waveCtx.save(); + this.waveCtx.setTransform(1, 0, 0, 1, 0, 0); this.waveCtx.clearRect( 0, 0, this.waveCtx.canvas.width, this.waveCtx.canvas.height ); + this.waveCtx.restore(); // progress if (this.hasProgressCanvas) { + this.progressCtx.save(); + this.progressCtx.setTransform(1, 0, 0, 1, 0, 0); this.progressCtx.clearRect( 0, 0, this.progressCtx.canvas.width, this.progressCtx.canvas.height ); + this.progressCtx.restore(); } } @@ -424,4 +435,36 @@ export default class CanvasEntry { return this.wave.toDataURL(format, quality); } } + + /** + * Stretches existing canvas + * @param {Number} newTotalWidth total width of wave in pixels + */ + stretchCanvas(newTotalWidth) { + //Calculate the start and width of this canvas + let start = Math.round(this.start * newTotalWidth); + let width = Math.round(this.end * newTotalWidth - start); + + //Stretch canvas + let elementSize = { width: width + 'px' }; + let elementStart = {left: start + 'px'}; + style(this.wave, elementSize); + style(this.wave, elementStart); + if (this.hasProgressCanvas) { + style(this.progress, elementSize); + style(this.progress, elementStart); + } + } + + /** + * Set the left offset of the canvas + * @param {Number} position in px for the canvas to start + */ + setLeft(position) { + let elementStart = {left: position + 'px'}; + style(this.wave, elementStart); + if (this.hasProgressCanvas) { + style(this.progress, elementStart); + } + } } diff --git a/src/drawer.js b/src/drawer.js index 8e3962b11..491c976d5 100644 --- a/src/drawer.js +++ b/src/drawer.js @@ -272,10 +272,6 @@ export default class Drawer extends util.Observer { * @return {boolean} Whether the width of the container was updated or not */ setWidth(width) { - if (this.width == width) { - return false; - } - this.width = width; if (this.params.fillParent || this.params.scrollParent) { diff --git a/src/drawer.multicanvas.js b/src/drawer.multicanvas.js index 74899338a..ef4cbd014 100644 --- a/src/drawer.multicanvas.js +++ b/src/drawer.multicanvas.js @@ -89,6 +89,14 @@ export default class MultiCanvas extends Drawer { * @type {boolean} */ this.vertical = params.vertical; + + /** + * Whether to use the optimsized zoom rendering + * Automatically toggles to true if stretchCanvases() function is called + * + * @type {boolean} + */ + this.optimiseZoom = false; } /** @@ -157,10 +165,16 @@ export default class MultiCanvas extends Drawer { let canvasWidth = this.maxCanvasWidth + this.overlap; const lastCanvas = this.canvases.length - 1; + let leftOffset = 0; this.canvases.forEach((entry, i) => { if (i == lastCanvas) { canvasWidth = this.width - this.maxCanvasWidth * lastCanvas; } + + //Set left offset and add to next entry + entry.setLeft(leftOffset); + leftOffset += canvasWidth / this.params.pixelRatio; + this.updateDimensions(entry, canvasWidth, this.height); entry.clearWave(); @@ -410,7 +424,39 @@ export default class MultiCanvas extends Drawer { this.canvases.forEach((entry, i) => { this.setFillStyles(entry, waveColor, progressColor); this.applyCanvasTransforms(entry, this.params.vertical); - entry.drawLines(peaks, absmax, halfH, offsetY, start, end); + + if (this.optimiseZoom) { + //Optimising zoom functionality + //If there's a wrapper, optimise for the view + let priority = 0; + if (this.wrapper) { + let canvasRect = entry.wave.getBoundingClientRect(); + let wrapperRect = this.wrapper.getBoundingClientRect(); + + //Determine whether canvas is in viewframe or not and assign priority + if (Math.floor(canvasRect['left']) > Math.ceil(wrapperRect['right'])) { + //Canvas is to the right of view window + let distance = canvasRect['left'] - wrapperRect['right']; + priority = Math.ceil(distance / wrapperRect['width']); + } else if (Math.ceil(canvasRect['right']) < Math.floor(wrapperRect['left'])) { + //Canvas is to the left of the view window + let distance = wrapperRect['left'] - canvasRect['right']; + priority = Math.ceil(distance / wrapperRect['width']); + } + } else { + //Everything is equal priority + } + + //This staggers the drawing of canvases so they don't all draw at once + entry.clearWave(); + clearTimeout(entry.drawTimeout); + entry.drawTimeout = setTimeout(function(){ + entry.drawLines(peaks, absmax, halfH, offsetY, start, end); + entry.drawTimeout = null; + }, 25 * priority); + } else { + entry.drawLines(peaks, absmax, halfH, offsetY, start, end); + } }); } @@ -606,6 +652,30 @@ export default class MultiCanvas extends Drawer { } } + /** + * Stretches the canvases to mimic zoom without recalculation + * + * @param {Number} desiredWidth new width of the wave display + * @param {Number} progress Value between 0 and 1 for wave progress + */ + stretchCanvases(desiredWidth, progress) { + if (!this.optimiseZoom) { + //Enable optimsed zooming + this.optimiseZoom = true; + } + let totalCanvasWidth = Math.round(desiredWidth / this.params.pixelRatio); + this.width = desiredWidth; + + for (let i = 0; i < this.canvases.length; i++) { + this.canvases[i].stretchCanvas(totalCanvasWidth); + } + + //Update progress + let progressPos = progress * totalCanvasWidth; + this.updateProgress(progressPos); + this.recenterOnPosition(progressPos, true); + } + /** * Render the new progress * diff --git a/src/plugin/spectrogram/index.js b/src/plugin/spectrogram/index.js index 75f8bfb1c..686735d93 100644 --- a/src/plugin/spectrogram/index.js +++ b/src/plugin/spectrogram/index.js @@ -92,6 +92,9 @@ export default class SpectrogramPlugin { this._onRender = () => { this.render(); }; + this._onZoom = () => { + this.stretchCanvases(); + }; this._onWrapperClick = e => { this._wrapperClickHandler(e); }; @@ -136,6 +139,9 @@ export default class SpectrogramPlugin { this.alpha = params.alpha; this.splitChannels = params.splitChannels; this.channels = this.splitChannels ? ws.backend.buffer.numberOfChannels : 1; + this.canvases = []; + this.canvasesTimeouts = []; + this.scrollLeftTracker = 0; //Tracks the desired scrollLeft value // Getting file's original samplerate is difficult(#1248). // So set 12kHz default to render like wavesurfer.js 5.x. @@ -143,10 +149,11 @@ export default class SpectrogramPlugin { this.frequencyMax = params.frequencyMax || 12000; this.createWrapper(); - this.createCanvas(); + this.addCanvas(); this.render(); drawer.wrapper.addEventListener('scroll', this._onScroll); + ws.on('zoom', this._onZoom); ws.on('redraw', this._onRender); }; } @@ -231,17 +238,49 @@ export default class SpectrogramPlugin { this.fireEvent('click', relX / this.width || 0); } - createCanvas() { - const canvas = (this.canvas = this.wrapper.appendChild( + /** + * Add a canvas to this.canvases + */ + addCanvas() { + const canvas = (this.wrapper.appendChild( document.createElement('canvas') )); - this.spectrCc = canvas.getContext('2d'); - this.util.style(canvas, { position: 'absolute', zIndex: 4 }); + + this.canvases.push(canvas); + this.canvasesTimeouts.push(null); + } + + /** + * Remove a canvas from this.canvases + */ + removeCanvas() { + //Stop drawing (if drawing) + clearTimeout(this.canvasesTimeouts[this.canvasesTimeouts.length - 1]); + + let lastEntry = this.canvases[this.canvases.length - 1]; + lastEntry.parentElement.removeChild(lastEntry); + + this.canvases.pop(); + this.canvasesTimeouts.pop(); + } + + /** + * Ensure the correct number of canvases for the size of the spectrogram + */ + updateCanvases() { + let canvasesRequired = Math.ceil(this.width / 4000); + + while (this.canvases.length < canvasesRequired) { + this.addCanvas(); + } + while (this.canvases.length > canvasesRequired) { + this.removeCanvas(); + } } render() { @@ -255,11 +294,14 @@ export default class SpectrogramPlugin { } updateCanvasStyle() { - const width = Math.round(this.width / this.pixelRatio) + 'px'; - this.canvas.width = this.width; - this.canvas.height = this.fftSamples / 2 * this.channels; - this.canvas.style.width = width; - this.canvas.style.height = this.height + 'px'; + this.updateCanvases(); + //width per canvas + for (let i = 0; i < this.canvases.length; i++) { + this.canvases[i].width = Math.round(this.width / this.canvases.length); + this.canvases[i].height = this.fftSamples / 2 * this.channels; + this.canvases[i].style.width = Math.round(this.canvases[i].width / this.pixelRatio) + 'px'; + this.canvases[i].style.height = this.height + 'px'; + } } drawSpectrogram(frequenciesData, my) { @@ -268,25 +310,61 @@ export default class SpectrogramPlugin { frequenciesData = [frequenciesData]; } - const spectrCc = my.spectrCc; + my.updateCanvasStyle(); + + //Stop canvases still being drawn + for (let i = 0; i < my.canvasesTimeouts.length; i++) { + clearTimeout(my.canvasesTimeouts[i]); + } + + const view = [my.scrollLeftTracker, my.scrollLeftTracker + my.wrapper.clientWidth]; + + for (let canvasNum = 0; canvasNum < my.canvases.length; canvasNum++) { + const canvasLeft = canvasNum * Math.floor(my.width / my.canvases.length / my.pixelRatio); + const canvasRight = (canvasNum + 1) * Math.floor(my.width / my.canvases.length / my.pixelRatio); + const canvasBound = [canvasLeft, canvasRight]; + my.canvases[canvasNum].style['left'] = canvasLeft + 'px'; + + //Optimise drawing for the view + let priority = 0; + if (canvasBound[0] > view[1]) { + //Canvas is to the right of view window + let distance = canvasBound[0] - view[1]; + priority = Math.ceil(distance / (view[1] - view[0])); + } else if (canvasBound[1] < view[0]) { + //Canvas is to the left of the view window + let distance = view[0] - canvasBound[1]; + priority = Math.ceil(distance / (view[1] - view[0])); + } + + //delay = 25ms * number of viewport widths away the canvas is + my.canvasesTimeouts[canvasNum] = setTimeout(my.drawToCanvas, 25 * priority, frequenciesData, my, canvasNum); + } + } + + /** + * Draw spectrogram channel to a specific canvas + * @param {[Number, Number, Number]} frequenciesData spectrogram data in [channel, sample, freq] format + * @param {SpectrogramPlugin} my variable with 'this' in it + * @param {Number} canvasNum Canvas to draw to + */ + drawToCanvas(frequenciesData, my, canvasNum) { const height = my.fftSamples / 2; - const width = my.width; const freqFrom = my.buffer.sampleRate / 2; const freqMin = my.frequencyMin; const freqMax = my.frequencyMax; - if (!spectrCc) { - return; - } + for (let channel = 0; channel < frequenciesData.length; channel++) { - for (let c = 0; c < frequenciesData.length; c++) { // for each channel - const pixels = my.resample(frequenciesData[c]); - const imageData = new ImageData(width, height); + //Get pixels from frequency data and apply to image + const relevantFreqs = frequenciesData[channel].slice(canvasNum * Math.round(frequenciesData[channel].length / my.canvases.length), (canvasNum + 1) * Math.round(frequenciesData[channel].length / my.canvases.length)); + const pixels = my.resample(relevantFreqs); + const imageData = new ImageData(pixels.length, height); for (let i = 0; i < pixels.length; i++) { for (let j = 0; j < pixels[i].length; j++) { const colorMap = my.colorMap[pixels[i][j]]; - const redIndex = ((height - j) * width + i) * 4; + const redIndex = ((height - j) * imageData.width + i) * 4; imageData.data[redIndex] = colorMap[0] * 255; imageData.data[redIndex + 1] = colorMap[1] * 255; imageData.data[redIndex + 2] = colorMap[2] * 255; @@ -294,16 +372,20 @@ export default class SpectrogramPlugin { } } - // scale and stack spectrograms - createImageBitmap(imageData).then(renderer => - spectrCc.drawImage(renderer, - 0, height * (1 - freqMax / freqFrom), // source x, y - width, height * (freqMax - freqMin) / freqFrom, // source width, height - 0, height * c, // destination x, y - width, height // destination width, height - ) - ); + //Draw image to canvas + createImageBitmap(imageData).then(renderer => { + if (my.canvases[canvasNum]) { //Check canvas still exists after creating image + my.canvases[canvasNum].getContext('2d').drawImage(renderer, + 0, height * (1 - freqMax / freqFrom), // source x, y + imageData.width, height * (freqMax - freqMin) / freqFrom, // source width, height + 0, height * channel, // destination x, y + my.canvases[canvasNum].width, height // destination width, height + ); + } + }); } + //Drawing is finished + my.canvasesTimeouts[canvasNum] = null; } getFrequencies(callback) { @@ -322,7 +404,7 @@ export default class SpectrogramPlugin { let noverlap = this.noverlap; if (!noverlap) { - const uniqueSamplesPerPx = buffer.length / this.canvas.width; + const uniqueSamplesPerPx = buffer.length / this.width; noverlap = Math.max(0, Math.round(fftSamples - uniqueSamplesPerPx)); } @@ -461,12 +543,13 @@ export default class SpectrogramPlugin { updateScroll(e) { if (this.wrapper) { + this.scrollLeftTracker = e.target.scrollLeft; this.wrapper.scrollLeft = e.target.scrollLeft; } } resample(oldMatrix) { - const columnsNumber = this.width; + const columnsNumber = oldMatrix.length; const newMatrix = []; const oldPiece = 1 / oldMatrix.length; @@ -519,4 +602,12 @@ export default class SpectrogramPlugin { return newMatrix; } + + stretchCanvases() { + for (let i = 0; i < this.canvases.length; i++) { + this.canvases[i].style.width = Math.round(this.drawer.width / this.canvases.length / this.pixelRatio) + 'px'; + const canvasLeft = i * Math.floor(this.drawer.width / this.canvases.length / this.pixelRatio); + this.canvases[i].style['left'] = canvasLeft + 'px'; + } + } } diff --git a/src/wavesurfer.js b/src/wavesurfer.js index 94c69e932..a6c238ff0 100755 --- a/src/wavesurfer.js +++ b/src/wavesurfer.js @@ -1275,6 +1275,21 @@ export default class WaveSurfer extends util.Observer { this.drawBuffer(); } + /** + * Calls getPeaks() and drawPeaks() + * @param {WaveSurfer} wavesurfer the Wavesurfer to get/draw peaks from/to + * @param {number} width The width of the area that should be drawn + * @param {number} start The x-offset of the beginning of the area that + * should be rendered + * @param {number} end The x-offset of the end of the area that should be + * rendered + */ + getAndDrawPeaks(wavesurfer, width, start, end) { + let peaks; + peaks = wavesurfer.backend.getPeaks(width, start, end); + wavesurfer.drawer.drawPeaks(peaks, width, start, end); + } + /** * Get the correct peaks for current wave view-port and render wave * @@ -1356,6 +1371,24 @@ export default class WaveSurfer extends util.Observer { this.fireEvent('zoom', pxPerSec); } + /** + * Call this function while moving the zoom slider to stretch the canvases + * of the wave without recalculating + * + * @param {Number} pxPerSec value returned from the zoom slider + */ + zooming(pxPerSec) { + //Calculate the new width, this cannot be smaller than the parent container width + let desiredWidth = Math.round(this.getDuration() * pxPerSec * this.params.pixelRatio); + let parentWidth = this.drawer.getWidth(); + desiredWidth = Math.max(parentWidth, desiredWidth); + + //Stretch canvases + this.drawer.stretchCanvases(desiredWidth, this.backend.getPlayedPercents()); + + this.fireEvent('zoom', pxPerSec); + } + /** * Decode buffer and load *