New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Beat detection and Onset detection #34

Closed
b2renger opened this Issue Apr 13, 2015 · 7 comments

Comments

Projects
None yet
2 participants
@b2renger
Contributor

b2renger commented Apr 13, 2015

I've been experimenting with beat detection and onset detection lately and I've come up with two class based on the getEnergy method, the results are pretty encouraging even if it's pretty basic.

It may be a nice feature to add to the library. Let me know if you have feedback !

Here's an example using those, you'll need p5.dom :

var file ='GotanProject_Epoca.mp3'


var source_file; // sound file
var src_length; // hold its duration

var fft;

var pg; // to draw waveform

var playing = false;
var button;

// detectors
var onsetLow,beatLow;
var onsetLowMid,beatLowMid;
var onsetMid, beatMid;



function preload(){
    source_file = loadSound(file); // preload the sound
}

function setup() {
  createCanvas(windowWidth, 250);
  textAlign(CENTER);

  src_length = source_file.duration();
  source_file.playMode('restart'); 
  println("source duration: " +src_length);

  // draw the waveform to an off-screen graphic
  var peaks = source_file.getPeaks(); // get an array of peaks
  pg = createGraphics(width,150);
  pg.background(100);
  pg.translate(0,75);
  pg.noFill();
  pg.stroke(0);
  for (var i = 0 ; i < peaks.length ; i++){
        var x = map(i,0,peaks.length,0,width);
        var y = map(peaks[i],0,1,0,150);
          pg.line(x,0,x,y);
          pg.line(x,0,x,-y);
   }


    // FFT
   fft = new p5.FFT();

   // instanciation of onset and beat detection from fft
   // low band : 40Hz-120Hz
   onsetLow = new OnsetDetect(40,120,"bass",0.025);
   beatLow = new BeatDetect(40,120,"bass",0.7);
   // lowMid band : 140Hz-400Hz
   onsetLowMid = new OnsetDetect(140,400,"lowMid",0.025);
   beatLowMid = new BeatDetect(140,400,"lowMid",0.7);
   // mid band : 400Hz-2.6kHz
   onsetMid = new OnsetDetect(400,2600,"Mid",0.025);
   beatMid = new BeatDetect(400,2600,"Mid",0.7);


    // gui
   button = createButton('play');
   button.position(3, 3);
   button.mousePressed(play);
}


function draw() {
    background(180);

    image(pg,0,100); // display our waveform representation

     // draw playhead position 
    fill(255,255,180,150);
    noStroke();
    rect(map(source_file.currentTime(),0,src_length,0,windowWidth),100,3,150);
    //display current time
    text("current time: "+nfc(source_file.currentTime(),1)+" s",60,50);

    // we need to call fft.analyse() before the update functions of our class
    // this is because we use the getEnergy method inside our class.
    var spectrum = fft.analyze();  

        // display and update our detector objects
    text("onset detection",350,15);
    text("amplitude treshold",750,15);

    onsetLow.display(250,50);
    onsetLow.update(fft);

    beatLow.display(650,50);
    beatLow.update(fft);

    onsetLowMid.display(350,50);
    onsetLowMid.update(fft);

    beatLowMid.display(750,50);
    beatLowMid.update(fft);

    onsetMid.display(450,50);
    onsetMid.update(fft);

    beatMid.display(850,50);
    beatMid.update(fft);

    if (source_file.currentTime()>=src_length-0.05){
        source_file.pause();
    }

}

function mouseClicked(){
    if(mouseY>100 && mouseY<350){       
        var playpos = constrain(map(mouseX,0,windowWidth,0,src_length),0,src_length);   
        source_file.play(); 
        source_file.play(0,1,1,playpos,src_length); 
        playing = true;
        button.html('pause');       
    }   
    return false;//callback for p5js
}

function keyTyped(){
    if (key == ' '){
        play();
    }
    return false; // callback for p5js
}

function play(){
    if(playing){
        source_file.pause();
        button.html('play');
        playing = false;
    }
    else{
        source_file.play();
        button.html('pause');
        playing = true;
    }   
}


function OnsetDetect(f1,f2,str,thresh){
    this.isDetected = false;
    this.f1=f1;
    this.f2=f2;
    this.str = str;
    this.treshold = thresh;
    this.energy = 0;
    this.penergy =0;
    this.siz = 10;
    this.sensitivity = 500;
}

OnsetDetect.prototype.display = function(x,y) {

    if(this.isDetected == true){
        this.siz = lerp(this.siz,40,0.99);
    }
    else if (this.isDetected == false){
        this.siz = lerp(this.siz,15,0.99);
    }
    fill(255,0,0);
    ellipse(x,y,this.siz,this.siz);
    fill(0);
    text(this.str,x,y);
    text("( "+this.f1+" - "+this.f2+"Hz )",x,y+10);
}

OnsetDetect.prototype.update = function(fftObject) {
    this.energy = fftObject.getEnergy(this.f1,this.f2)/255;

    if(this.isDetected == false){
        if (this.energy-this.penergy > this.treshold){
            this.isDetected = true;
            var self = this;
            setTimeout(function () {
                self.isDetected = false;
            },this.sensitivity);
        }
    }

    this.penergy = this.energy;

}


function BeatDetect(f1,f2,str,thresh){
    this.isDetected = false;
    this.f1=f1;
    this.f2=f2;
    this.str = str;
    this.treshold = thresh;
    this.energy = 0;

    this.siz = 10;
    this.sensitivity = 500;
}

BeatDetect.prototype.display = function(x,y) {

    if(this.isDetected == true){
        this.siz = lerp(this.siz,40,0.99);
    }
    else if (this.isDetected == false){
        this.siz = lerp(this.siz,15,0.99);
    }
    fill(255,0,0);
    ellipse(x,y,this.siz,this.siz);
    fill(0);
    text(this.str,x,y);
    text("( "+this.f1+" - "+this.f2+"Hz )",x,y+10);
}

BeatDetect.prototype.update = function(fftObject) {
    this.energy = fftObject.getEnergy(this.f1,this.f2)/255;

    if(this.isDetected == false){
        if (this.energy > this.treshold){
            this.isDetected = true;
            var self = this;
            setTimeout(function () {
                self.isDetected = false;
            },this.sensitivity);
        }
    }   
}
@therewasaguy

This comment has been minimized.

Show comment
Hide comment
@therewasaguy

therewasaguy Apr 13, 2015

Member

@b2renger this is fantastic, thank you!! I've been hoping to add these types of features and your example is already an excellent start.

I'd like to turn these into a Class, or possibly multiple Classes. This would involve separating out their display methods.

For BeatDetect, it might be a useful option be able to provide a bpm estimate/range, which would default to something like 120, but could be customized. This would determine how often the BeatDetect can register a new beat. It might also be useful to add a beat cutoff parameter, which is set to just slightly greater than the threshold when a beat is detected. Then, a decay parameter can determines how fast the cutoff reverts to the original threshold level. That way, if there is a beat followed by a much louder beat, it can register both.

Related to beatDetect, I'd like to add a feature that can pre-analyze a SoundFIle and return JSON data. It'd use getPeaks() and a technique like this. I'm not sure if this would be a method of a BeatDetect class, or of p5.SoundFIle.

I'm going to play around with this all some more soon. In the meantime, if you'd like to add your example-in-progress to the examples folder, feel free to submit a PR.

Member

therewasaguy commented Apr 13, 2015

@b2renger this is fantastic, thank you!! I've been hoping to add these types of features and your example is already an excellent start.

I'd like to turn these into a Class, or possibly multiple Classes. This would involve separating out their display methods.

For BeatDetect, it might be a useful option be able to provide a bpm estimate/range, which would default to something like 120, but could be customized. This would determine how often the BeatDetect can register a new beat. It might also be useful to add a beat cutoff parameter, which is set to just slightly greater than the threshold when a beat is detected. Then, a decay parameter can determines how fast the cutoff reverts to the original threshold level. That way, if there is a beat followed by a much louder beat, it can register both.

Related to beatDetect, I'd like to add a feature that can pre-analyze a SoundFIle and return JSON data. It'd use getPeaks() and a technique like this. I'm not sure if this would be a method of a BeatDetect class, or of p5.SoundFIle.

I'm going to play around with this all some more soon. In the meantime, if you'd like to add your example-in-progress to the examples folder, feel free to submit a PR.

@b2renger

This comment has been minimized.

Show comment
Hide comment
@b2renger

b2renger Apr 13, 2015

Contributor

To add a few thougts to the design of the class, it could be nice to have an event Emitter in a more js style like kind of code ?

I have a sensitivity value that acts on the setTimeout delay to be able to filter multiple 'too close' detections. So you can try to play with it with several tracks and see how it goes I didn't test it that much I wanted the concept working !

We need an electronica kind of sample for those :) do you have something in mind ?

the cutoff feature sounds nice :)

for the tempo detection I actually did this from the same article it works nicely with electronica sounds :) it could be nice to have a class that would be able to pre-analyse a sound file (waveform, spectrogram , tempo detection, beats in several band of frequencies etc...)

let me share the code (mostly not mine anyhow), I've stripped it out of a bigger example but it should work almost right away

var file ='03 - Gotan Project - Epoca.mp3'

var source_file; // sound file
var src_length; // hold its duration

var pg_beats; // to draw the beat detection on preprocessed song
var pg_tempo; // draw our guessed tempi

function preload(){
    source_file = loadSound(file); // preload the sound
   // clave = loadSound('cl.wav');
}


function setup() {
  createCanvas(windowWidth, 800);
  textAlign(CENTER);

  src_length = source_file.duration();
  source_file.playMode('restart'); 
  println("source duration: " +src_length);

   // prepare our tempo display
   pg_tempo = createGraphics(width,100);
   pg_tempo.background(100);
   pg_tempo.strokeWeight(3);
   pg_tempo.noFill();
   pg_tempo.stroke(255);
   pg_tempo.rect(0,0,width,100);
   pg_tempo.strokeWeight(1);
   //prepare our beats graph display
   pg_beats = createGraphics(width,50);
   pg_beats.background(100);
   pg_beats.strokeWeight(3);
   pg_beats.noFill();
   pg_beats.stroke(255);
   pg_beats.rect(0,0,width,50);
   pg_beats.strokeWeight(1);

   // find beat preprocessing the source file with lowpass
   preprocess(); // it will draw in pg_beats and pg_tempo
}

function draw() {
    background(180);

    image(pg_tempo,0,0); // display detected tempi
    image(pg_beats,0,100); // display filtered beats we found preprocesing with a lp filter
}

function drawPeaksAtTreshold(data, threshold) {
  for(var i = 0; i <  data.length;) {   
    if (data[i] > threshold) {
      var xpos = map(i,0,data.length,0,width);
      var intensity = map(data[i],threshold,0.3,0,255);
      var hei = map(data[i],threshold,0.3,3,50);
      pg_beats.stroke(intensity,0,0);
      pg_beats.line(xpos,3,xpos,47);
      // Skip forward ~ 1/8s to get past this peak.
      i += 6000; //////////////////////////////////////////////////////////////////////// attention magic number !!!
    }
    i++;
  }
}

// Function to identify peaks
function getPeaksAtThreshold(data, threshold) {
  var peaksArray = [];
  var length = data.length;
  for(var i = 0; i < length;) {
    if (data[i] > threshold) {
      peaksArray.push(i);
      // Skip forward ~ 1/8s to get past this peak.
      i += 6000; //////////////////////////////////////////////////////////////////////// attention magic number !!!
    }
    i++;
  }
  return peaksArray;
}

// Function used to return a histogram of peak intervals
function countIntervalsBetweenNearbyPeaks(peaks) {
  var intervalCounts = [];
  peaks.forEach(function(peak, index) {
    for(var i = 0; i < 10; i++) {
      var interval = peaks[index + i] - peak;
      var foundInterval = intervalCounts.some(function(intervalCount) {
        if (intervalCount.interval === interval)
          return intervalCount.count++;
      });
      // store with JSson like formatting
      if (!foundInterval) {
        intervalCounts.push({
          interval: interval,
          count: 1
        });
      }
    }
  });
  return intervalCounts;
}

// Function used to return a histogram of tempo candidates.
function groupNeighborsByTempo(intervalCounts, sampleRate) {
  var tempoCounts = [];
  intervalCounts.forEach(function(intervalCount, i) {
    if (intervalCount.interval !== 0) {
      // Convert an interval to tempo
      var theoreticalTempo = 60 / (intervalCount.interval / sampleRate );
      // Adjust the tempo to fit within the 90-180 BPM range
      while (theoreticalTempo < 90) theoreticalTempo *= 2;
      while (theoreticalTempo > 180) theoreticalTempo /= 2;
      theoreticalTempo = Math.round(theoreticalTempo);
      var foundTempo = tempoCounts.some(function(tempoCount) {
        if (tempoCount.tempo === theoreticalTempo)
          return tempoCount.count += intervalCount.count;
      });
      // store with Json like formating
      if (!foundTempo) {
        tempoCounts.push({
          tempo: theoreticalTempo,
          count: intervalCount.count
        });
      }
    }
  });
  return tempoCounts;
}


function preprocess(){
    // Create offline context
    var offlineContext = new OfflineAudioContext(1, source_file.buffer.length, source_file.buffer.sampleRate);
    // Create buffer source
    var source = offlineContext.createBufferSource();
    source.buffer = source_file.buffer; // copy from source file
    // Create filter
    var filter = offlineContext.createBiquadFilter();
    filter.type = "lowpass";
    source.connect(filter);
    filter.connect(offlineContext.destination);
    // start playing at time:0
    source.start(0);    
    offlineContext.startRendering(); // Render the song

    // Act on the result
    offlineContext.oncomplete = function(e) {
        // Filtered buffer!
        var filteredBuffer = e.renderedBuffer;
        var lowpeaks = drawPeaksAtTreshold(e.renderedBuffer.getChannelData(0), 0.25); // store and draw peaks

        // get tempo on a subset of the sample
        var peaks,
                initialThresold = 0.9,
                thresold = initialThresold,
                minThresold = 0.22,
                minPeaks = 200;

            do {
              peaks = getPeaksAtThreshold(e.renderedBuffer.getChannelData(0), thresold);
              thresold -= 0.005;
            } while (peaks.length < minPeaks && thresold >= minThresold);
         // find and group intervals
         var intervals = countIntervalsBetweenNearbyPeaks(peaks);
         var groups = groupNeighborsByTempo(intervals, filteredBuffer.sampleRate);
         // sort top intervals
         var top = groups.sort(function(intA, intB) {
              return intB.count - intA.count;
            }).splice(0,5);
         // dsplay them
         for (var i = 0 ; i < top.length; i++){
            pg_tempo.noStroke();
            pg_tempo.fill(255,255,180);
            pg_tempo.textSize(12);
            pg_tempo.text("Tempo n"+i+" : " + int(top[i].tempo) + "bpm  /  found " +top[i].count +" times.", width-220,15+i*18);
         }
    };
}
Contributor

b2renger commented Apr 13, 2015

To add a few thougts to the design of the class, it could be nice to have an event Emitter in a more js style like kind of code ?

I have a sensitivity value that acts on the setTimeout delay to be able to filter multiple 'too close' detections. So you can try to play with it with several tracks and see how it goes I didn't test it that much I wanted the concept working !

We need an electronica kind of sample for those :) do you have something in mind ?

the cutoff feature sounds nice :)

for the tempo detection I actually did this from the same article it works nicely with electronica sounds :) it could be nice to have a class that would be able to pre-analyse a sound file (waveform, spectrogram , tempo detection, beats in several band of frequencies etc...)

let me share the code (mostly not mine anyhow), I've stripped it out of a bigger example but it should work almost right away

var file ='03 - Gotan Project - Epoca.mp3'

var source_file; // sound file
var src_length; // hold its duration

var pg_beats; // to draw the beat detection on preprocessed song
var pg_tempo; // draw our guessed tempi

function preload(){
    source_file = loadSound(file); // preload the sound
   // clave = loadSound('cl.wav');
}


function setup() {
  createCanvas(windowWidth, 800);
  textAlign(CENTER);

  src_length = source_file.duration();
  source_file.playMode('restart'); 
  println("source duration: " +src_length);

   // prepare our tempo display
   pg_tempo = createGraphics(width,100);
   pg_tempo.background(100);
   pg_tempo.strokeWeight(3);
   pg_tempo.noFill();
   pg_tempo.stroke(255);
   pg_tempo.rect(0,0,width,100);
   pg_tempo.strokeWeight(1);
   //prepare our beats graph display
   pg_beats = createGraphics(width,50);
   pg_beats.background(100);
   pg_beats.strokeWeight(3);
   pg_beats.noFill();
   pg_beats.stroke(255);
   pg_beats.rect(0,0,width,50);
   pg_beats.strokeWeight(1);

   // find beat preprocessing the source file with lowpass
   preprocess(); // it will draw in pg_beats and pg_tempo
}

function draw() {
    background(180);

    image(pg_tempo,0,0); // display detected tempi
    image(pg_beats,0,100); // display filtered beats we found preprocesing with a lp filter
}

function drawPeaksAtTreshold(data, threshold) {
  for(var i = 0; i <  data.length;) {   
    if (data[i] > threshold) {
      var xpos = map(i,0,data.length,0,width);
      var intensity = map(data[i],threshold,0.3,0,255);
      var hei = map(data[i],threshold,0.3,3,50);
      pg_beats.stroke(intensity,0,0);
      pg_beats.line(xpos,3,xpos,47);
      // Skip forward ~ 1/8s to get past this peak.
      i += 6000; //////////////////////////////////////////////////////////////////////// attention magic number !!!
    }
    i++;
  }
}

// Function to identify peaks
function getPeaksAtThreshold(data, threshold) {
  var peaksArray = [];
  var length = data.length;
  for(var i = 0; i < length;) {
    if (data[i] > threshold) {
      peaksArray.push(i);
      // Skip forward ~ 1/8s to get past this peak.
      i += 6000; //////////////////////////////////////////////////////////////////////// attention magic number !!!
    }
    i++;
  }
  return peaksArray;
}

// Function used to return a histogram of peak intervals
function countIntervalsBetweenNearbyPeaks(peaks) {
  var intervalCounts = [];
  peaks.forEach(function(peak, index) {
    for(var i = 0; i < 10; i++) {
      var interval = peaks[index + i] - peak;
      var foundInterval = intervalCounts.some(function(intervalCount) {
        if (intervalCount.interval === interval)
          return intervalCount.count++;
      });
      // store with JSson like formatting
      if (!foundInterval) {
        intervalCounts.push({
          interval: interval,
          count: 1
        });
      }
    }
  });
  return intervalCounts;
}

// Function used to return a histogram of tempo candidates.
function groupNeighborsByTempo(intervalCounts, sampleRate) {
  var tempoCounts = [];
  intervalCounts.forEach(function(intervalCount, i) {
    if (intervalCount.interval !== 0) {
      // Convert an interval to tempo
      var theoreticalTempo = 60 / (intervalCount.interval / sampleRate );
      // Adjust the tempo to fit within the 90-180 BPM range
      while (theoreticalTempo < 90) theoreticalTempo *= 2;
      while (theoreticalTempo > 180) theoreticalTempo /= 2;
      theoreticalTempo = Math.round(theoreticalTempo);
      var foundTempo = tempoCounts.some(function(tempoCount) {
        if (tempoCount.tempo === theoreticalTempo)
          return tempoCount.count += intervalCount.count;
      });
      // store with Json like formating
      if (!foundTempo) {
        tempoCounts.push({
          tempo: theoreticalTempo,
          count: intervalCount.count
        });
      }
    }
  });
  return tempoCounts;
}


function preprocess(){
    // Create offline context
    var offlineContext = new OfflineAudioContext(1, source_file.buffer.length, source_file.buffer.sampleRate);
    // Create buffer source
    var source = offlineContext.createBufferSource();
    source.buffer = source_file.buffer; // copy from source file
    // Create filter
    var filter = offlineContext.createBiquadFilter();
    filter.type = "lowpass";
    source.connect(filter);
    filter.connect(offlineContext.destination);
    // start playing at time:0
    source.start(0);    
    offlineContext.startRendering(); // Render the song

    // Act on the result
    offlineContext.oncomplete = function(e) {
        // Filtered buffer!
        var filteredBuffer = e.renderedBuffer;
        var lowpeaks = drawPeaksAtTreshold(e.renderedBuffer.getChannelData(0), 0.25); // store and draw peaks

        // get tempo on a subset of the sample
        var peaks,
                initialThresold = 0.9,
                thresold = initialThresold,
                minThresold = 0.22,
                minPeaks = 200;

            do {
              peaks = getPeaksAtThreshold(e.renderedBuffer.getChannelData(0), thresold);
              thresold -= 0.005;
            } while (peaks.length < minPeaks && thresold >= minThresold);
         // find and group intervals
         var intervals = countIntervalsBetweenNearbyPeaks(peaks);
         var groups = groupNeighborsByTempo(intervals, filteredBuffer.sampleRate);
         // sort top intervals
         var top = groups.sort(function(intA, intB) {
              return intB.count - intA.count;
            }).splice(0,5);
         // dsplay them
         for (var i = 0 ; i < top.length; i++){
            pg_tempo.noStroke();
            pg_tempo.fill(255,255,180);
            pg_tempo.textSize(12);
            pg_tempo.text("Tempo n"+i+" : " + int(top[i].tempo) + "bpm  /  found " +top[i].count +" times.", width-220,15+i*18);
         }
    };
}
@therewasaguy

This comment has been minimized.

Show comment
Hide comment
@therewasaguy

therewasaguy Apr 15, 2015

Member

That's great you referenced the same article! Web audio's a small world... but growing. I'll check in with the author about building off his example code.

Event emitters are a good call—we need more events in p5.sound in general, and less relying on the animation/draw loop.

It'd be nice to offer a callback onBeat, that fires when isDetected goes from false to true. Beyond that, I'm not sure how much good events would do for live analysis, since it won't work unless we tell the FFT to update during the draw loop, and pass that data to the Detectors.

I've started working on the ability to schedule events to audio playback. So if we pre-analyze a soundfile, there can be an event on every beat.

Member

therewasaguy commented Apr 15, 2015

That's great you referenced the same article! Web audio's a small world... but growing. I'll check in with the author about building off his example code.

Event emitters are a good call—we need more events in p5.sound in general, and less relying on the animation/draw loop.

It'd be nice to offer a callback onBeat, that fires when isDetected goes from false to true. Beyond that, I'm not sure how much good events would do for live analysis, since it won't work unless we tell the FFT to update during the draw loop, and pass that data to the Detectors.

I've started working on the ability to schedule events to audio playback. So if we pre-analyze a soundfile, there can be an event on every beat.

@b2renger

This comment has been minimized.

Show comment
Hide comment
@b2renger

b2renger May 27, 2015

Contributor

submitted an example to be able to work on it !

Contributor

b2renger commented May 27, 2015

submitted an example to be able to work on it !

@therewasaguy

This comment has been minimized.

Show comment
Hide comment
@therewasaguy

therewasaguy May 28, 2015

Member

@b2renger's offlinePeakDetection example allows you to get peaks from a soundfile above a certain threshold, a sort of beat detection with a good guess at the tempo. This would be a great feature to add to the p5.SoundFile class.

But there is already a method p5.SoundFile.getPeaks(). It returns an array of peaks useful for displaying the waveform.

Should we change the name of that method to distinguish it from the new Peaks functionality that looks for peaks above a certain threshold?

Member

therewasaguy commented May 28, 2015

@b2renger's offlinePeakDetection example allows you to get peaks from a soundfile above a certain threshold, a sort of beat detection with a good guess at the tempo. This would be a great feature to add to the p5.SoundFile class.

But there is already a method p5.SoundFile.getPeaks(). It returns an array of peaks useful for displaying the waveform.

Should we change the name of that method to distinguish it from the new Peaks functionality that looks for peaks above a certain threshold?

@b2renger

This comment has been minimized.

Show comment
Hide comment
@b2renger

b2renger May 28, 2015

Contributor

hum ... maybe a getWaveform() for display ? and getPeaks() for actual beat detection ?

Contributor

b2renger commented May 28, 2015

hum ... maybe a getWaveform() for display ? and getPeaks() for actual beat detection ?

@b2renger

This comment has been minimized.

Show comment
Hide comment
@b2renger

b2renger Jun 4, 2015

Contributor

considering the contributions from community convention, I guess we can consider this as closed ! Nice work guys !

Contributor

b2renger commented Jun 4, 2015

considering the contributions from community convention, I guess we can consider this as closed ! Nice work guys !

@b2renger b2renger closed this Jun 4, 2015

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment