From be7209be8e5e23412827d444e539a1462b8387e0 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 17:34:21 +0100 Subject: [PATCH 01/22] add skeleton of bufnoveltyfeature --- doc/BufNoveltyFeature.rst | 104 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 doc/BufNoveltyFeature.rst diff --git a/doc/BufNoveltyFeature.rst b/doc/BufNoveltyFeature.rst new file mode 100644 index 0000000..9e41e25 --- /dev/null +++ b/doc/BufNoveltyFeature.rst @@ -0,0 +1,104 @@ +:digest: Buffer-Based Novelty-Based Slicer +:species: buffer-proc +:sc-categories: Libraries>FluidDecomposition, UGens>Buffer +:sc-related: Guides/FluidCorpusManipulationToolkit +:see-also: NoveltySlice, BufAmpSlice, BufOnsetSlice, BufTransientSlice +:description: A non-real-time slicer using an algorithm assessing novelty in the signal to estimate the slicing points. +:discussion: + A novelty curve is derived from running a kernel across the diagonal of the similarity matrix, and looking for peak of changes. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. + + The process will return a buffer which contains indices (in sample) of estimated starting points of different slices. + +:process: This is the method that calls for the slicing to be calculated on a given source buffer. +:output: Nothing, as the various destination buffers are declared in the function call. + + +:control source: + + The index of the buffer to use as the source material to be sliced through novelty identification. The different channels of multichannel buffers will be summed. + +:control startFrame: + + Where in the srcBuf should the slicing process start, in sample. + +:control numFrames: + + How many frames should be processed. + +:control startChan: + + For multichannel srcBuf, which channel should be processed. + +:control numChans: + + For multichannel srcBuf, how many channel should be summed. + +:control indices: + + The index of the buffer where the indices (in sample) of the estimated starting points of slices will be written. The first and last points are always the boundary points of the analysis. + +:control feature: + + The feature on which novelty is computed. + + :enum: + + :0: + Spectrum – The magnitude of the full spectrum. + + :1: + MFCC – 13 Mel-Frequency Cepstrum Coefficients. + + :2: + Chroma - The contour of a 12-band chromagram. + + :3: + Pitch – The pitch and its confidence. + + :4: + Loudness – The true peak and loudness. + +:control kernelSize: + + The granularity of the window in which the algorithm looks for change, in samples. A small number will be sensitive to short term changes, and a large number should look for long term changes. + +:control threshold: + + The normalised threshold, between 0 an 1, on the novelty curve to consider it a segmentation point. + +:control filterSize: + + The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes. + +:control minSliceLength: + + The minimum duration of a slice in number of hopSize. + +:control windowSize: + + The window size. As novelty estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty + +:control hopSize: + + The window hop size. As novelty estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. + +:control fftSize: + + The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. + +:control maxFFTSize: + + How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated. + +:control maxKernelSize: + + This cannot be modulated. + +:control maxFilterSize: + + This cannot be modulated. + +:control action: + + A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The function will be passed indices as an argument. + From ab6a833ea2cb585f8ccb0fc8498fc10beae10321 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:11:20 +0100 Subject: [PATCH 02/22] caveat novelty feature --- doc/BufNoveltyFeature.rst | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/doc/BufNoveltyFeature.rst b/doc/BufNoveltyFeature.rst index 9e41e25..ead0504 100644 --- a/doc/BufNoveltyFeature.rst +++ b/doc/BufNoveltyFeature.rst @@ -1,18 +1,17 @@ -:digest: Buffer-Based Novelty-Based Slicer +:digest: Buffer-Based Novelty Feature :species: buffer-proc :sc-categories: Libraries>FluidDecomposition, UGens>Buffer :sc-related: Guides/FluidCorpusManipulationToolkit -:see-also: NoveltySlice, BufAmpSlice, BufOnsetSlice, BufTransientSlice -:description: A non-real-time slicer using an algorithm assessing novelty in the signal to estimate the slicing points. +:see-also: BufNoveltySlice, BufNoveltyFeature, BufAmpFeature, BufOnsetFeature +:description: A non-realtime that calculates novelty of audio stored in a buffer. Used by :fluid-obj:`BufNoveltySlice`. :discussion: - A novelty curve is derived from running a kernel across the diagonal of the similarity matrix, and looking for peak of changes. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. + Novelty is derived by running a kernel across the diagonal of the similarity matrix. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. - The process will return a buffer which contains indices (in sample) of estimated starting points of different slices. + The process will return a buffer containing a time series that describes the novelty feature changing over time in the source buffer. :process: This is the method that calls for the slicing to be calculated on a given source buffer. :output: Nothing, as the various destination buffers are declared in the function call. - :control source: The index of the buffer to use as the source material to be sliced through novelty identification. The different channels of multichannel buffers will be summed. @@ -33,9 +32,9 @@ For multichannel srcBuf, how many channel should be summed. -:control indices: +:control features: - The index of the buffer where the indices (in sample) of the estimated starting points of slices will be written. The first and last points are always the boundary points of the analysis. + The index of the buffer where the novelty feature will be written. :control feature: @@ -62,21 +61,13 @@ The granularity of the window in which the algorithm looks for change, in samples. A small number will be sensitive to short term changes, and a large number should look for long term changes. -:control threshold: - - The normalised threshold, between 0 an 1, on the novelty curve to consider it a segmentation point. - :control filterSize: The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes. -:control minSliceLength: - - The minimum duration of a slice in number of hopSize. - :control windowSize: - The window size. As novelty estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty + The window size. As novelty estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally. :control hopSize: From a62d7d7de85b1e9579d6057342a9b8ac652bb286 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:11:24 +0100 Subject: [PATCH 03/22] add onset feature --- doc/BufOnsetFeature.rst | 102 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 doc/BufOnsetFeature.rst diff --git a/doc/BufOnsetFeature.rst b/doc/BufOnsetFeature.rst new file mode 100644 index 0000000..af21638 --- /dev/null +++ b/doc/BufOnsetFeature.rst @@ -0,0 +1,102 @@ +:digest: Buffer-Based Spectral Difference Feature +:species: buffer-proc +:sc-categories: Libraries>FluidDecomposition +:sc-related: Guides/FluidCorpusManipulationToolkit +:see-also: BufOnsetSlice, BufNoveltyFeature, BufAmpFeature, BufOnsetFeature +:description: Calculate the spectral features used by :fluid-obj:`BufOnsetSlice`. +:discussion: + Calculates the feature used by :fluid-obj:`BufOnsetSlice` in its slicing and copies it to a buffer. + + The metric for calculating difference can be chosen from a curated selection, lending the algorithm toward slicing a broad range of musical materials. + +:process: This is the method that calls for the slicing to be calculated on a given source buffer. +:output: Nothing, as the various destination buffers are declared in the function call. + +:control source: + + The index of the buffer to use as the source material to be sliced through novelty identification. The different channels of multichannel buffers will be summed. + +:control startFrame: + + Where in the srcBuf should the slicing process start, in sample. + +:control numFrames: + + How many frames should be processed. + +:control startChan: + + For multichannel sources, which channel should be processed. + +:control numChans: + + For multichannel sources, how many channel should be summed. + +:control features: + + The index of the buffer where the onset features will be written to. + +:control metric: + + The metric used to derive a difference curve between spectral frames. It can be any of the following: + + :enum: + + :0: + **Energy** thresholds on (sum of squares of magnitudes / nBins) (like Onsets \power) + + :1: + **HFC** thresholds on (sum of (squared magnitudes * binNum) / nBins) + + :2: + **SpectralFlux** thresholds on (diffence in magnitude between consecutive frames, half rectified) + + :3: + **MKL** thresholds on (sum of log of magnitude ratio per bin) (or equivalently, sum of difference of the log magnitude per bin) (like Onsets mkl) + + :4: + **IS** (WILL PROBABLY BE REMOVED) Itakura - Saito divergence (see literature) + + :5: + **Cosine** thresholds on (cosine distance between comparison frames) + + :6: + **PhaseDev** takes the past 2 frames, projects to the current, as anticipated if it was a steady state, then compute the sum of the differences, on which it thresholds (like Onsets \phase) + + :7: + **WPhaseDev** same as PhaseDev, but weighted by the magnitude in order to remove chaos noise floor (like Onsets \wphase) + + :8: + **ComplexDev** same as PhaseDev, but in the complex domain - the anticipated amp is considered steady, and the phase is projected, then a complex subtraction is done with the actual present frame. The sum of magnitudes is used to threshold (like Onsets \complex) + + :9: + **RComplexDev** same as above, but rectified (like Onsets \rcomplex) + +:control filterSize: + + The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes. + +:control frameDelta: + + For certain metrics (HFC, SpectralFlux, MKL, Cosine), the distance does not have to be computed between consecutive frames. By default (0) it is, otherwise this sets the distane between the comparison window in samples. + +:control windowSize: + + The window size. As spectral differencing relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty + +:control hopSize: + + The window hop size. As spectral differencing relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2). + +:control fftSize: + + The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to windowSize. + +:control maxFFTSize: + + How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated. + +:control action: + + A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The function will be passed indices as an argument. + From 6869bc2826403ec14c27dbbab3655f9d4a5a60de Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:17:37 +0100 Subject: [PATCH 04/22] remove space --- doc/BufOnsetFeature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/BufOnsetFeature.rst b/doc/BufOnsetFeature.rst index af21638..d2ffb6a 100644 --- a/doc/BufOnsetFeature.rst +++ b/doc/BufOnsetFeature.rst @@ -1,4 +1,4 @@ -:digest: Buffer-Based Spectral Difference Feature +:digest: Buffer-Based Spectral Difference Feature :species: buffer-proc :sc-categories: Libraries>FluidDecomposition :sc-related: Guides/FluidCorpusManipulationToolkit From 39c1584c1264ace71748dcdedf6895ab7b4a9f16 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:17:41 +0100 Subject: [PATCH 05/22] add bufampfeature --- doc/BufAmpFeature.rst | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 doc/BufAmpFeature.rst diff --git a/doc/BufAmpFeature.rst b/doc/BufAmpFeature.rst new file mode 100644 index 0000000..366387c --- /dev/null +++ b/doc/BufAmpFeature.rst @@ -0,0 +1,61 @@ +:digest: Buffer-Based Amplitude Differential Feature +:species: buffer-proc +:sc-categories: Libraries>FluidDecomposition +:sc-related: Guides/FluidCorpusManipulationToolkit +:see-also: AmpGate, BufAmpSlice, BufOnsetSlice, BufNoveltySlice, BufTransientSlice +:description: Calculate the amplitude differential feature used by both :fluid-obj:`BufAmpSlice`. +:discussion: + :fluid-obj:`BufAmpSlice` uses the differential of a fast and a slow envelope follower to determine changes in amplitude. This object calculates the amplitude differential and copies to an output buffer. + +:output: Nothing, as the destination buffer is declared in the function call. + + +:control source: + + The index of the buffer to use as the source material to be sliced through novelty identification. The different channels of multichannel buffers will be summed. + +:control startFrame: + + Where in the srcBuf should the slicing process start, in sample. + +:control numFrames: + + How many frames should be processed. + +:control startChan: + + For multichannel sources, which channel should be processed. + +:control numChans: + + For multichannel sources, how many channel should be summed. + +:control features: + + The index of the buffer where the amplitude differential feature will be copied to. + +:control fastRampUp: + + The number of samples the relative envelope follower will take to reach the next value when raising. Typically, this will be faster than slowRampUp. + +:control fastRampDown: + + The number of samples the relative envelope follower will take to reach the next value when falling. Typically, this will be faster than slowRampDown. + +:control slowRampUp: + + The number of samples the absolute envelope follower will take to reach the next value when raising. + +:control slowRampDown: + + The number of samples the absolute envelope follower will take to reach the next value when falling. + +:control floor: + + The level in dB the slowRamp needs to be above to consider a detected difference valid, allowing to ignore the slices in the noise floor. + +:control highPassFreq: + + The frequency of the fourth-order Linkwitz–Riley high-pass filter (https://en.wikipedia.org/wiki/Linkwitz%E2%80%93Riley_filter). This is done first on the signal to minimise low frequency intermodulation with very fast ramp lengths. A frequency of 0 bypasses the filter. + + From 14ad993a8b45a314ae5336018f318d6c4d020913 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:30:41 +0100 Subject: [PATCH 06/22] realtime amp feature --- doc/AmpFeature.rst | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 doc/AmpFeature.rst diff --git a/doc/AmpFeature.rst b/doc/AmpFeature.rst new file mode 100644 index 0000000..ea9d6ca --- /dev/null +++ b/doc/AmpFeature.rst @@ -0,0 +1,41 @@ +:digest: Realtime Amplitude Differential Feature +:species: descriptor +:sc-categories: Libraries>FluidDecomposition +:sc-related: Guides/FluidCorpusManipulationToolkit +:see-also: AmpGate, BufAmpSlice, BufOnsetSlice, BufNoveltySlice, BufTransientSlice +:description: Calculate the amplitude differential feature used by :fluid-obj:`BufAmpSlice`. +:discussion: + :fluid-obj:`BufAmpSlice` uses the differential between a fast and a slow envelope follower to determine changes in amplitude. This object calculates the amplitude differential and copies it to an output buffer. + +:process: The audio rate in, control rate out version of the object. +:output: The featurea as a signal. + +:control in: + + The audio to be processed. + +:control fastRampUp: + + The number of samples the relative envelope follower will take to reach the next value when raising. Typically, this will be faster than slowRampUp. + +:control fastRampDown: + + The number of samples the relative envelope follower will take to reach the next value when falling. Typically, this will be faster than slowRampDown. + +:control slowRampUp: + + The number of samples the absolute envelope follower will take to reach the next value when raising. + +:control slowRampDown: + + The number of samples the absolute envelope follower will take to reach the next value when falling. + +:control floor: + + The level in dB the slowRamp needs to be above to consider a detected difference valid, allowing to ignore the slices in the noise floor. + +:control highPassFreq: + + The frequency of the fourth-order Linkwitz–Riley high-pass filter (https://en.wikipedia.org/wiki/Linkwitz%E2%80%93Riley_filter). This is done first on the signal to minimise low frequency intermodulation with very fast ramp lengths. A frequency of 0 bypasses the filter. + + From 2430b56026ae283f2bc581c2ef138fe4ddbd3e19 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:32:16 +0100 Subject: [PATCH 07/22] update output of AmpFeature --- doc/AmpFeature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/AmpFeature.rst b/doc/AmpFeature.rst index ea9d6ca..0df1e67 100644 --- a/doc/AmpFeature.rst +++ b/doc/AmpFeature.rst @@ -8,7 +8,7 @@ :fluid-obj:`BufAmpSlice` uses the differential between a fast and a slow envelope follower to determine changes in amplitude. This object calculates the amplitude differential and copies it to an output buffer. :process: The audio rate in, control rate out version of the object. -:output: The featurea as a signal. +:output: A KR signal of the feature. :control in: From 35133c4beb889fc54144caa4fe87af3f02108dda Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:32:28 +0100 Subject: [PATCH 08/22] clean up the language --- doc/BufAmpFeature.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/BufAmpFeature.rst b/doc/BufAmpFeature.rst index 366387c..a17b9c4 100644 --- a/doc/BufAmpFeature.rst +++ b/doc/BufAmpFeature.rst @@ -2,10 +2,10 @@ :species: buffer-proc :sc-categories: Libraries>FluidDecomposition :sc-related: Guides/FluidCorpusManipulationToolkit -:see-also: AmpGate, BufAmpSlice, BufOnsetSlice, BufNoveltySlice, BufTransientSlice -:description: Calculate the amplitude differential feature used by both :fluid-obj:`BufAmpSlice`. +:see-also: BufAmpSlice, BufNoveltyFeature, BufAmpFeature, BufOnsetFeature +:description: Calculate the amplitude differential feature used by :fluid-obj:`BufAmpSlice`. :discussion: - :fluid-obj:`BufAmpSlice` uses the differential of a fast and a slow envelope follower to determine changes in amplitude. This object calculates the amplitude differential and copies to an output buffer. + :fluid-obj:`BufAmpSlice` uses the differential between a fast and a slow envelope follower to determine changes in amplitude. This object calculates the amplitude differential and copies it to an output buffer. :output: Nothing, as the destination buffer is declared in the function call. From 470244b3f19379b73891fc7aa9ad0ae210bbb1a1 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:39:27 +0100 Subject: [PATCH 09/22] caveats for bufonset feature --- doc/BufOnsetFeature.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/BufOnsetFeature.rst b/doc/BufOnsetFeature.rst index d2ffb6a..4e15588 100644 --- a/doc/BufOnsetFeature.rst +++ b/doc/BufOnsetFeature.rst @@ -3,9 +3,9 @@ :sc-categories: Libraries>FluidDecomposition :sc-related: Guides/FluidCorpusManipulationToolkit :see-also: BufOnsetSlice, BufNoveltyFeature, BufAmpFeature, BufOnsetFeature -:description: Calculate the spectral features used by :fluid-obj:`BufOnsetSlice`. +:description: Calculate the spectral difference feature used by :fluid-obj:`BufOnsetSlice`. :discussion: - Calculates the feature used by :fluid-obj:`BufOnsetSlice` in its slicing and copies it to a buffer. + Given a source buffer, calculates the feature used by :fluid-obj:`BufOnsetSlice` and copies it to another buffer. The metric for calculating difference can be chosen from a curated selection, lending the algorithm toward slicing a broad range of musical materials. From c31be0b1e23b045b4cdee9d25b55e62e675bb7b4 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:40:24 +0100 Subject: [PATCH 10/22] add onset feature --- doc/OnsetFeature.rst | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 doc/OnsetFeature.rst diff --git a/doc/OnsetFeature.rst b/doc/OnsetFeature.rst new file mode 100644 index 0000000..b9b3be7 --- /dev/null +++ b/doc/OnsetFeature.rst @@ -0,0 +1,73 @@ +:digest: Buffer-Based Spectral Difference Feature +:species: descriptor +:sc-categories: Libraries>FluidDecomposition +:sc-related: Guides/FluidCorpusManipulationToolkit +:see-also: OnsetSlice, NoveltyFeature, AmpFeature, OnsetFeature +:description: Calculate the spectral difference feature used by :fluid-obj:`OnsetSlice`. +:discussion: + Calculates the feature used by :fluid-obj:`OnsetSlice` for slicing in realtime. + + The metric for calculating difference can be chosen from a curated selection, lending the algorithm toward slicing a broad range of musical materials. + +:process: The audio rate version of the object. +:output: A KR signal of the feature. + +:control metric: + + The metric used to derive a difference curve between spectral frames. It can be any of the following: + + :enum: + + :0: + **Energy** thresholds on (sum of squares of magnitudes / nBins) (like Onsets \power) + + :1: + **HFC** thresholds on (sum of (squared magnitudes * binNum) / nBins) + + :2: + **SpectralFlux** thresholds on (diffence in magnitude between consecutive frames, half rectified) + + :3: + **MKL** thresholds on (sum of log of magnitude ratio per bin) (or equivalently, sum of difference of the log magnitude per bin) (like Onsets mkl) + + :4: + **IS** (WILL PROBABLY BE REMOVED) Itakura - Saito divergence (see literature) + + :5: + **Cosine** thresholds on (cosine distance between comparison frames) + + :6: + **PhaseDev** takes the past 2 frames, projects to the current, as anticipated if it was a steady state, then compute the sum of the differences, on which it thresholds (like Onsets \phase) + + :7: + **WPhaseDev** same as PhaseDev, but weighted by the magnitude in order to remove chaos noise floor (like Onsets \wphase) + + :8: + **ComplexDev** same as PhaseDev, but in the complex domain - the anticipated amp is considered steady, and the phase is projected, then a complex subtraction is done with the actual present frame. The sum of magnitudes is used to threshold (like Onsets \complex) + + :9: + **RComplexDev** same as above, but rectified (like Onsets \rcomplex) + +:control filterSize: + + The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes. + +:control frameDelta: + + For certain metrics (HFC, SpectralFlux, MKL, Cosine), the distance does not have to be computed between consecutive frames. By default (0) it is, otherwise this sets the distane between the comparison window in samples. + +:control windowSize: + + The window size. As spectral differencing relies on spectral frames, we need to decide what precision we give it spectrally and temporally, in line with Gabor Uncertainty principles. http://www.subsurfwiki.org/wiki/Gabor_uncertainty + +:control hopSize: + + The window hop size. As spectral differencing relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. The -1 default value will default to half of windowSize (overlap of 2). + +:control fftSize: + + The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. The -1 default value will default to windowSize. + +:control maxFFTSize: + + How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated. \ No newline at end of file From 1cfe3e598d655d384dc8c1a995b2b22d06833859 Mon Sep 17 00:00:00 2001 From: weefuzzy Date: Mon, 28 Mar 2022 18:46:33 +0100 Subject: [PATCH 11/22] Add JSON generators for Slicers' feature counterparts --- src/clients/AmpFeature/AmpFeature.cpp | 19 +++++++++++++++++++ src/clients/AmpFeature/CMakeLists.txt | 16 ++++++++++++++++ src/clients/BufAmpFeature/BufAmpFeature.cpp | 19 +++++++++++++++++++ src/clients/BufAmpFeature/CMakeLists.txt | 16 ++++++++++++++++ .../BufNoveltyFeature/BufNoveltyFeature.cpp | 19 +++++++++++++++++++ src/clients/BufNoveltyFeature/CMakeLists.txt | 16 ++++++++++++++++ .../BufOnsetFeature/BufOnsetFeature.cpp | 19 +++++++++++++++++++ src/clients/BufOnsetFeature/CMakeLists.txt | 16 ++++++++++++++++ src/clients/NoveltyFeature/CMakeLists.txt | 16 ++++++++++++++++ src/clients/NoveltyFeature/NoveltyFeature.cpp | 19 +++++++++++++++++++ src/clients/OnsetFeature/CMakeLists.txt | 16 ++++++++++++++++ src/clients/OnsetFeature/OnsetFeature.cpp | 19 +++++++++++++++++++ 12 files changed, 210 insertions(+) create mode 100644 src/clients/AmpFeature/AmpFeature.cpp create mode 100644 src/clients/AmpFeature/CMakeLists.txt create mode 100644 src/clients/BufAmpFeature/BufAmpFeature.cpp create mode 100644 src/clients/BufAmpFeature/CMakeLists.txt create mode 100644 src/clients/BufNoveltyFeature/BufNoveltyFeature.cpp create mode 100644 src/clients/BufNoveltyFeature/CMakeLists.txt create mode 100644 src/clients/BufOnsetFeature/BufOnsetFeature.cpp create mode 100644 src/clients/BufOnsetFeature/CMakeLists.txt create mode 100644 src/clients/NoveltyFeature/CMakeLists.txt create mode 100644 src/clients/NoveltyFeature/NoveltyFeature.cpp create mode 100644 src/clients/OnsetFeature/CMakeLists.txt create mode 100644 src/clients/OnsetFeature/OnsetFeature.cpp diff --git a/src/clients/AmpFeature/AmpFeature.cpp b/src/clients/AmpFeature/AmpFeature.cpp new file mode 100644 index 0000000..86e32b0 --- /dev/null +++ b/src/clients/AmpFeature/AmpFeature.cpp @@ -0,0 +1,19 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#include +#include +int main(int argc, char *argv[]) { + using namespace fluid::client; + if (!argc) std::cerr << "Please pass a folder to write to"; + std::cout << "Write JSON to " << argv[1]; + ParameterDump::dump("AmpFeature", argv[1]); + return 0; +} diff --git a/src/clients/AmpFeature/CMakeLists.txt b/src/clients/AmpFeature/CMakeLists.txt new file mode 100644 index 0000000..b07cf17 --- /dev/null +++ b/src/clients/AmpFeature/CMakeLists.txt @@ -0,0 +1,16 @@ +# Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +# Copyright 2017-2019 University of Huddersfield. +# Licensed under the BSD-3 License. +# See license.md file in the project root for full license information. +# This project has received funding from the European Research Council (ERC) +# under the European Union’s Horizon 2020 research and innovation programme +# (grant agreement No 725899). + +cmake_minimum_required(VERSION 3.11) +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-pretarget.cmake) +add_executable( + ${PROJECT_NAME} + ${PROJECT_NAME}.cpp +) + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-posttarget.cmake) diff --git a/src/clients/BufAmpFeature/BufAmpFeature.cpp b/src/clients/BufAmpFeature/BufAmpFeature.cpp new file mode 100644 index 0000000..13ca288 --- /dev/null +++ b/src/clients/BufAmpFeature/BufAmpFeature.cpp @@ -0,0 +1,19 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#include +#include +int main(int argc, char *argv[]) { + using namespace fluid::client; + if (!argc) std::cerr << "Please pass a folder to write to"; + std::cout << "Write JSON to " << argv[1]; + ParameterDump::dump("BufAmpFeature", argv[1]); + return 0; +} diff --git a/src/clients/BufAmpFeature/CMakeLists.txt b/src/clients/BufAmpFeature/CMakeLists.txt new file mode 100644 index 0000000..b07cf17 --- /dev/null +++ b/src/clients/BufAmpFeature/CMakeLists.txt @@ -0,0 +1,16 @@ +# Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +# Copyright 2017-2019 University of Huddersfield. +# Licensed under the BSD-3 License. +# See license.md file in the project root for full license information. +# This project has received funding from the European Research Council (ERC) +# under the European Union’s Horizon 2020 research and innovation programme +# (grant agreement No 725899). + +cmake_minimum_required(VERSION 3.11) +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-pretarget.cmake) +add_executable( + ${PROJECT_NAME} + ${PROJECT_NAME}.cpp +) + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-posttarget.cmake) diff --git a/src/clients/BufNoveltyFeature/BufNoveltyFeature.cpp b/src/clients/BufNoveltyFeature/BufNoveltyFeature.cpp new file mode 100644 index 0000000..c016b8a --- /dev/null +++ b/src/clients/BufNoveltyFeature/BufNoveltyFeature.cpp @@ -0,0 +1,19 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#include +#include +int main(int argc, char *argv[]) { + using namespace fluid::client; + if (!argc) std::cerr << "Please pass a folder to write to"; + std::cout << "Write JSON to " << argv[1]; + ParameterDump::dump("BufNoveltyFeature", argv[1]); + return 0; +} diff --git a/src/clients/BufNoveltyFeature/CMakeLists.txt b/src/clients/BufNoveltyFeature/CMakeLists.txt new file mode 100644 index 0000000..b07cf17 --- /dev/null +++ b/src/clients/BufNoveltyFeature/CMakeLists.txt @@ -0,0 +1,16 @@ +# Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +# Copyright 2017-2019 University of Huddersfield. +# Licensed under the BSD-3 License. +# See license.md file in the project root for full license information. +# This project has received funding from the European Research Council (ERC) +# under the European Union’s Horizon 2020 research and innovation programme +# (grant agreement No 725899). + +cmake_minimum_required(VERSION 3.11) +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-pretarget.cmake) +add_executable( + ${PROJECT_NAME} + ${PROJECT_NAME}.cpp +) + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-posttarget.cmake) diff --git a/src/clients/BufOnsetFeature/BufOnsetFeature.cpp b/src/clients/BufOnsetFeature/BufOnsetFeature.cpp new file mode 100644 index 0000000..7bb5968 --- /dev/null +++ b/src/clients/BufOnsetFeature/BufOnsetFeature.cpp @@ -0,0 +1,19 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#include +#include +int main(int argc, char *argv[]) { + using namespace fluid::client; + if (!argc) std::cerr << "Please pass a folder to write to"; + std::cout << "Write JSON to " << argv[1]; + ParameterDump::dump("BufOnsetFeature", argv[1]); + return 0; +} diff --git a/src/clients/BufOnsetFeature/CMakeLists.txt b/src/clients/BufOnsetFeature/CMakeLists.txt new file mode 100644 index 0000000..b07cf17 --- /dev/null +++ b/src/clients/BufOnsetFeature/CMakeLists.txt @@ -0,0 +1,16 @@ +# Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +# Copyright 2017-2019 University of Huddersfield. +# Licensed under the BSD-3 License. +# See license.md file in the project root for full license information. +# This project has received funding from the European Research Council (ERC) +# under the European Union’s Horizon 2020 research and innovation programme +# (grant agreement No 725899). + +cmake_minimum_required(VERSION 3.11) +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-pretarget.cmake) +add_executable( + ${PROJECT_NAME} + ${PROJECT_NAME}.cpp +) + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-posttarget.cmake) diff --git a/src/clients/NoveltyFeature/CMakeLists.txt b/src/clients/NoveltyFeature/CMakeLists.txt new file mode 100644 index 0000000..b07cf17 --- /dev/null +++ b/src/clients/NoveltyFeature/CMakeLists.txt @@ -0,0 +1,16 @@ +# Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +# Copyright 2017-2019 University of Huddersfield. +# Licensed under the BSD-3 License. +# See license.md file in the project root for full license information. +# This project has received funding from the European Research Council (ERC) +# under the European Union’s Horizon 2020 research and innovation programme +# (grant agreement No 725899). + +cmake_minimum_required(VERSION 3.11) +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-pretarget.cmake) +add_executable( + ${PROJECT_NAME} + ${PROJECT_NAME}.cpp +) + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-posttarget.cmake) diff --git a/src/clients/NoveltyFeature/NoveltyFeature.cpp b/src/clients/NoveltyFeature/NoveltyFeature.cpp new file mode 100644 index 0000000..9c97da6 --- /dev/null +++ b/src/clients/NoveltyFeature/NoveltyFeature.cpp @@ -0,0 +1,19 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#include +#include +int main(int argc, char *argv[]) { + using namespace fluid::client; + if (!argc) std::cerr << "Please pass a folder to write to"; + std::cout << "Write JSON to " << argv[1]; + ParameterDump::dump("NoveltyFeature", argv[1]); + return 0; +} diff --git a/src/clients/OnsetFeature/CMakeLists.txt b/src/clients/OnsetFeature/CMakeLists.txt new file mode 100644 index 0000000..b07cf17 --- /dev/null +++ b/src/clients/OnsetFeature/CMakeLists.txt @@ -0,0 +1,16 @@ +# Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +# Copyright 2017-2019 University of Huddersfield. +# Licensed under the BSD-3 License. +# See license.md file in the project root for full license information. +# This project has received funding from the European Research Council (ERC) +# under the European Union’s Horizon 2020 research and innovation programme +# (grant agreement No 725899). + +cmake_minimum_required(VERSION 3.11) +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-pretarget.cmake) +add_executable( + ${PROJECT_NAME} + ${PROJECT_NAME}.cpp +) + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../script/dump-posttarget.cmake) diff --git a/src/clients/OnsetFeature/OnsetFeature.cpp b/src/clients/OnsetFeature/OnsetFeature.cpp new file mode 100644 index 0000000..630a621 --- /dev/null +++ b/src/clients/OnsetFeature/OnsetFeature.cpp @@ -0,0 +1,19 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#include +#include +int main(int argc, char *argv[]) { + using namespace fluid::client; + if (!argc) std::cerr << "Please pass a folder to write to"; + std::cout << "Write JSON to " << argv[1]; + ParameterDump::dump("OnsetFeature", argv[1]); + return 0; +} From d7d251490c898f915154e99e2635304ab2b4c585 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:49:48 +0100 Subject: [PATCH 12/22] rt noveltyfeature --- doc/NoveltyFeature.rst | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 doc/NoveltyFeature.rst diff --git a/doc/NoveltyFeature.rst b/doc/NoveltyFeature.rst new file mode 100644 index 0000000..aa6d5b8 --- /dev/null +++ b/doc/NoveltyFeature.rst @@ -0,0 +1,66 @@ +:digest: Realtime Novelty Feature +:species: descriptor +:sc-categories: Libraries>FluidDecomposition, UGens>Buffer +:sc-related: Guides/FluidCorpusManipulationToolkit +:see-also: NoveltySlice, AmpFeature, OnsetFeature +:description: Calculates novelty of audio stored in a buffer. Used by :fluid-obj:`BufNoveltySlice`. +:discussion: + Calculate novelty in realtime, the feature used by :fluid-obj:`NoveltySlice`. + + Novelty derived by running a kernel across the diagonal of the similarity matrix. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. + +:process: The audio rate version of the object. +:output: A KR signal of the feature. + +:control algorithm: + + The feature on which novelty is computed. + + :enum: + + :0: + Spectrum – The magnitude of the full spectrum. + + :1: + MFCC – 13 Mel-Frequency Cepstrum Coefficients. + + :2: + Chroma - The contour of a 12-band chromagram. + + :3: + Pitch – The pitch and its confidence. + + :4: + Loudness – The true peak and loudness. + +:control kernelSize: + + The granularity of the window in which the algorithm looks for change, in samples. A small number will be sensitive to short term changes, and a large number should look for long term changes. + +:control filterSize: + + The size of a smoothing filter that is applied on the novelty curve. A larger filter filter size allows for cleaner cuts on very sharp changes. + +:control windowSize: + + The window size. As novelty estimation relies on spectral frames, we need to decide what precision we give it spectrally and temporally. + +:control hopSize: + + The window hop size. As novelty estimation relies on spectral frames, we need to move the window forward. It can be any size but low overlap will create audible artefacts. + +:control fftSize: + + The inner FFT/IFFT size. It should be at least 4 samples long, at least the size of the window, and a power of 2. Making it larger allows an oversampling of the spectral precision. + +:control maxFFTSize: + + How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated. + +:control maxKernelSize: + + This cannot be modulated. + +:control maxFilterSize: + + This cannot be modulated. \ No newline at end of file From f581b69b32921ad268f4c721618807c9597df25a Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:49:55 +0100 Subject: [PATCH 13/22] fix various errors --- doc/AmpFeature.rst | 2 +- doc/BufNoveltyFeature.rst | 2 +- doc/OnsetFeature.rst | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/AmpFeature.rst b/doc/AmpFeature.rst index 0df1e67..6dc4ef1 100644 --- a/doc/AmpFeature.rst +++ b/doc/AmpFeature.rst @@ -2,7 +2,7 @@ :species: descriptor :sc-categories: Libraries>FluidDecomposition :sc-related: Guides/FluidCorpusManipulationToolkit -:see-also: AmpGate, BufAmpSlice, BufOnsetSlice, BufNoveltySlice, BufTransientSlice +:see-also: AmpGate, AmpSlice, OnsetFeature, NoveltyFeature :description: Calculate the amplitude differential feature used by :fluid-obj:`BufAmpSlice`. :discussion: :fluid-obj:`BufAmpSlice` uses the differential between a fast and a slow envelope follower to determine changes in amplitude. This object calculates the amplitude differential and copies it to an output buffer. diff --git a/doc/BufNoveltyFeature.rst b/doc/BufNoveltyFeature.rst index ead0504..ccb9bfa 100644 --- a/doc/BufNoveltyFeature.rst +++ b/doc/BufNoveltyFeature.rst @@ -36,7 +36,7 @@ The index of the buffer where the novelty feature will be written. -:control feature: +:control algorithm: The feature on which novelty is computed. diff --git a/doc/OnsetFeature.rst b/doc/OnsetFeature.rst index b9b3be7..f4127ab 100644 --- a/doc/OnsetFeature.rst +++ b/doc/OnsetFeature.rst @@ -1,8 +1,8 @@ -:digest: Buffer-Based Spectral Difference Feature +:digest: Realtime Spectral Difference Feature :species: descriptor :sc-categories: Libraries>FluidDecomposition :sc-related: Guides/FluidCorpusManipulationToolkit -:see-also: OnsetSlice, NoveltyFeature, AmpFeature, OnsetFeature +:see-also: OnsetSlice, NoveltyFeature, AmpFeature :description: Calculate the spectral difference feature used by :fluid-obj:`OnsetSlice`. :discussion: Calculates the feature used by :fluid-obj:`OnsetSlice` for slicing in realtime. From 4e0c142452d14491087262ef7345f2021ad3d7f7 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 18:52:40 +0100 Subject: [PATCH 14/22] robots in disguise --- doc/AmpFeature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/AmpFeature.rst b/doc/AmpFeature.rst index 6dc4ef1..253963f 100644 --- a/doc/AmpFeature.rst +++ b/doc/AmpFeature.rst @@ -1,5 +1,5 @@ :digest: Realtime Amplitude Differential Feature -:species: descriptor +:species: transformer :sc-categories: Libraries>FluidDecomposition :sc-related: Guides/FluidCorpusManipulationToolkit :see-also: AmpGate, AmpSlice, OnsetFeature, NoveltyFeature From 0d65c88bd3cd5592266403afdb00aa890f83f32f Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 19:19:43 +0100 Subject: [PATCH 15/22] change feature -> algorithm --- doc/BufNoveltySlice.rst | 2 +- doc/NoveltySlice.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/BufNoveltySlice.rst b/doc/BufNoveltySlice.rst index 9e41e25..eca824b 100644 --- a/doc/BufNoveltySlice.rst +++ b/doc/BufNoveltySlice.rst @@ -37,7 +37,7 @@ The index of the buffer where the indices (in sample) of the estimated starting points of slices will be written. The first and last points are always the boundary points of the analysis. -:control feature: +:control algorithm: The feature on which novelty is computed. diff --git a/doc/NoveltySlice.rst b/doc/NoveltySlice.rst index c8cd84a..7faef63 100644 --- a/doc/NoveltySlice.rst +++ b/doc/NoveltySlice.rst @@ -16,7 +16,7 @@ The audio to be processed. -:control feature: +:control algorithm: The feature on which novelty is computed. From 1e796ba878ccfd50829cdb8ce113b9dc101601ca Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 19:26:07 +0100 Subject: [PATCH 16/22] cleanup noveltyfeature --- doc/BufNoveltyFeature.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/BufNoveltyFeature.rst b/doc/BufNoveltyFeature.rst index ccb9bfa..4c7bf37 100644 --- a/doc/BufNoveltyFeature.rst +++ b/doc/BufNoveltyFeature.rst @@ -3,8 +3,8 @@ :sc-categories: Libraries>FluidDecomposition, UGens>Buffer :sc-related: Guides/FluidCorpusManipulationToolkit :see-also: BufNoveltySlice, BufNoveltyFeature, BufAmpFeature, BufOnsetFeature -:description: A non-realtime that calculates novelty of audio stored in a buffer. Used by :fluid-obj:`BufNoveltySlice`. -:discussion: +:description: Calculates the novelty feature of audio stored in a buffer. +:discussion: Novelty is derived by running a kernel across the diagonal of the similarity matrix. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. The process will return a buffer containing a time series that describes the novelty feature changing over time in the source buffer. From 2a5b3f2d417050a1d9ec882ed463044d6859d90d Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 19:28:05 +0100 Subject: [PATCH 17/22] make noveltyfeature more consistent --- doc/BufNoveltyFeature.rst | 8 +++++--- doc/NoveltyFeature.rst | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/BufNoveltyFeature.rst b/doc/BufNoveltyFeature.rst index 4c7bf37..17ce07a 100644 --- a/doc/BufNoveltyFeature.rst +++ b/doc/BufNoveltyFeature.rst @@ -5,9 +5,11 @@ :see-also: BufNoveltySlice, BufNoveltyFeature, BufAmpFeature, BufOnsetFeature :description: Calculates the novelty feature of audio stored in a buffer. :discussion: - Novelty is derived by running a kernel across the diagonal of the similarity matrix. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. - - The process will return a buffer containing a time series that describes the novelty feature changing over time in the source buffer. + Calculate novelty of audio stored in a buffer, the feature used by :fluid-obj:`BufNoveltySlice` to perform segmentation. + + Novelty is derived by running a kernel across the diagonal of the similarity matrix. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. + + The process will return a buffer containing a time series that describes the novelty feature changing over time in the source buffer. :process: This is the method that calls for the slicing to be calculated on a given source buffer. :output: Nothing, as the various destination buffers are declared in the function call. diff --git a/doc/NoveltyFeature.rst b/doc/NoveltyFeature.rst index aa6d5b8..88f449c 100644 --- a/doc/NoveltyFeature.rst +++ b/doc/NoveltyFeature.rst @@ -3,9 +3,9 @@ :sc-categories: Libraries>FluidDecomposition, UGens>Buffer :sc-related: Guides/FluidCorpusManipulationToolkit :see-also: NoveltySlice, AmpFeature, OnsetFeature -:description: Calculates novelty of audio stored in a buffer. Used by :fluid-obj:`BufNoveltySlice`. +:description: Calculates the novelty feature of audio in realtime. :discussion: - Calculate novelty in realtime, the feature used by :fluid-obj:`NoveltySlice`. + Calculate novelty in realtime, the feature used by :fluid-obj:`NoveltySlice` to perform segmentation. Novelty derived by running a kernel across the diagonal of the similarity matrix. It implements the seminal results published in 'Automatic Audio Segmentation Using a Measure of Audio Novelty' by J Foote. From ecc8e7cb3562eba39c1c2593e6bb888d4c6ea251 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Mon, 28 Mar 2022 19:28:47 +0100 Subject: [PATCH 18/22] remove superfluous spaces --- doc/BufOnsetFeature.rst | 2 +- doc/OnsetFeature.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/BufOnsetFeature.rst b/doc/BufOnsetFeature.rst index 4e15588..409aafd 100644 --- a/doc/BufOnsetFeature.rst +++ b/doc/BufOnsetFeature.rst @@ -3,7 +3,7 @@ :sc-categories: Libraries>FluidDecomposition :sc-related: Guides/FluidCorpusManipulationToolkit :see-also: BufOnsetSlice, BufNoveltyFeature, BufAmpFeature, BufOnsetFeature -:description: Calculate the spectral difference feature used by :fluid-obj:`BufOnsetSlice`. +:description: Calculate the spectral difference feature used by :fluid-obj:`BufOnsetSlice`. :discussion: Given a source buffer, calculates the feature used by :fluid-obj:`BufOnsetSlice` and copies it to another buffer. diff --git a/doc/OnsetFeature.rst b/doc/OnsetFeature.rst index f4127ab..30647f6 100644 --- a/doc/OnsetFeature.rst +++ b/doc/OnsetFeature.rst @@ -3,7 +3,7 @@ :sc-categories: Libraries>FluidDecomposition :sc-related: Guides/FluidCorpusManipulationToolkit :see-also: OnsetSlice, NoveltyFeature, AmpFeature -:description: Calculate the spectral difference feature used by :fluid-obj:`OnsetSlice`. +:description: Calculate the spectral difference feature used by :fluid-obj:`OnsetSlice`. :discussion: Calculates the feature used by :fluid-obj:`OnsetSlice` for slicing in realtime. From 3e4a2b5f5f8427386f001f4f6f92ce3ea58406fa Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Tue, 29 Mar 2022 11:16:34 +0100 Subject: [PATCH 19/22] fix description --- doc/AmpFeature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/AmpFeature.rst b/doc/AmpFeature.rst index 253963f..2c4957a 100644 --- a/doc/AmpFeature.rst +++ b/doc/AmpFeature.rst @@ -3,7 +3,7 @@ :sc-categories: Libraries>FluidDecomposition :sc-related: Guides/FluidCorpusManipulationToolkit :see-also: AmpGate, AmpSlice, OnsetFeature, NoveltyFeature -:description: Calculate the amplitude differential feature used by :fluid-obj:`BufAmpSlice`. +:description: Calculate the amplitude differential feature in realtime. :discussion: :fluid-obj:`BufAmpSlice` uses the differential between a fast and a slow envelope follower to determine changes in amplitude. This object calculates the amplitude differential and copies it to an output buffer. From d7c43ff464fc51cca04ef6191d2a29ba1d3b2c8e Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Tue, 29 Mar 2022 11:17:24 +0100 Subject: [PATCH 20/22] fix reference to buffer version in rt object --- doc/AmpFeature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/AmpFeature.rst b/doc/AmpFeature.rst index 2c4957a..4425a7b 100644 --- a/doc/AmpFeature.rst +++ b/doc/AmpFeature.rst @@ -5,7 +5,7 @@ :see-also: AmpGate, AmpSlice, OnsetFeature, NoveltyFeature :description: Calculate the amplitude differential feature in realtime. :discussion: - :fluid-obj:`BufAmpSlice` uses the differential between a fast and a slow envelope follower to determine changes in amplitude. This object calculates the amplitude differential and copies it to an output buffer. + :fluid-obj:`AmpSlice` uses the differential between a fast and a slow envelope follower to determine changes in amplitude. This object calculates the amplitude differential and copies it to an output buffer. :process: The audio rate in, control rate out version of the object. :output: A KR signal of the feature. From a9982b61121be944a8bb8369e607e3c026fcac31 Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Tue, 29 Mar 2022 12:22:45 +0100 Subject: [PATCH 21/22] add padding --- doc/BufAmpFeature.rst | 6 ++++++ doc/BufNoveltyFeature.rst | 4 ++++ doc/BufOnsetFeature.rst | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/doc/BufAmpFeature.rst b/doc/BufAmpFeature.rst index a17b9c4..b121e70 100644 --- a/doc/BufAmpFeature.rst +++ b/doc/BufAmpFeature.rst @@ -58,4 +58,10 @@ The frequency of the fourth-order Linkwitz–Riley high-pass filter (https://en.wikipedia.org/wiki/Linkwitz%E2%80%93Riley_filter). This is done first on the signal to minimise low frequency intermodulation with very fast ramp lengths. A frequency of 0 bypasses the filter. +:control padding: + Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material. + +:control action: + + A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The function will be passed indices as an argument. \ No newline at end of file diff --git a/doc/BufNoveltyFeature.rst b/doc/BufNoveltyFeature.rst index 17ce07a..08a5e6f 100644 --- a/doc/BufNoveltyFeature.rst +++ b/doc/BufNoveltyFeature.rst @@ -91,6 +91,10 @@ This cannot be modulated. +:control padding: + + Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material. + :control action: A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The function will be passed indices as an argument. diff --git a/doc/BufOnsetFeature.rst b/doc/BufOnsetFeature.rst index 409aafd..9ed4e2a 100644 --- a/doc/BufOnsetFeature.rst +++ b/doc/BufOnsetFeature.rst @@ -96,6 +96,10 @@ How large can the FFT be, by allocating memory at instantiation time. This cannot be modulated. +:control padding: + + Controls the zero-padding added to either end of the source buffer or segment. Possible values are 0 (no padding), 1 (default, half the window size), or 2 (window size - hop size). Padding ensures that all input samples are completely analysed: with no padding, the first analysis window starts at time 0, and the samples at either end will be tapered by the STFT windowing function. Mode 1 has the effect of centering the first sample in the analysis window and ensuring that the very start and end of the segment are accounted for in the analysis. Mode 2 can be useful when the overlap factor (window size / hop size) is greater than 2, to ensure that the input samples at either end of the segment are covered by the same number of analysis frames as the rest of the analysed material. + :control action: A Function to be evaluated once the offline process has finished and indices instance variables have been updated on the client side. The function will be passed indices as an argument. From 1de1157c3daed365a3da9ab65621a37b49de0aef Mon Sep 17 00:00:00 2001 From: Ted Moore Date: Tue, 29 Mar 2022 18:49:04 +0100 Subject: [PATCH 22/22] sc example code for slicer-features --- example-code/sc/AmpFeature.scd | 51 +++++++++++++++++++++++++++ example-code/sc/BufAmpFeature.scd | 13 +++++++ example-code/sc/BufNoveltyFeature.scd | 13 +++++++ example-code/sc/BufOnsetFeature.scd | 13 +++++++ example-code/sc/NoveltyFeature.scd | 38 ++++++++++++++++++++ example-code/sc/OnsetFeature.scd | 39 ++++++++++++++++++++ 6 files changed, 167 insertions(+) create mode 100644 example-code/sc/AmpFeature.scd create mode 100644 example-code/sc/BufAmpFeature.scd create mode 100644 example-code/sc/BufNoveltyFeature.scd create mode 100644 example-code/sc/BufOnsetFeature.scd create mode 100644 example-code/sc/NoveltyFeature.scd create mode 100644 example-code/sc/OnsetFeature.scd diff --git a/example-code/sc/AmpFeature.scd b/example-code/sc/AmpFeature.scd new file mode 100644 index 0000000..e2ac382 --- /dev/null +++ b/example-code/sc/AmpFeature.scd @@ -0,0 +1,51 @@ +strong::Use the output as a stream of decibels to manipulate another (or the same) audio signal:: +code:: +~drums = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); + +// listen to the original +~drums.play; + +// add extra pop to the drums by using the AmpFeature envelope +( +{ + var sig = PlayBuf.ar(~drums.numChannels,~drums,BufRateScale.ir(~drums),loop:1); + var amp_feature = FluidAmpFeature.ar(sig,100,100,44100,44100,-40); + amp_feature.poll; + sig * amp_feature.dbamp * -30.dbamp; +}.play; +) + +// really suppress the amplitude peaks by inverting the feature +( +{ + var sig = PlayBuf.ar(~drums.numChannels,~drums,BufRateScale.ir(~drums),loop:1); + var amp_feature = FluidAmpFeature.ar(sig,100,100,44100,44100,-40) * -1; + amp_feature.poll; + sig * amp_feature.dbamp +}.play; +) + +:: +strong::Make your own threshold, similar to how FluidAmpSlice works:: +code:: + +~src = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); + +// listen to the original +~src.play; + +( +{ + var sig = PlayBuf.ar(~src.numChannels,~src,BufRateScale.ir(~src),loop:1); + var amp_feature = FluidAmpFeature.ar(sig,100,100,44100,44100,-40); + var trig = Schmidt.ar(amp_feature,0,6); + SendReply.ar(trig,"/amp_onset"); + sig; +}.play; + +OSCdef(\amp_onset,{ + arg msg; + "onset".postln; +},"/amp_onset"); +) +:: \ No newline at end of file diff --git a/example-code/sc/BufAmpFeature.scd b/example-code/sc/BufAmpFeature.scd new file mode 100644 index 0000000..02d890b --- /dev/null +++ b/example-code/sc/BufAmpFeature.scd @@ -0,0 +1,13 @@ +strong::Plot the BufAmpFeature curve along with the indices from BufAmpSlice:: +code:: +~drums = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); + +( +~indices = Buffer(s); +~feature = Buffer(s); +FluidBufAmpSlice.processBlocking(s,~drums,indices:~indices,fastRampUp:100,fastRampDown:100,slowRampUp:4410,slowRampDown:4410,onThreshold:10,offThreshold:0,floor:-40); +FluidBufAmpFeature.processBlocking(s,~drums,features:~feature,fastRampUp:10,fastRampDown:10,slowRampUp:4410,slowRampDown:4410,floor:-60); +FluidWaveform(~drums,~indices,~feature,bounds:Rect(0,0,1600,400)); +) + +:: \ No newline at end of file diff --git a/example-code/sc/BufNoveltyFeature.scd b/example-code/sc/BufNoveltyFeature.scd new file mode 100644 index 0000000..847047b --- /dev/null +++ b/example-code/sc/BufNoveltyFeature.scd @@ -0,0 +1,13 @@ +strong::Plot the BufNoveltyFeature curve along with the indices from BufNoveltySlice:: +code:: + +~src = Buffer.read(s,FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav")); + +( +~indices = Buffer(s); +~feature = Buffer(s); +FluidBufNoveltySlice.processBlocking(s,~src,indices:~indices,algorithm:1,kernelSize:5,threshold:1.1); +FluidBufNoveltyFeature.processBlocking(s,~src,features:~feature,algorithm:1,kernelSize:5); +FluidWaveform(~src,~indices,~feature,bounds:Rect(0,0,1600,400)); +) +:: \ No newline at end of file diff --git a/example-code/sc/BufOnsetFeature.scd b/example-code/sc/BufOnsetFeature.scd new file mode 100644 index 0000000..26d81a2 --- /dev/null +++ b/example-code/sc/BufOnsetFeature.scd @@ -0,0 +1,13 @@ +strong::Plot the BufOnsetFeature curve along with the indices from BufOnsetSlice:: +code:: + +~src = Buffer.readChannel(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"),channels:[0]); + +( +~indices = Buffer(s); +~feature = Buffer(s); +FluidBufOnsetSlice.processBlocking(s,~src,indices:~indices,metric:9,threshold:0.2); +FluidBufOnsetFeature.processBlocking(s,~src,features:~feature,metric:9); +FluidWaveform(~src,~indices,~feature,bounds:Rect(0,0,1600,400),lineWidth:2); +) +:: \ No newline at end of file diff --git a/example-code/sc/NoveltyFeature.scd b/example-code/sc/NoveltyFeature.scd new file mode 100644 index 0000000..583e014 --- /dev/null +++ b/example-code/sc/NoveltyFeature.scd @@ -0,0 +1,38 @@ +code:: +~src = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcBassGuit-Melo-M.wav")); + +( +{ + var sig = PlayBuf.ar(~src.numChannels,~src,BufRateScale.ir(~src),loop:1); + var feature = FluidNoveltyFeature.kr(sig,3,5,1); // feature is pitch and pich confidence + SendReply.kr(Impulse.kr(30),"/feature",feature); + sig.dup; +}.play; + +OSCdef(\feature,{ + arg msg; + {"*".post} ! (msg[3] * 20); + " %".format(msg[3].round(0.1)).postln; +},"/feature"); +) + +:: +strong::Make your own threshold and Schmidt Trigger:: +code:: +~src = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav")); + +( +{ + var sig = PlayBuf.ar(~src.numChannels,~src,BufRateScale.ir(~src),loop:1); + var feature = FluidNoveltyFeature.kr(sig,1); + var trig = Schmidt.kr(feature,0.3,0.5); + SendReply.kr(trig,"/onset"); + sig.dup; +}.play; + +OSCdef(\onset,{ + arg msg; + "onset".postln; +},"/onset"); +) +:: \ No newline at end of file diff --git a/example-code/sc/OnsetFeature.scd b/example-code/sc/OnsetFeature.scd new file mode 100644 index 0000000..e81f10c --- /dev/null +++ b/example-code/sc/OnsetFeature.scd @@ -0,0 +1,39 @@ +code:: +~src = Buffer.readChannel(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"),channels:[0]); + +( +{ + var sig = PlayBuf.ar(~src.numChannels,~src,BufRateScale.ir(~src),loop:1); + var feature = FluidOnsetFeature.kr(sig,9,filterSize:5,windowSize:512); + SendReply.kr(Impulse.kr(30),"/feature",feature); + sig.dup; +}.play; + +OSCdef(\feature,{ + arg msg; + {"*".post} ! (msg[3] * 1000).abs; + "".postln; +},"/feature"); +) + +:: +strong::Make your own threshold and Schmidt Trigger:: +code:: + +~src = Buffer.readChannel(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"),channels:[0]); + +( +{ + var sig = PlayBuf.ar(~src.numChannels,~src,BufRateScale.ir(~src),loop:1); + var feature = FluidOnsetFeature.kr(sig,9); + var trig = Schmidt.kr(feature,0.01,0.1); + SendReply.kr(trig,"/onset"); + sig; +}.play; + +OSCdef(\onset,{ + arg msg; + "onset".postln; +},"/onset"); +) +:: \ No newline at end of file