Permalink
Browse files

Add parallelization to find audio duplicates MUCH faster

Use __gnu_parallel::for_each so all CPUs are used at the same time
to calculate audio duplicates which greatly increases the speed.
Previously it took each iteration around 150 seconds (570 songs/second).
After this patch, each iteration takes around 8.5 seconds (11200
songs/second).
  • Loading branch information...
antlarr committed Jun 28, 2018
1 parent b949ecd commit 8759edb15433fbd5548ec4b10aa9ff5f9f763659
Showing with 23 additions and 21 deletions.
  1. +1 −0 bard/bard.py
  2. +19 −19 bard/bard_ext.cpp
  3. +3 −2 setup.py
@@ -921,6 +921,7 @@ def findAudioDuplicates2(self, from_song_id=None):
# return
start_time = time.time()
result = fpm.addSongAndCompare(songID, dfp[0], storeThreshold)
result.sort(key=lambda x: x[0])
for (songID2, offset, similarity) in result:
print('******** %d %d %d %f' % (songID2, songID,
@@ -24,6 +24,8 @@
#include <vector>
#include <map>
#include <iostream>
#include <parallel/algorithm>
#include <mutex>
template<typename T>
inline
@@ -60,12 +62,12 @@ class FingerprintManager
std::pair<int, double> compareSongs(long songID1, long songID2, double cancelThreshold=0.55);
boost::python::list compareSongsVerbose(long songID1, long songID2);
std::pair<int, double> compareChromaprintFingerprintsAndOffset(std::vector<int> fp1, std::vector<int> fp2, double cancelThreshold) const;
std::pair<int, double> compareChromaprintFingerprintsAndOffset(const std::vector<int> &fp1, const std::vector<int> &fp2, double cancelThreshold) const;
boost::python::list compareChromaprintFingerprintsAndOffsetVerbose(std::vector<int> fp1, std::vector<int> fp2) const;
private:
int m_maxoffset;
std::map<int, std::vector<int> > m_fingerprints;
std::map<int, std::vector<int>> m_fingerprints;
};
FingerprintManager::FingerprintManager(): m_maxoffset(50)
@@ -92,32 +94,30 @@ void FingerprintManager::addSong(long songID, boost::python::list &fingerprint)
boost::python::list FingerprintManager::addSongAndCompare(long songID, boost::python::list &fingerprint, double cancelThreshold)
{
std::mutex result_mutex;
boost::python::list result;
auto v = to_std_vector<int>(fingerprint);
// std::cout << "len: " << v.size() << std::endl;
v.insert(v.begin(), m_maxoffset, 0);
// std::cout << "new len: " << v.size() << std::endl;
for (auto & [itSongID, itFingerprint]: m_fingerprints)
{
auto [offset, similarity] = compareChromaprintFingerprintsAndOffset(itFingerprint, v, cancelThreshold);
if (similarity > cancelThreshold)
{
// std::cout << "****" << songID << " " << itSongID << " " << offset << " " << similarity << std::endl;
result.append(boost::python::make_tuple(itSongID, offset, similarity));
} /*else {
if (similarity < 0 )
std::cout << songID << " " << itSongID << " different" << std::endl;
else
std::cout << songID << " " << itSongID << " " << offset << " " << similarity << std::endl;
}*/
}
auto vectorizedFP = std::vector<std::pair<int,std::vector<int>>>(m_fingerprints.begin(), m_fingerprints.end());
__gnu_parallel::for_each(vectorizedFP.begin(), vectorizedFP.end(),
[&](const auto &itSong)
{
auto & [itSongID, itFingerprint] = itSong;
auto [offset, similarity] = compareChromaprintFingerprintsAndOffset(itFingerprint, v, cancelThreshold);
if (similarity > cancelThreshold)
{
result_mutex.lock();

This comment has been minimized.

@scribam

scribam Jul 11, 2018

@antlarr Have you though about using a std::lock_guard? I am wondering if it can help to boost the performance a bit more 🤔 Nice blog post by the way 👍

This comment has been minimized.

@antlarr

antlarr Jul 11, 2018

Owner

Thanks for the suggestion, but I think in this case it isn't worth using std::lock_guard. It's useful for cases where there are multiple exit points of a function with some complex code, but here the code is so simple (I just need to protect a single line that shouldn't throw any exception) that it would probably be slower to create a lock_guard object in the stack just to destroy it two lines below.

result.append(boost::python::make_tuple(itSongID, offset, similarity));
result_mutex.unlock();
}
}, __gnu_parallel::parallel_balanced);
m_fingerprints[songID]=v;
return result;
}
std::pair<int, double> FingerprintManager::compareChromaprintFingerprintsAndOffset(std::vector<int> fp1, std::vector<int> fp2, double cancelThreshold) const
std::pair<int, double> FingerprintManager::compareChromaprintFingerprintsAndOffset(const std::vector<int> &fp1, const std::vector<int> &fp2, double cancelThreshold) const
{
std::vector<int>::const_iterator it1, it2;
int offset;
@@ -3,12 +3,13 @@
module1 = Extension('bard_ext',
define_macros=[('MAJOR_VERSION', '1'),
('MINOR_VERSION', '0')],
('MINOR_VERSION', '0'),
('_GLIBCXX_PARALLEL', None)],
include_dirs=['/usr/include/boost'],
libraries=['boost_python-py3'],
library_dirs=['/usr/lib'],
sources=['bard/bard_ext.cpp'],
extra_compile_args=['-std=c++1z'])
extra_compile_args=['-std=gnu++17', '-fopenmp'])
setup(
# Application name:

0 comments on commit 8759edb

Please sign in to comment.