Add parallelization to find audio duplicates MUCH faster

Use __gnu_parallel::for_each so all CPUs are used at the same time to calculate audio duplicates which greatly increases the speed. Previously it took each iteration around 150 seconds (570 songs/second). After this patch, each iteration takes around 8.5 seconds (11200 songs/second).
antlarr · Jul 10, 2018 · 8759edb · scribam · Jul 11, 2018 · antlarr
1 parent b949ecd
commit 8759edb
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 21 deletions.
diff --git a/bard/bard.py b/bard/bard.py
@@ -921,6 +921,7 @@ def findAudioDuplicates2(self, from_song_id=None):
                 #     return
                 start_time = time.time()
                 result = fpm.addSongAndCompare(songID, dfp[0], storeThreshold)
+                result.sort(key=lambda x: x[0])
 
             for (songID2, offset, similarity) in result:
                 print('******** %d %d %d %f' % (songID2, songID,

diff --git a/bard/bard_ext.cpp b/bard/bard_ext.cpp
@@ -24,6 +24,8 @@
 #include <vector>
 #include <map>
 #include <iostream>
+#include <parallel/algorithm>
+#include <mutex>
 
 template<typename T>
 inline
@@ -60,12 +62,12 @@ class FingerprintManager
     std::pair<int, double> compareSongs(long songID1, long songID2, double cancelThreshold=0.55);
     boost::python::list compareSongsVerbose(long songID1, long songID2);
 
-    std::pair<int, double> compareChromaprintFingerprintsAndOffset(std::vector<int> fp1, std::vector<int> fp2, double cancelThreshold) const;
+    std::pair<int, double> compareChromaprintFingerprintsAndOffset(const std::vector<int> &fp1, const std::vector<int> &fp2, double cancelThreshold) const;
     boost::python::list compareChromaprintFingerprintsAndOffsetVerbose(std::vector<int> fp1, std::vector<int> fp2) const;
 
 private:
     int m_maxoffset;
-    std::map<int, std::vector<int> > m_fingerprints;
+    std::map<int, std::vector<int>> m_fingerprints;
 };
 
 FingerprintManager::FingerprintManager(): m_maxoffset(50)
@@ -92,32 +94,30 @@ void FingerprintManager::addSong(long songID, boost::python::list &fingerprint)
 
 boost::python::list FingerprintManager::addSongAndCompare(long songID, boost::python::list &fingerprint, double cancelThreshold)
 {
+    std::mutex result_mutex;
     boost::python::list result;
     auto v = to_std_vector<int>(fingerprint);
-//    std::cout << "len: " << v.size() << std::endl;
     v.insert(v.begin(), m_maxoffset, 0);
-//    std::cout << "new len: " << v.size() << std::endl;
-    for (auto & [itSongID, itFingerprint]: m_fingerprints)
-    {
-        auto [offset, similarity] = compareChromaprintFingerprintsAndOffset(itFingerprint, v, cancelThreshold);
-        if (similarity > cancelThreshold)
-        {
-//            std::cout << "****" << songID << " " << itSongID << " " << offset << " " << similarity << std::endl;
-            result.append(boost::python::make_tuple(itSongID, offset, similarity));
-        } /*else {
-            if (similarity < 0 )
-                std::cout << songID << " " << itSongID << " different" << std::endl;
-            else
-                std::cout << songID << " " << itSongID << " " << offset << " " << similarity << std::endl;
-        }*/
 
-    }
+    auto vectorizedFP = std::vector<std::pair<int,std::vector<int>>>(m_fingerprints.begin(), m_fingerprints.end());
 
+    __gnu_parallel::for_each(vectorizedFP.begin(), vectorizedFP.end(),
+        [&](const auto &itSong)
+        {
+            auto & [itSongID, itFingerprint] = itSong;
+            auto [offset, similarity] = compareChromaprintFingerprintsAndOffset(itFingerprint, v, cancelThreshold);
+            if (similarity > cancelThreshold)
+            {
+                result_mutex.lock();
+                result.append(boost::python::make_tuple(itSongID, offset, similarity));
+                result_mutex.unlock();
+            }
+        }, __gnu_parallel::parallel_balanced);
     m_fingerprints[songID]=v;
     return result;
 }
 
-std::pair<int, double> FingerprintManager::compareChromaprintFingerprintsAndOffset(std::vector<int> fp1, std::vector<int> fp2, double cancelThreshold) const
+std::pair<int, double> FingerprintManager::compareChromaprintFingerprintsAndOffset(const std::vector<int> &fp1, const std::vector<int> &fp2, double cancelThreshold) const
 {
     std::vector<int>::const_iterator it1, it2;
     int offset;

diff --git a/setup.py b/setup.py
@@ -3,12 +3,13 @@
 
 module1 = Extension('bard_ext',
                     define_macros=[('MAJOR_VERSION', '1'),
-                                   ('MINOR_VERSION', '0')],
+                                   ('MINOR_VERSION', '0'),
+                                   ('_GLIBCXX_PARALLEL', None)],
                     include_dirs=['/usr/include/boost'],
                     libraries=['boost_python-py3'],
                     library_dirs=['/usr/lib'],
                     sources=['bard/bard_ext.cpp'],
-                    extra_compile_args=['-std=c++1z'])
+                    extra_compile_args=['-std=gnu++17', '-fopenmp'])
 
 setup(
     # Application name: