diff --git a/pyannote/audio/pipelines/speaker_diarization.py b/pyannote/audio/pipelines/speaker_diarization.py
index da4d0bb58..89ab11c6e 100644
--- a/pyannote/audio/pipelines/speaker_diarization.py
+++ b/pyannote/audio/pipelines/speaker_diarization.py
@@ -417,7 +417,10 @@ def oracle_cost_func(Y, y):
                         for i in permutation
                     ]
                 )
-                chunk_idx.extend([c] * len(permutation))
+                chunk_idx.extend([
+                        c if ((i is not None) and (active_reference[i])) else -1
+                        for i in permutation
+                ])
 
             oracle_clusters = np.array(oracle_clusters)
             oracle = 1.0 * squareform(pdist(oracle_clusters, metric="equal"))