diff --git a/src/governance.cpp b/src/governance.cpp
index 7017231a6f372..5ce8ac5ee48e9 100644
--- a/src/governance.cpp
+++ b/src/governance.cpp
@@ -557,10 +557,6 @@ void CGovernanceManager::DoMaintenance(CConnman& connman)
 {
     if (fLiteMode || !masternodeSync.IsSynced() || ShutdownRequested()) return;
 
-    if (deterministicMNManager->IsDIP3Enforced()) {
-        RemoveInvalidVotes();
-    }
-
     // CHECK OBJECTS WE'VE ASKED FOR, REMOVE OLD ENTRIES
 
     CleanOrphanObjects();
@@ -1330,11 +1326,15 @@ void CGovernanceManager::CleanOrphanObjects()
 
 void CGovernanceManager::RemoveInvalidVotes()
 {
-    auto curMNList = deterministicMNManager->GetListAtChainTip();
-    auto diff = lastMNListForVotingKeys.BuildDiff(curMNList);
+    if (!masternodeSync.IsSynced()) {
+        return;
+    }
 
     LOCK(cs);
 
+    auto curMNList = deterministicMNManager->GetListAtChainTip();
+    auto diff = lastMNListForVotingKeys.BuildDiff(curMNList);
+
     std::vector<COutPoint> changedKeyMNs;
     for (const auto& p : diff.updatedMNs) {
         auto oldDmn = lastMNListForVotingKeys.GetMN(p.first);