diff --git a/modules/wallet/update.go b/modules/wallet/update.go
index 70b959fd3a..cb5ea4153f 100644
--- a/modules/wallet/update.go
+++ b/modules/wallet/update.go
@@ -455,6 +455,29 @@ func (w *Wallet) ProcessConsensusChange(cc modules.ConsensusChange) {
 	}
 }
 
+// isRelevantTxn checks if a tranaction is relevant to the wallet
+func (w *Wallet) isRelevantTxn(txn types.Transaction) (relevant bool) {
+	// determine whether transaction is relevant to the wallet
+	for _, sci := range txn.SiacoinInputs {
+		relevant = relevant || w.isWalletAddress(sci.UnlockConditions.UnlockHash())
+	}
+	for _, sco := range txn.SiacoinOutputs {
+		relevant = relevant || w.isWalletAddress(sco.UnlockHash)
+	}
+	return
+}
+
+// isRelevantTSet checks if a set of transactions is relevant to the wallet. It
+// is relevant if at least one transaction is relevant.
+func (w *Wallet) isRelevantTSet(txns []types.Transaction) bool {
+	for _, txn := range txns {
+		if w.isRelevantTxn(txn) {
+			return true
+		}
+	}
+	return false
+}
+
 // isSuperset is a helper function that checks if super is a superset of sub
 func isSuperset(super []types.TransactionID, sub []types.TransactionID) bool {
 	if len(super) < len(sub) {
@@ -504,11 +527,13 @@ func (w *Wallet) ReceiveUpdatedUnconfirmedTransactions(diff *modules.Transaction
 	// unconfirmedProcessedTransactions to no longer have the dropped
 	// transactions.
 	if len(droppedTransactions) != 0 {
-		// Capacity can't be reduced, because we have no way of knowing if the
-		// dropped transactions are relevant to the wallet or not, and some will
-		// not be relevant to the wallet, meaning they don't have a counterpart
-		// in w.unconfirmedProcessedTransactions.
-		newUPT := make([]modules.ProcessedTransaction, 0, len(w.unconfirmedProcessedTransactions))
+		// droppedTransactions should only contain transactions relevant to the
+		// wallet. Therefore we can safely reduce the allocated memory.
+		newLen := len(w.unconfirmedProcessedTransactions) - len(droppedTransactions)
+		if newLen < 0 {
+			newLen = 0
+		}
+		newUPT := make([]modules.ProcessedTransaction, 0, newLen)
 		for _, txn := range w.unconfirmedProcessedTransactions {
 			_, exists := droppedTransactions[txn.TransactionID]
 			if !exists {
@@ -524,12 +549,15 @@ func (w *Wallet) ReceiveUpdatedUnconfirmedTransactions(diff *modules.Transaction
 
 	// Scroll through all of the diffs and add any new transactions.
 	for _, unconfirmedTxnSet := range diff.AppliedTransactions {
+		// We only need to do that for transactions relevant to the wallet
+		if !w.isRelevantTSet(unconfirmedTxnSet.Transactions) {
+			continue
+		}
+
 		// Mark all of the transactions that appeared in this set.
 		//
-		// TODO: Technically only necessary to mark the ones that are relevant
-		// to the wallet, but overhead should be low.
-		//
-		// Check if commitTransactionSet already added the transaction
+		// Check if commitTransactionSet already added the transaction to the
+		// unconfirmedSets
 		if _, exists := w.unconfirmedSets[unconfirmedTxnSet.ID]; !exists {
 			// Maybe acceptTransactionSet added a subset with a different
 			// id. If that's the case, replace it with the one from the
@@ -574,17 +602,8 @@ func (w *Wallet) ReceiveUpdatedUnconfirmedTransactions(diff *modules.Transaction
 
 		// Add each transaction to our set of unconfirmed transactions.
 		for i, txn := range unconfirmedTxnSet.Transactions {
-			// determine whether transaction is relevant to the wallet
-			relevant := false
-			for _, sci := range txn.SiacoinInputs {
-				relevant = relevant || w.isWalletAddress(sci.UnlockConditions.UnlockHash())
-			}
-			for _, sco := range txn.SiacoinOutputs {
-				relevant = relevant || w.isWalletAddress(sco.UnlockHash)
-			}
-
 			// only create a ProcessedTransaction if txn is relevant
-			if !relevant {
+			if !w.isRelevantTxn(txn) {
 				continue
 			}