diff --git a/Source/Core/Core/HW/DSPLLE/DSPDebugInterface.cpp b/Source/Core/Core/HW/DSPLLE/DSPDebugInterface.cpp
@@ -17,6 +17,11 @@ namespace DSP
 {
 namespace LLE
 {
+void DSPPatches::Patch(std::size_t index)
+{
+  PanicAlert("Patch functionality not supported in DSP module.");
+}
+
 std::size_t DSPDebugInterface::SetWatch(u32 address, const std::string& name)
 {
   return m_watches.SetWatch(address, name);
@@ -87,6 +92,51 @@ void DSPDebugInterface::ClearWatches()
   m_watches.Clear();
 }
 
+void DSPDebugInterface::SetPatch(u32 address, u32 value)
+{
+  m_patches.SetPatch(address, value);
+}
+
+void DSPDebugInterface::SetPatch(u32 address, std::vector<u8> value)
+{
+  m_patches.SetPatch(address, value);
+}
+
+const std::vector<Common::Debug::MemoryPatch>& DSPDebugInterface::GetPatches() const
+{
+  return m_patches.GetPatches();
+}
+
+void DSPDebugInterface::UnsetPatch(u32 address)
+{
+  m_patches.UnsetPatch(address);
+}
+
+void DSPDebugInterface::EnablePatch(std::size_t index)
+{
+  m_patches.EnablePatch(index);
+}
+
+void DSPDebugInterface::DisablePatch(std::size_t index)
+{
+  m_patches.DisablePatch(index);
+}
+
+void DSPDebugInterface::RemovePatch(std::size_t index)
+{
+  m_patches.RemovePatch(index);
+}
+
+bool DSPDebugInterface::HasEnabledPatch(u32 address) const
+{
+  return m_patches.HasEnabledPatch(address);
+}
+
+void DSPDebugInterface::ClearPatches()
+{
+  m_patches.ClearPatches();
+}
+
 std::string DSPDebugInterface::Disassemble(unsigned int address)
 {
   // we'll treat addresses as line numbers.
@@ -202,11 +252,6 @@ void DSPDebugInterface::ToggleMemCheck(unsigned int address, bool read, bool wri
   PanicAlert("MemCheck functionality not supported in DSP module.");
 }
 
-void DSPDebugInterface::Patch(unsigned int address, unsigned int value)
-{
-  PanicAlert("Patch functionality not supported in DSP module.");
-}
-
 // =======================================================
 // Separate the blocks with colors.
 // -------------
@@ -264,6 +309,7 @@ void DSPDebugInterface::RunToBreakpoint()
 
 void DSPDebugInterface::Clear()
 {
+  ClearPatches();
   ClearWatches();
 }
 }  // namespace LLE

diff --git a/Source/Core/Core/HW/DSPLLE/DSPDebugInterface.h b/Source/Core/Core/HW/DSPLLE/DSPDebugInterface.h
@@ -14,6 +14,12 @@ namespace DSP
 {
 namespace LLE
 {
+class DSPPatches : public Common::Debug::MemoryPatches
+{
+private:
+  void Patch(std::size_t index) override;
+};
+
 class DSPDebugInterface final : public DebugInterface
 {
 public:
@@ -34,6 +40,17 @@ class DSPDebugInterface final : public DebugInterface
   std::vector<std::string> SaveWatchesToStrings() const override;
   void ClearWatches() override;
 
+  // Memory Patches
+  void SetPatch(u32 address, u32 value);
+  void SetPatch(u32 address, std::vector<u8> value);
+  const std::vector<Common::Debug::MemoryPatch>& GetPatches() const;
+  void UnsetPatch(u32 address);
+  void EnablePatch(std::size_t index);
+  void DisablePatch(std::size_t index);
+  void RemovePatch(std::size_t index);
+  bool HasEnabledPatch(u32 address) const;
+  void ClearPatches();
+
   std::string Disassemble(unsigned int address) override;
   std::string GetRawMemoryString(int memory, unsigned int address) override;
   int GetInstructionSize(int instruction) override { return 1; }
@@ -53,14 +70,14 @@ class DSPDebugInterface final : public DebugInterface
   void SetPC(unsigned int address) override;
   void Step() override {}
   void RunToBreakpoint() override;
-  void Patch(unsigned int address, unsigned int value) override;
   int GetColor(unsigned int address) override;
   std::string GetDescription(unsigned int address) override;
 
   void Clear() override;
 
 private:
   Common::Debug::Watches m_watches;
+  DSPPatches m_patches;
 };
 }  // namespace LLE
 }  // namespace DSP
diff --git a/Source/Core/DolphinQt2/CheatsManager.cpp b/Source/Core/DolphinQt2/CheatsManager.cpp
@@ -549,7 +549,7 @@ void CheatsManager::Update()
       {
         if (m_watch[i].locked)
         {
-          PowerPC::debug_interface.Patch(m_watch[i].address, m_watch[i].locked_value);
+          PowerPC::debug_interface.SetPatch(m_watch[i].address, m_watch[i].locked_value);
         }
 
         switch (m_watch[i].type)

diff --git a/Source/Core/DolphinQt2/Debugger/CodeViewWidget.cpp b/Source/Core/DolphinQt2/Debugger/CodeViewWidget.cpp
@@ -199,26 +199,8 @@ void CodeViewWidget::SetAddress(u32 address, SetAddressUpdate update)
 
 void CodeViewWidget::ReplaceAddress(u32 address, ReplaceWith replace)
 {
-  auto found = std::find_if(m_repl_list.begin(), m_repl_list.end(),
-                            [address](ReplStruct r) { return r.address == address; });
-
-  if (found != m_repl_list.end())
-  {
-    PowerPC::debug_interface.WriteExtraMemory(0, found->old_value, address);
-    m_repl_list.erase(found);
-  }
-  else
-  {
-    ReplStruct repl;
-
-    repl.address = address;
-    repl.old_value = PowerPC::debug_interface.ReadInstruction(address);
-
-    m_repl_list.push_back(repl);
-
-    PowerPC::debug_interface.Patch(address, replace == ReplaceWith::BLR ? 0x4e800020 : 0x60000000);
-  }
-
+  PowerPC::debug_interface.UnsetPatch(address);
+  PowerPC::debug_interface.SetPatch(address, replace == ReplaceWith::BLR ? 0x4e800020 : 0x60000000);
   Update();
 }
 
@@ -261,6 +243,8 @@ void CodeViewWidget::OnContextMenu()
   auto* insert_nop_action = AddAction(menu, tr("Insert &nop"), this, &CodeViewWidget::OnInsertNOP);
   auto* replace_action =
       AddAction(menu, tr("Re&place instruction"), this, &CodeViewWidget::OnReplaceInstruction);
+  auto* restore_action =
+      AddAction(menu, tr("Restore instruction"), this, &CodeViewWidget::OnRestoreInstruction);
 
   follow_branch_action->setEnabled(running && GetBranchFromAddress(addr));
 
@@ -271,6 +255,8 @@ void CodeViewWidget::OnContextMenu()
   for (auto* action : {symbol_rename_action, symbol_size_action, symbol_end_action})
     action->setEnabled(has_symbol);
 
+  restore_action->setEnabled(running && PowerPC::debug_interface.HasEnabledPatch(addr));
+
   menu->exec(QCursor::pos());
   Update();
 }
@@ -474,11 +460,20 @@ void CodeViewWidget::OnReplaceInstruction()
 
   if (good)
   {
-    PowerPC::debug_interface.Patch(addr, code);
+    PowerPC::debug_interface.UnsetPatch(addr);
+    PowerPC::debug_interface.SetPatch(addr, code);
     Update();
   }
 }
 
+void CodeViewWidget::OnRestoreInstruction()
+{
+  const u32 addr = GetContextAddress();
+
+  PowerPC::debug_interface.UnsetPatch(addr);
+  Update();
+}
+
 void CodeViewWidget::resizeEvent(QResizeEvent*)
 {
   Update();

diff --git a/Source/Core/DolphinQt2/Debugger/CodeViewWidget.h b/Source/Core/DolphinQt2/Debugger/CodeViewWidget.h
@@ -70,14 +70,8 @@ class CodeViewWidget : public QTableWidget
   void OnInsertBLR();
   void OnInsertNOP();
   void OnReplaceInstruction();
+  void OnRestoreInstruction();
 
-  struct ReplStruct
-  {
-    u32 address;
-    u32 old_value;
-  };
-
-  std::vector<ReplStruct> m_repl_list;
   bool m_updating = false;
 
   u32 m_address = 0;

diff --git a/Source/Core/DolphinWX/Debugger/CodeView.cpp b/Source/Core/DolphinWX/Debugger/CodeView.cpp
@@ -45,6 +45,7 @@ enum
   IDM_INSERTBLR,
   IDM_INSERTNOP,
   IDM_ASSEMBLE,
+  IDM_RESTORE,
   IDM_RUNTOHERE,
   IDM_JITRESULTS,
   IDM_FOLLOWBRANCH,
@@ -201,36 +202,10 @@ u32 CCodeView::AddrToBranch(u32 addr)
   return 0;
 }
 
-void CCodeView::InsertBlrNop(int Blr)
+void CCodeView::InsertBlrNop(int blr)
 {
-  // Check if this address has been modified
-  int find = -1;
-  for (u32 i = 0; i < m_blrList.size(); i++)
-  {
-    if (m_blrList.at(i).address == m_selection)
-    {
-      find = i;
-      break;
-    }
-  }
-
-  // Save the old value
-  if (find >= 0)
-  {
-    m_debugger->WriteExtraMemory(0, m_blrList.at(find).oldValue, m_selection);
-    m_blrList.erase(m_blrList.begin() + find);
-  }
-  else
-  {
-    BlrStruct temp;
-    temp.address = m_selection;
-    temp.oldValue = m_debugger->ReadMemory(m_selection);
-    m_blrList.push_back(temp);
-    if (Blr == 0)
-      m_debugger->Patch(m_selection, 0x4e800020);
-    else
-      m_debugger->Patch(m_selection, 0x60000000);
-  }
+  m_debugger->UnsetPatch(m_selection);
+  m_debugger->SetPatch(m_selection, (blr == 0) ? 0x4e800020 : 0x60000000);
   Refresh();
 }
 
@@ -320,13 +295,19 @@ void CCodeView::OnPopupMenu(wxCommandEvent& event)
       unsigned long code;
       if (dialog.GetValue().ToULong(&code, 0) && code <= std::numeric_limits<u32>::max())
       {
-        m_debugger->Patch(m_selection, code);
+        m_debugger->UnsetPatch(m_selection);
+        m_debugger->SetPatch(m_selection, code);
         Refresh();
       }
     }
     break;
   }
 
+  case IDM_RESTORE:
+    m_debugger->UnsetPatch(m_selection);
+    Refresh();
+    break;
+
   case IDM_JITRESULTS:
   {
     // Propagate back to the parent window and tell it
@@ -451,6 +432,8 @@ void CCodeView::OnMouseUpR(wxMouseEvent& event)
   menu.Append(IDM_INSERTBLR, _("&Insert blr"))->Enable(Core::IsRunning());
   menu.Append(IDM_INSERTNOP, _("Insert &nop"))->Enable(Core::IsRunning());
   menu.Append(IDM_ASSEMBLE, _("Re&place instruction"))->Enable(Core::IsRunning());
+  menu.Append(IDM_RESTORE, _("Restore instruction"))
+      ->Enable(Core::IsRunning() && m_debugger->HasEnabledPatch(m_selection));
   // menu.Append(IDM_PATCHALERT, _("Patch alert"))->Enable(Core::IsRunning());
   PopupMenu(&menu);
   event.Skip();

diff --git a/Source/Core/DolphinWX/Debugger/CodeView.h b/Source/Core/DolphinWX/Debugger/CodeView.h
@@ -55,13 +55,6 @@ class CCodeView : public wxControl
   u32 AddrToBranch(u32 addr);
   void OnResize(wxSizeEvent& event);
 
-  struct BlrStruct  // for IDM_INSERTBLR
-  {
-    u32 address;
-    u32 oldValue;
-  };
-  std::vector<BlrStruct> m_blrList;
-
   static constexpr int LEFT_COL_WIDTH = 16;
 
   DebugInterface* m_debugger;