diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 2a813061a62b6..4d8059145f4c9 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -24,7 +24,6 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/BlockFrequency.h"
 #include "llvm/Support/BranchProbability.h"
 #include "llvm/Support/CommandLine.h"
@@ -548,7 +547,6 @@ namespace bfi_detail {
 template <class BlockT> struct TypeMap {};
 template <> struct TypeMap<BasicBlock> {
   using BlockT = BasicBlock;
-  using BlockKeyT = AssertingVH<const BasicBlock>;
   using FunctionT = Function;
   using BranchProbabilityInfoT = BranchProbabilityInfo;
   using LoopT = Loop;
@@ -556,7 +554,6 @@ template <> struct TypeMap<BasicBlock> {
 };
 template <> struct TypeMap<MachineBasicBlock> {
   using BlockT = MachineBasicBlock;
-  using BlockKeyT = const MachineBasicBlock *;
   using FunctionT = MachineFunction;
   using BranchProbabilityInfoT = MachineBranchProbabilityInfo;
   using LoopT = MachineLoop;
@@ -848,7 +845,6 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
   friend struct bfi_detail::BlockEdgesAdder<BT>;
 
   using BlockT = typename bfi_detail::TypeMap<BT>::BlockT;
-  using BlockKeyT = typename bfi_detail::TypeMap<BT>::BlockKeyT;
   using FunctionT = typename bfi_detail::TypeMap<BT>::FunctionT;
   using BranchProbabilityInfoT =
       typename bfi_detail::TypeMap<BT>::BranchProbabilityInfoT;
@@ -861,11 +857,9 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
   const LoopInfoT *LI = nullptr;
   const FunctionT *F = nullptr;
 
-  class BFICallbackVH;
-
   // All blocks in reverse postorder.
   std::vector<const BlockT *> RPOT;
-  DenseMap<BlockKeyT, std::pair<BlockNode, BFICallbackVH>> Nodes;
+  DenseMap<const BlockT *, BlockNode> Nodes;
 
   using rpot_iterator = typename std::vector<const BlockT *>::const_iterator;
 
@@ -877,8 +871,7 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
   BlockNode getNode(const rpot_iterator &I) const {
     return BlockNode(getIndex(I));
   }
-
-  BlockNode getNode(const BlockT *BB) const { return Nodes.lookup(BB).first; }
+  BlockNode getNode(const BlockT *BB) const { return Nodes.lookup(BB); }
 
   const BlockT *getBlock(const BlockNode &Node) const {
     assert(Node.Index < RPOT.size());
@@ -999,13 +992,6 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
 
   void setBlockFreq(const BlockT *BB, uint64_t Freq);
 
-  void forgetBlock(const BlockT *BB) {
-    // We don't erase corresponding items from `Freqs`, `RPOT` and other to
-    // avoid invalidating indices. Doing so would have saved some memory, but
-    // it's not worth it.
-    Nodes.erase(BB);
-  }
-
   Scaled64 getFloatingBlockFreq(const BlockT *BB) const {
     return BlockFrequencyInfoImplBase::getFloatingBlockFreq(getNode(BB));
   }
@@ -1033,32 +1019,6 @@ template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
   }
 };
 
-template <>
-class BlockFrequencyInfoImpl<BasicBlock>::BFICallbackVH : public CallbackVH {
-  BlockFrequencyInfoImpl<BasicBlock> *BFIImpl;
-
-public:
-  BFICallbackVH() = default;
-
-  BFICallbackVH(const BasicBlock *BB,
-                BlockFrequencyInfoImpl<BasicBlock> *BFIImpl)
-      : CallbackVH(BB), BFIImpl(BFIImpl) {}
-
-  void deleted() override {
-    BFIImpl->forgetBlock(cast<BasicBlock>(getValPtr()));
-  }
-};
-
-/// Dummy implementation since MachineBasicBlocks aren't Values, so ValueHandles
-/// don't apply to them.
-template <>
-class BlockFrequencyInfoImpl<MachineBasicBlock>::BFICallbackVH {
-public:
-  BFICallbackVH() = default;
-  BFICallbackVH(const MachineBasicBlock *,
-                BlockFrequencyInfoImpl<MachineBasicBlock> *) {}
-};
-
 template <class BT>
 void BlockFrequencyInfoImpl<BT>::calculate(const FunctionT &F,
                                            const BranchProbabilityInfoT &BPI,
@@ -1106,7 +1066,7 @@ void BlockFrequencyInfoImpl<BT>::setBlockFreq(const BlockT *BB, uint64_t Freq) {
     // BlockNode for it assigned with a new index. The index can be determined
     // by the size of Freqs.
     BlockNode NewNode(Freqs.size());
-    Nodes[BB] = {NewNode, BFICallbackVH(BB, this)};
+    Nodes[BB] = NewNode;
     Freqs.emplace_back();
     BlockFrequencyInfoImplBase::setBlockFreq(NewNode, Freq);
   }
@@ -1126,7 +1086,7 @@ template <class BT> void BlockFrequencyInfoImpl<BT>::initializeRPOT() {
     BlockNode Node = getNode(I);
     LLVM_DEBUG(dbgs() << " - " << getIndex(I) << ": " << getBlockName(Node)
                       << "\n");
-    Nodes[*I] = {Node, BFICallbackVH(*I, this)};
+    Nodes[*I] = Node;
   }
 
   Working.reserve(RPOT.size());
diff --git a/llvm/include/llvm/IR/ValueHandle.h b/llvm/include/llvm/IR/ValueHandle.h
index 81438c6cdacae..a21d017b193a6 100644
--- a/llvm/include/llvm/IR/ValueHandle.h
+++ b/llvm/include/llvm/IR/ValueHandle.h
@@ -414,7 +414,6 @@ class CallbackVH : public ValueHandleBase {
 public:
   CallbackVH() : ValueHandleBase(Callback) {}
   CallbackVH(Value *P) : ValueHandleBase(Callback, P) {}
-  CallbackVH(const Value *P) : CallbackVH(const_cast<Value *>(P)) {}
 
   operator Value*() const {
     return getValPtr();