diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -519,6 +519,7 @@ class UnexpectedSymbolDefinitions : public ErrorInfo<UnexpectedSymbolDefinitions
 /// symbols of an error.
 class MaterializationResponsibility {
   friend class ExecutionSession;
+  friend class JITDylib;
 
 public:
   MaterializationResponsibility(MaterializationResponsibility &&) = delete;
@@ -535,10 +536,10 @@ class MaterializationResponsibility {
 
   /// Returns the target JITDylib that these symbols are being materialized
   ///        into.
-  JITDylib &getTargetJITDylib() const { return *JD; }
+  JITDylib &getTargetJITDylib() const { return JD; }
 
   /// Returns the ExecutionSession for this instance.
-  ExecutionSession &getExecutionSession();
+  ExecutionSession &getExecutionSession() const;
 
   /// Returns the symbol flags map for this responsibility instance.
   /// Note: The returned flags may have transient flags (Lazy, Materializing)
@@ -640,15 +641,16 @@ class MaterializationResponsibility {
 private:
   /// Create a MaterializationResponsibility for the given JITDylib and
   ///        initial symbols.
-  MaterializationResponsibility(JITDylibSP JD, SymbolFlagsMap SymbolFlags,
+  MaterializationResponsibility(ResourceTrackerSP RT,
+                                SymbolFlagsMap SymbolFlags,
                                 SymbolStringPtr InitSymbol)
-      : JD(std::move(JD)), SymbolFlags(std::move(SymbolFlags)),
-        InitSymbol(std::move(InitSymbol)) {
-    assert(this->JD && "Cannot initialize with null JITDylib");
+      : JD(RT->getJITDylib()), RT(std::move(RT)),
+        SymbolFlags(std::move(SymbolFlags)), InitSymbol(std::move(InitSymbol)) {
     assert(!this->SymbolFlags.empty() && "Materializing nothing?");
   }
 
-  JITDylibSP JD;
+  JITDylib &JD;
+  ResourceTrackerSP RT;
   SymbolFlagsMap SymbolFlags;
   SymbolStringPtr InitSymbol;
 };
@@ -913,12 +915,26 @@ class DefinitionGenerator {
                               const SymbolLookupSet &LookupSet) = 0;
 };
 
-/// A symbol table that supports asynchoronous symbol queries.
+/// Represents a JIT'd dynamic library.
+///
+/// This class aims to mimic the behavior of a regular dylib or shared object,
+/// but without requiring the contained program representations to be compiled
+/// up-front. The JITDylib's content is defined by adding MaterializationUnits,
+/// and contained MaterializationUnits will typically rely on the JITDylib's
+/// links-against order to resolve external references (similar to a regular
+/// dylib).
+///
+/// The JITDylib object is a thin wrapper that references state held by the
+/// ExecutionSession. JITDylibs can be removed, clearing this underlying state
+/// and leaving the JITDylib object in a defunct state. In this state the
+/// JITDylib's name is guaranteed to remain accessible. If the ExecutionSession
+/// is still alive then other operations are callable but will return an Error
+/// or null result (depending on the API). It is illegal to call any operation
+/// other than getName on a JITDylib after the ExecutionSession has been torn
+/// down.
 ///
-/// Represents a virtual shared object. Instances can not be copied or moved, so
-/// their addresses may be used as keys for resource management.
-/// JITDylib state changes must be made via an ExecutionSession to guarantee
-/// that they are synchronized with respect to other JITDylib operations.
+/// JITDylibs cannot be moved or copied. Their address is stable, and useful as
+/// a key in some JIT data structures.
 class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
                  public jitlink::JITLinkDylib {
   friend class AsynchronousSymbolQuery;
@@ -931,23 +947,43 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   JITDylib &operator=(const JITDylib &) = delete;
   JITDylib(JITDylib &&) = delete;
   JITDylib &operator=(JITDylib &&) = delete;
+  ~JITDylib();
 
   /// Get a reference to the ExecutionSession for this JITDylib.
+  ///
+  /// It is legal to call this method on a defunct JITDylib, however the result
+  /// will only usable if the ExecutionSession is still alive. If this JITDylib
+  /// is held by an error that may have torn down the JIT then the result
+  /// should not be used.
   ExecutionSession &getExecutionSession() const { return ES; }
 
+  /// Dump current JITDylib state to OS.
+  ///
+  /// It is legal to call this method on a defunct JITDylib.
+  void dump(raw_ostream &OS);
+
   /// Calls remove on all trackers currently associated with this JITDylib.
   /// Does not run static deinits.
   ///
   /// Note that removal happens outside the session lock, so new code may be
   /// added concurrently while the clear is underway, and the newly added
   /// code will *not* be cleared. Adding new code concurrently with a clear
   /// is usually a bug and should be avoided.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   Error clear();
 
   /// Get the default resource tracker for this JITDylib.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   ResourceTrackerSP getDefaultResourceTracker();
 
   /// Create a resource tracker for this JITDylib.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   ResourceTrackerSP createResourceTracker();
 
   /// Adds a definition generator to this JITDylib and returns a referenece to
@@ -956,13 +992,19 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   /// When JITDylibs are searched during lookup, if no existing definition of
   /// a symbol is found, then any generators that have been added are run (in
   /// the order that they were added) to potentially generate a definition.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   template <typename GeneratorT>
   GeneratorT &addGenerator(std::unique_ptr<GeneratorT> DefGenerator);
 
   /// Remove a definition generator from this JITDylib.
   ///
   /// The given generator must exist in this JITDylib's generators list (i.e.
   /// have been added and not yet removed).
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   void removeGenerator(DefinitionGenerator &G);
 
   /// Set the link order to be used when fixing up definitions in JITDylib.
@@ -983,26 +1025,41 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   /// as the first in the link order (instead of this dylib) ensures that
   /// definitions within this dylib resolve to the lazy-compiling stubs,
   /// rather than immediately materializing the definitions in this dylib.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   void setLinkOrder(JITDylibSearchOrder NewSearchOrder,
                     bool LinkAgainstThisJITDylibFirst = true);
 
   /// Add the given JITDylib to the link order for definitions in this
   /// JITDylib.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   void addToLinkOrder(JITDylib &JD,
                       JITDylibLookupFlags JDLookupFlags =
                           JITDylibLookupFlags::MatchExportedSymbolsOnly);
 
   /// Replace OldJD with NewJD in the link order if OldJD is present.
   /// Otherwise this operation is a no-op.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   void replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
                           JITDylibLookupFlags JDLookupFlags =
                               JITDylibLookupFlags::MatchExportedSymbolsOnly);
 
   /// Remove the given JITDylib from the link order for this JITDylib if it is
   /// present. Otherwise this operation is a no-op.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   void removeFromLinkOrder(JITDylib &JD);
 
   /// Do something with the link order (run under the session lock).
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   template <typename Func>
   auto withLinkOrderDo(Func &&F)
       -> decltype(F(std::declval<const JITDylibSearchOrder &>()));
@@ -1014,6 +1071,9 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   ///
   /// This overload always takes ownership of the MaterializationUnit. If any
   /// errors occur, the MaterializationUnit consumed.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   template <typename MaterializationUnitType>
   Error define(std::unique_ptr<MaterializationUnitType> &&MU,
                ResourceTrackerSP RT = nullptr);
@@ -1025,6 +1085,9 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   /// generated. If an error occurs, ownership remains with the caller. This
   /// may allow the caller to modify the MaterializationUnit to correct the
   /// issue, then re-call define.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   template <typename MaterializationUnitType>
   Error define(std::unique_ptr<MaterializationUnitType> &MU,
                ResourceTrackerSP RT = nullptr);
@@ -1039,28 +1102,40 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
   ///
   /// On success, all symbols are removed. On failure, the JITDylib state is
   /// left unmodified (no symbols are removed).
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   Error remove(const SymbolNameSet &Names);
 
-  /// Dump current JITDylib state to OS.
-  void dump(raw_ostream &OS);
-
   /// Returns the given JITDylibs and all of their transitive dependencies in
   /// DFS order (based on linkage relationships). Each JITDylib will appear
   /// only once.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   static std::vector<JITDylibSP> getDFSLinkOrder(ArrayRef<JITDylibSP> JDs);
 
   /// Returns the given JITDylibs and all of their transitive dependensies in
   /// reverse DFS order (based on linkage relationships). Each JITDylib will
   /// appear only once.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   static std::vector<JITDylibSP>
   getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs);
 
   /// Return this JITDylib and its transitive dependencies in DFS order
   /// based on linkage relationships.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   std::vector<JITDylibSP> getDFSLinkOrder();
 
   /// Rteurn this JITDylib and its transitive dependencies in reverse DFS order
   /// based on linkage relationships.
+  ///
+  /// It is illegal to call this method on a defunct JITDylib and the client
+  /// is responsible for ensuring that they do not do so.
   std::vector<JITDylibSP> getReverseDFSLinkOrder();
 
 private:
@@ -1151,7 +1226,6 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
 
   JITDylib(ExecutionSession &ES, std::string Name);
 
-  ResourceTrackerSP getTracker(MaterializationResponsibility &MR);
   std::pair<AsynchronousSymbolQuerySet, std::shared_ptr<SymbolDependenceMap>>
   removeTracker(ResourceTracker &RT);
 
@@ -1197,8 +1271,8 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
       failSymbols(FailedSymbolsWorklist);
 
   ExecutionSession &ES;
+  enum { Open, Closing, Closed } State = Open;
   std::mutex GeneratorsMutex;
-  bool Open = true;
   SymbolTable Symbols;
   UnmaterializedInfosMap UnmaterializedInfos;
   MaterializingInfosMap MaterializingInfos;
@@ -1208,7 +1282,8 @@ class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
 
   // Map trackers to sets of symbols tracked.
   DenseMap<ResourceTracker *, SymbolNameVector> TrackerSymbols;
-  DenseMap<MaterializationResponsibility *, ResourceTracker *> MRTrackers;
+  DenseMap<ResourceTracker *, DenseSet<MaterializationResponsibility *>>
+      TrackerMRs;
 };
 
 /// Platforms set up standard symbols and mediate interactions between dynamic
@@ -1363,6 +1438,18 @@ class ExecutionSession {
   /// If no Platform is attached this call is equivalent to createBareJITDylib.
   Expected<JITDylib &> createJITDylib(std::string Name);
 
+  /// Closes the given JITDylib.
+  ///
+  /// This method clears all resources held for the JITDylib, puts it in the
+  /// closed state, and clears all references held by the ExecutionSession and
+  /// other JITDylibs. No further code can be added to the JITDylib, and the
+  /// object will be freed once any remaining JITDylibSPs to it are destroyed.
+  ///
+  /// This method does *not* run static destructors.
+  ///
+  /// This method can only be called once for each JITDylib.
+  Error removeJITDylib(JITDylib &JD);
+
   /// Set the error reporter function.
   ExecutionSession &setErrorReporter(ErrorReporter ReportError) {
     this->ReportError = std::move(ReportError);
@@ -1574,9 +1661,9 @@ class ExecutionSession {
                                       SymbolStringPtr InitSymbol) {
     auto &JD = RT.getJITDylib();
     std::unique_ptr<MaterializationResponsibility> MR(
-        new MaterializationResponsibility(&JD, std::move(Symbols),
+        new MaterializationResponsibility(&RT, std::move(Symbols),
                                           std::move(InitSymbol)));
-    JD.MRTrackers[MR.get()] = &RT;
+    JD.TrackerMRs[&RT].insert(MR.get());
     return MR;
   }
 
@@ -1660,33 +1747,35 @@ class ExecutionSession {
       JITDispatchHandlers;
 };
 
-inline ExecutionSession &MaterializationResponsibility::getExecutionSession() {
-  return JD->getExecutionSession();
+inline ExecutionSession &
+MaterializationResponsibility::getExecutionSession() const {
+  return JD.getExecutionSession();
 }
 
 template <typename Func>
 Error MaterializationResponsibility::withResourceKeyDo(Func &&F) const {
-  return JD->getExecutionSession().runSessionLocked([&]() -> Error {
-    auto I = JD->MRTrackers.find(this);
-    assert(I != JD->MRTrackers.end() && "No tracker for this MR");
-    if (I->second->isDefunct())
-      return make_error<ResourceTrackerDefunct>(I->second);
-    F(I->second->getKeyUnsafe());
+  return JD.getExecutionSession().runSessionLocked([&]() -> Error {
+    if (RT->isDefunct())
+      return make_error<ResourceTrackerDefunct>(RT);
+    F(RT->getKeyUnsafe());
     return Error::success();
   });
 }
 
 template <typename GeneratorT>
 GeneratorT &JITDylib::addGenerator(std::unique_ptr<GeneratorT> DefGenerator) {
   auto &G = *DefGenerator;
-  std::lock_guard<std::mutex> Lock(GeneratorsMutex);
-  DefGenerators.push_back(std::move(DefGenerator));
+  ES.runSessionLocked([&] {
+    assert(State == Open && "Cannot add generator to closed JITDylib");
+    DefGenerators.push_back(std::move(DefGenerator));
+  });
   return G;
 }
 
 template <typename Func>
 auto JITDylib::withLinkOrderDo(Func &&F)
     -> decltype(F(std::declval<const JITDylibSearchOrder &>())) {
+  assert(State == Open && "Cannot use link order of closed JITDylib");
   return ES.runSessionLocked([&]() { return F(LinkOrder); });
 }
 
@@ -1715,6 +1804,8 @@ Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &&MU,
     });
 
   return ES.runSessionLocked([&, this]() -> Error {
+    assert(State == Open && "JD is defunct");
+
     if (auto Err = defineImpl(*MU))
       return Err;
 
@@ -1756,6 +1847,8 @@ Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &MU,
     });
 
   return ES.runSessionLocked([&, this]() -> Error {
+    assert(State == Open && "JD is defunct");
+
     if (auto Err = defineImpl(*MU))
       return Err;
 
@@ -1800,50 +1893,50 @@ class ReexportsGenerator : public DefinitionGenerator {
 // ---------------------------------------------
 
 inline MaterializationResponsibility::~MaterializationResponsibility() {
-  JD->getExecutionSession().OL_destroyMaterializationResponsibility(*this);
+  getExecutionSession().OL_destroyMaterializationResponsibility(*this);
 }
 
 inline SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const {
-  return JD->getExecutionSession().OL_getRequestedSymbols(*this);
+  return getExecutionSession().OL_getRequestedSymbols(*this);
 }
 
 inline Error MaterializationResponsibility::notifyResolved(
     const SymbolMap &Symbols) {
-  return JD->getExecutionSession().OL_notifyResolved(*this, Symbols);
+  return getExecutionSession().OL_notifyResolved(*this, Symbols);
 }
 
 inline Error MaterializationResponsibility::notifyEmitted() {
-  return JD->getExecutionSession().OL_notifyEmitted(*this);
+  return getExecutionSession().OL_notifyEmitted(*this);
 }
 
 inline Error MaterializationResponsibility::defineMaterializing(
     SymbolFlagsMap SymbolFlags) {
-  return JD->getExecutionSession().OL_defineMaterializing(
-      *this, std::move(SymbolFlags));
+  return getExecutionSession().OL_defineMaterializing(*this,
+                                                      std::move(SymbolFlags));
 }
 
 inline void MaterializationResponsibility::failMaterialization() {
-  JD->getExecutionSession().OL_notifyFailed(*this);
+  getExecutionSession().OL_notifyFailed(*this);
 }
 
 inline Error MaterializationResponsibility::replace(
     std::unique_ptr<MaterializationUnit> MU) {
-  return JD->getExecutionSession().OL_replace(*this, std::move(MU));
+  return getExecutionSession().OL_replace(*this, std::move(MU));
 }
 
 inline Expected<std::unique_ptr<MaterializationResponsibility>>
 MaterializationResponsibility::delegate(const SymbolNameSet &Symbols) {
-  return JD->getExecutionSession().OL_delegate(*this, Symbols);
+  return getExecutionSession().OL_delegate(*this, Symbols);
 }
 
 inline void MaterializationResponsibility::addDependencies(
     const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) {
-  JD->getExecutionSession().OL_addDependencies(*this, Name, Dependencies);
+  getExecutionSession().OL_addDependencies(*this, Name, Dependencies);
 }
 
 inline void MaterializationResponsibility::addDependenciesForAll(
     const SymbolDependenceMap &Dependencies) {
-  JD->getExecutionSession().OL_addDependenciesForAll(*this, Dependencies);
+  getExecutionSession().OL_addDependenciesForAll(*this, Dependencies);
 }
 
 } // End namespace orc

diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -612,9 +612,14 @@ void LookupState::continueLookup(Error Err) {
 
 DefinitionGenerator::~DefinitionGenerator() {}
 
+JITDylib::~JITDylib() {
+  LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
+}
+
 Error JITDylib::clear() {
   std::vector<ResourceTrackerSP> TrackersToRemove;
   ES.runSessionLocked([&]() {
+    assert(State != Closed && "JD is defunct");
     for (auto &KV : TrackerSymbols)
       TrackersToRemove.push_back(KV.first);
     TrackersToRemove.push_back(getDefaultResourceTracker());
@@ -628,6 +633,7 @@ Error JITDylib::clear() {
 
 ResourceTrackerSP JITDylib::getDefaultResourceTracker() {
   return ES.runSessionLocked([this] {
+    assert(State != Closed && "JD is defunct");
     if (!DefaultTracker)
       DefaultTracker = new ResourceTracker(this);
     return DefaultTracker;
@@ -636,19 +642,22 @@ ResourceTrackerSP JITDylib::getDefaultResourceTracker() {
 
 ResourceTrackerSP JITDylib::createResourceTracker() {
   return ES.runSessionLocked([this] {
+    assert(State == Open && "JD is defunct");
     ResourceTrackerSP RT = new ResourceTracker(this);
     return RT;
   });
 }
 
 void JITDylib::removeGenerator(DefinitionGenerator &G) {
-  std::lock_guard<std::mutex> Lock(GeneratorsMutex);
-  auto I = llvm::find_if(DefGenerators,
-                         [&](const std::shared_ptr<DefinitionGenerator> &H) {
-                           return H.get() == &G;
-                         });
-  assert(I != DefGenerators.end() && "Generator not found");
-  DefGenerators.erase(I);
+  ES.runSessionLocked([&] {
+    assert(State == Open && "JD is defunct");
+    auto I = llvm::find_if(DefGenerators,
+                           [&](const std::shared_ptr<DefinitionGenerator> &H) {
+                             return H.get() == &G;
+                           });
+    assert(I != DefGenerators.end() && "Generator not found");
+    DefGenerators.erase(I);
+  });
 }
 
 Expected<SymbolFlagsMap>
@@ -708,10 +717,8 @@ Error JITDylib::replace(MaterializationResponsibility &FromMR,
 
   auto Err =
       ES.runSessionLocked([&, this]() -> Error {
-        auto RT = getTracker(FromMR);
-
-        if (RT->isDefunct())
-          return make_error<ResourceTrackerDefunct>(std::move(RT));
+        if (FromMR.RT->isDefunct())
+          return make_error<ResourceTrackerDefunct>(std::move(FromMR.RT));
 
 #ifndef NDEBUG
         for (auto &KV : MU->getSymbols()) {
@@ -735,18 +742,17 @@ Error JITDylib::replace(MaterializationResponsibility &FromMR,
           if (MII != MaterializingInfos.end()) {
             if (MII->second.hasQueriesPending()) {
               MustRunMR = ES.createMaterializationResponsibility(
-                  *RT, std::move(MU->SymbolFlags), std::move(MU->InitSymbol));
+                  *FromMR.RT, std::move(MU->SymbolFlags),
+                  std::move(MU->InitSymbol));
               MustRunMU = std::move(MU);
               return Error::success();
             }
           }
         }
 
         // Otherwise, make MU responsible for all the symbols.
-        auto RTI = MRTrackers.find(&FromMR);
-        assert(RTI != MRTrackers.end() && "No tracker for FromMR");
-        auto UMI =
-            std::make_shared<UnmaterializedInfo>(std::move(MU), RTI->second);
+        auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU),
+                                                        FromMR.RT.get());
         for (auto &KV : UMI->MU->getSymbols()) {
           auto SymI = Symbols.find(KV.first);
           assert(SymI->second.getState() == SymbolState::Materializing &&
@@ -787,13 +793,11 @@ JITDylib::delegate(MaterializationResponsibility &FromMR,
 
   return ES.runSessionLocked(
       [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
-        auto RT = getTracker(FromMR);
-
-        if (RT->isDefunct())
-          return make_error<ResourceTrackerDefunct>(std::move(RT));
+        if (FromMR.RT->isDefunct())
+          return make_error<ResourceTrackerDefunct>(std::move(FromMR.RT));
 
         return ES.createMaterializationResponsibility(
-            *RT, std::move(SymbolFlags), std::move(InitSymbol));
+            *FromMR.RT, std::move(SymbolFlags), std::move(InitSymbol));
       });
 }
 
@@ -903,10 +907,13 @@ Error JITDylib::resolve(MaterializationResponsibility &MR,
   AsynchronousSymbolQuerySet CompletedQueries;
 
   if (auto Err = ES.runSessionLocked([&, this]() -> Error {
-        auto RTI = MRTrackers.find(&MR);
-        assert(RTI != MRTrackers.end() && "No resource tracker for MR?");
-        if (RTI->second->isDefunct())
-          return make_error<ResourceTrackerDefunct>(RTI->second);
+        if (MR.RT->isDefunct())
+          return make_error<ResourceTrackerDefunct>(MR.RT);
+
+        if (State != Open)
+          return make_error<StringError>("JITDylib " + getName() +
+                                             " is defunct",
+                                         inconvertibleErrorCode());
 
         struct WorklistEntry {
           SymbolTable::iterator SymI;
@@ -1001,10 +1008,13 @@ Error JITDylib::emit(MaterializationResponsibility &MR,
   DenseMap<JITDylib *, SymbolNameVector> ReadySymbols;
 
   if (auto Err = ES.runSessionLocked([&, this]() -> Error {
-        auto RTI = MRTrackers.find(&MR);
-        assert(RTI != MRTrackers.end() && "No resource tracker for MR?");
-        if (RTI->second->isDefunct())
-          return make_error<ResourceTrackerDefunct>(RTI->second);
+        if (MR.RT->isDefunct())
+          return make_error<ResourceTrackerDefunct>(MR.RT);
+
+        if (State != Open)
+          return make_error<StringError>("JITDylib " + getName() +
+                                             " is defunct",
+                                         inconvertibleErrorCode());
 
         SymbolNameSet SymbolsInErrorState;
         std::vector<SymbolTable::iterator> Worklist;
@@ -1149,9 +1159,12 @@ Error JITDylib::emit(MaterializationResponsibility &MR,
 void JITDylib::unlinkMaterializationResponsibility(
     MaterializationResponsibility &MR) {
   ES.runSessionLocked([&]() {
-    auto I = MRTrackers.find(&MR);
-    assert(I != MRTrackers.end() && "MaterializationResponsibility not linked");
-    MRTrackers.erase(I);
+    auto I = TrackerMRs.find(MR.RT.get());
+    assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
+    assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
+    I->second.erase(&MR);
+    if (I->second.empty())
+      TrackerMRs.erase(MR.RT.get());
   });
 }
 
@@ -1169,8 +1182,16 @@ JITDylib::failSymbols(FailedSymbolsWorklist Worklist) {
 
     (*FailedSymbolsMap)[&JD].insert(Name);
 
-    assert(JD.Symbols.count(Name) && "No symbol table entry for Name");
-    auto &Sym = JD.Symbols[Name];
+    // Look up the symbol to fail.
+    auto SymI = JD.Symbols.find(Name);
+
+    // It's possible that this symbol has already been removed, e.g. if a
+    // materialization failure happens concurrently with a ResourceTracker or
+    // JITDylib removal. In that case we can safely skip this symbol and
+    // continue.
+    if (SymI == JD.Symbols.end())
+      continue;
+    auto &Sym = SymI->second;
 
     // Move the symbol into the error state.
     // Note that this may be redundant: The symbol might already have been
@@ -1267,6 +1288,7 @@ JITDylib::failSymbols(FailedSymbolsWorklist Worklist) {
 void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
                             bool LinkAgainstThisJITDylibFirst) {
   ES.runSessionLocked([&]() {
+    assert(State == Open && "JD is defunct");
     if (LinkAgainstThisJITDylibFirst) {
       LinkOrder.clear();
       if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
@@ -1285,6 +1307,7 @@ void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
 void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
                                   JITDylibLookupFlags JDLookupFlags) {
   ES.runSessionLocked([&]() {
+    assert(State == Open && "JD is defunct");
     for (auto &KV : LinkOrder)
       if (KV.first == &OldJD) {
         KV = {&NewJD, JDLookupFlags};
@@ -1295,6 +1318,7 @@ void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
 
 void JITDylib::removeFromLinkOrder(JITDylib &JD) {
   ES.runSessionLocked([&]() {
+    assert(State == Open && "JD is defunct");
     auto I = llvm::find_if(LinkOrder,
                            [&](const JITDylibSearchOrder::value_type &KV) {
                              return KV.first == &JD;
@@ -1306,6 +1330,7 @@ void JITDylib::removeFromLinkOrder(JITDylib &JD) {
 
 Error JITDylib::remove(const SymbolNameSet &Names) {
   return ES.runSessionLocked([&]() -> Error {
+    assert(State == Open && "JD is defunct");
     using SymbolMaterializerItrPair =
         std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>;
     std::vector<SymbolMaterializerItrPair> SymbolsToRemove;
@@ -1365,8 +1390,23 @@ Error JITDylib::remove(const SymbolNameSet &Names) {
 void JITDylib::dump(raw_ostream &OS) {
   ES.runSessionLocked([&, this]() {
     OS << "JITDylib \"" << getName() << "\" (ES: "
-       << format("0x%016" PRIx64, reinterpret_cast<uintptr_t>(&ES)) << "):\n"
-       << "Link order: " << LinkOrder << "\n"
+       << format("0x%016" PRIx64, reinterpret_cast<uintptr_t>(&ES))
+       << ", State = ";
+    switch (State) {
+    case Open:
+      OS << "Open";
+      break;
+    case Closing:
+      OS << "Closing";
+      break;
+    case Closed:
+      OS << "Closed";
+      break;
+    }
+    OS << ")\n";
+    if (State == Closed)
+      return;
+    OS << "Link order: " << LinkOrder << "\n"
        << "Symbol table:\n";
 
     for (auto &KV : Symbols) {
@@ -1454,17 +1494,11 @@ JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
   LinkOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols});
 }
 
-ResourceTrackerSP JITDylib::getTracker(MaterializationResponsibility &MR) {
-  auto I = MRTrackers.find(&MR);
-  assert(I != MRTrackers.end() && "MR is not linked");
-  assert(I->second && "Linked tracker is null");
-  return I->second;
-}
-
 std::pair<JITDylib::AsynchronousSymbolQuerySet,
           std::shared_ptr<SymbolDependenceMap>>
 JITDylib::removeTracker(ResourceTracker &RT) {
   // Note: Should be called under the session lock.
+  assert(State != Closed && "JD is defunct");
 
   SymbolNameVector SymbolsToRemove;
   std::vector<std::pair<JITDylib *, SymbolStringPtr>> SymbolsToFail;
@@ -1525,6 +1559,7 @@ JITDylib::removeTracker(ResourceTracker &RT) {
 }
 
 void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
+  assert(State != Closed && "JD is defunct");
   assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
   assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
   assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
@@ -1536,9 +1571,22 @@ void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
   }
 
   // Update trackers for any active materialization responsibilities.
-  for (auto &KV : MRTrackers) {
-    if (KV.second == &SrcRT)
-      KV.second = &DstRT;
+  {
+    auto I = TrackerMRs.find(&SrcRT);
+    if (I != TrackerMRs.end()) {
+      auto &SrcMRs = I->second;
+      auto &DstMRs = TrackerMRs[&DstRT];
+      for (auto *MR : SrcMRs)
+        MR->RT = &DstRT;
+      if (DstMRs.empty())
+        DstMRs = std::move(SrcMRs);
+      else
+        for (auto *MR : SrcMRs)
+          DstMRs.insert(MR);
+      // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
+      // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
+      TrackerMRs.erase(&SrcRT);
+    }
   }
 
   // If we're transfering to the default tracker we just need to delete the
@@ -1872,6 +1920,40 @@ Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) {
   return JD;
 }
 
+Error ExecutionSession::removeJITDylib(JITDylib &JD) {
+  // Keep JD alive throughout this routine, even if all other references
+  // have been dropped.
+  JITDylibSP JDKeepAlive = &JD;
+
+  // Set JD to 'Closing' state and remove JD from the ExecutionSession.
+  runSessionLocked([&] {
+    assert(JD.State == JITDylib::Open && "JD already closed");
+    JD.State = JITDylib::Closing;
+    auto I = llvm::find(JDs, &JD);
+    assert(I != JDs.end() && "JD does not appear in session JDs");
+    JDs.erase(I);
+  });
+
+  // Clear the JITDylib.
+  auto Err = JD.clear();
+
+  // Set JD to closed state. Clear remaining data structures.
+  runSessionLocked([&] {
+    assert(JD.State == JITDylib::Closing && "JD should be closing");
+    JD.State = JITDylib::Closed;
+    assert(JD.Symbols.empty() && "JD.Symbols is not empty after clear");
+    assert(JD.UnmaterializedInfos.empty() &&
+           "JD.UnmaterializedInfos is not empty after clear");
+    assert(JD.MaterializingInfos.empty() &&
+           "JD.MaterializingInfos is not empty after clear");
+    assert(JD.TrackerSymbols.empty() &&
+           "TrackerSymbols is not empty after clear");
+    JD.DefGenerators.clear();
+    JD.LinkOrder.clear();
+  });
+  return Err;
+}
+
 std::vector<JITDylibSP> JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
   if (JDs.empty())
     return {};
@@ -1883,6 +1965,8 @@ std::vector<JITDylibSP> JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) {
 
     for (auto &JD : JDs) {
 
+      assert(JD->State == Open && "JD is defunct");
+
       if (Visited.count(JD.get()))
         continue;
 
@@ -2311,8 +2395,11 @@ void ExecutionSession::OL_applyQueryPhase1(
       });
 
       // Build the definition generator stack for this JITDylib.
-      for (auto &DG : reverse(JD.DefGenerators))
-        IPLS->CurDefGeneratorStack.push_back(DG);
+      runSessionLocked([&] {
+        IPLS->CurDefGeneratorStack.reserve(JD.DefGenerators.size());
+        for (auto &DG : reverse(JD.DefGenerators))
+          IPLS->CurDefGeneratorStack.push_back(DG);
+      });
 
       // Flag that we've done our initialization.
       IPLS->NewJITDylib = false;
@@ -2635,11 +2722,9 @@ void ExecutionSession::OL_completeLookup(
                  << " MUs.\n";
         });
         for (auto &UMI : KV.second) {
-          std::unique_ptr<MaterializationResponsibility> MR(
-              new MaterializationResponsibility(
-                  &JD, std::move(UMI->MU->SymbolFlags),
-                  std::move(UMI->MU->InitSymbol)));
-          JD.MRTrackers[MR.get()] = UMI->RT;
+          auto MR = createMaterializationResponsibility(
+              *UMI->RT, std::move(UMI->MU->SymbolFlags),
+              std::move(UMI->MU->InitSymbol));
           OutstandingMUs.push_back(
               std::make_pair(std::move(UMI->MU), std::move(MR)));
         }
@@ -2757,18 +2842,18 @@ void ExecutionSession::OL_destroyMaterializationResponsibility(
 
   assert(MR.SymbolFlags.empty() &&
          "All symbols should have been explicitly materialized or failed");
-  MR.JD->unlinkMaterializationResponsibility(MR);
+  MR.JD.unlinkMaterializationResponsibility(MR);
 }
 
 SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
     const MaterializationResponsibility &MR) {
-  return MR.JD->getRequestedSymbols(MR.SymbolFlags);
+  return MR.JD.getRequestedSymbols(MR.SymbolFlags);
 }
 
 Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
                                           const SymbolMap &Symbols) {
   LLVM_DEBUG({
-    dbgs() << "In " << MR.JD->getName() << " resolving " << Symbols << "\n";
+    dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
   });
 #ifndef NDEBUG
   for (auto &KV : Symbols) {
@@ -2783,15 +2868,16 @@ Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
   }
 #endif
 
-  return MR.JD->resolve(MR, Symbols);
+  return MR.JD.resolve(MR, Symbols);
 }
 
 Error ExecutionSession::OL_notifyEmitted(MaterializationResponsibility &MR) {
   LLVM_DEBUG({
-    dbgs() << "In " << MR.JD->getName() << " emitting " << MR.SymbolFlags << "\n";
+    dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
+           << "\n";
   });
 
-  if (auto Err = MR.JD->emit(MR, MR.SymbolFlags))
+  if (auto Err = MR.JD.emit(MR, MR.SymbolFlags))
     return Err;
 
   MR.SymbolFlags.clear();
@@ -2802,10 +2888,11 @@ Error ExecutionSession::OL_defineMaterializing(
     MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
 
   LLVM_DEBUG({
-    dbgs() << "In " << MR.JD->getName() << " defining materializing symbols "
+    dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
            << NewSymbolFlags << "\n";
   });
-  if (auto AcceptedDefs = MR.JD->defineMaterializing(std::move(NewSymbolFlags))) {
+  if (auto AcceptedDefs =
+          MR.JD.defineMaterializing(std::move(NewSymbolFlags))) {
     // Add all newly accepted symbols to this responsibility object.
     for (auto &KV : *AcceptedDefs)
       MR.SymbolFlags.insert(KV);
@@ -2817,14 +2904,14 @@ Error ExecutionSession::OL_defineMaterializing(
 void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
 
   LLVM_DEBUG({
-    dbgs() << "In " << MR.JD->getName() << " failing materialization for "
+    dbgs() << "In " << MR.JD.getName() << " failing materialization for "
            << MR.SymbolFlags << "\n";
   });
 
   JITDylib::FailedSymbolsWorklist Worklist;
 
   for (auto &KV : MR.SymbolFlags)
-    Worklist.push_back(std::make_pair(MR.JD.get(), KV.first));
+    Worklist.push_back(std::make_pair(&MR.JD, KV.first));
   MR.SymbolFlags.clear();
 
   if (Worklist.empty())
@@ -2834,9 +2921,8 @@ void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
   std::shared_ptr<SymbolDependenceMap> FailedSymbols;
 
   runSessionLocked([&]() {
-    auto RTI = MR.JD->MRTrackers.find(&MR);
-    assert(RTI != MR.JD->MRTrackers.end() && "No tracker for this");
-    if (RTI->second->isDefunct())
+    // If the tracker is defunct then there's nothing to do here.
+    if (MR.RT->isDefunct())
       return;
 
     std::tie(FailedQueries, FailedSymbols) =
@@ -2858,12 +2944,12 @@ Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
   if (MU->getInitializerSymbol() == MR.InitSymbol)
     MR.InitSymbol = nullptr;
 
-  LLVM_DEBUG(MR.JD->getExecutionSession().runSessionLocked([&]() {
-    dbgs() << "In " << MR.JD->getName() << " replacing symbols with " << *MU
+  LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
+    dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
            << "\n";
   }););
 
-  return MR.JD->replace(MR, std::move(MU));
+  return MR.JD.replace(MR, std::move(MU));
 }
 
 Expected<std::unique_ptr<MaterializationResponsibility>>
@@ -2886,8 +2972,8 @@ ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
     MR.SymbolFlags.erase(I);
   }
 
-  return MR.JD->delegate(MR, std::move(DelegatedFlags),
-                         std::move(DelegatedInitSymbol));
+  return MR.JD.delegate(MR, std::move(DelegatedFlags),
+                        std::move(DelegatedInitSymbol));
 }
 
 void ExecutionSession::OL_addDependencies(
@@ -2899,7 +2985,7 @@ void ExecutionSession::OL_addDependencies(
   });
   assert(MR.SymbolFlags.count(Name) &&
          "Symbol not covered by this MaterializationResponsibility instance");
-  MR.JD->addDependencies(Name, Dependencies);
+  MR.JD.addDependencies(Name, Dependencies);
 }
 
 void ExecutionSession::OL_addDependenciesForAll(
@@ -2910,7 +2996,7 @@ void ExecutionSession::OL_addDependenciesForAll(
            << Dependencies << "\n";
   });
   for (auto &KV : MR.SymbolFlags)
-    MR.JD->addDependencies(KV.first, Dependencies);
+    MR.JD.addDependencies(KV.first, Dependencies);
 }
 
 #ifndef NDEBUG

diff --git a/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
@@ -1496,4 +1496,57 @@ TEST(JITDylibTest, GetDFSLinkOrderCycle) {
       << "Incorrect DFS link order for libC";
 }
 
+TEST_F(CoreAPIsStandardTest, RemoveJITDylibs) {
+  // Foo will be fully materialized.
+  cantFail(JD.define(absoluteSymbols({{Foo, FooSym}})));
+
+  // Bar should not be materialized at all.
+  bool BarMaterializerDestroyed = false;
+  cantFail(JD.define(std::make_unique<SimpleMaterializationUnit>(
+      SymbolFlagsMap({{Bar, BarSym.getFlags()}}),
+      [&](std::unique_ptr<MaterializationResponsibility> MR) {
+        llvm_unreachable("Unexpected call to materialize");
+      },
+      nullptr,
+      [](const JITDylib &, SymbolStringPtr Name) {
+        llvm_unreachable("Unexpected call to discard");
+      },
+      [&]() { BarMaterializerDestroyed = true; })));
+
+  // Baz will be in the materializing state.
+  std::unique_ptr<MaterializationResponsibility> BazMR;
+  cantFail(JD.define(std::make_unique<SimpleMaterializationUnit>(
+      SymbolFlagsMap({{Baz, BazSym.getFlags()}}),
+      [&](std::unique_ptr<MaterializationResponsibility> MR) {
+        BazMR = std::move(MR);
+      })));
+
+  // Lookup to force materialization of Foo.
+  cantFail(ES.lookup(makeJITDylibSearchOrder(&JD), {Foo}));
+
+  // Start a lookup to force materialization of Baz.
+  bool BazLookupFailed = false;
+  ES.lookup(
+      LookupKind::Static, makeJITDylibSearchOrder(&JD), SymbolLookupSet({Baz}),
+      SymbolState::Ready,
+      [&](Expected<SymbolMap> Result) {
+        if (!Result) {
+          BazLookupFailed = true;
+          consumeError(Result.takeError());
+        }
+      },
+      NoDependenciesToRegister);
+
+  // Remove the JITDylib.
+  auto Err = ES.removeJITDylib(JD);
+  EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+
+  EXPECT_TRUE(BarMaterializerDestroyed);
+  EXPECT_TRUE(BazLookupFailed);
+
+  EXPECT_THAT_ERROR(BazMR->notifyResolved({{Baz, BazSym}}), Failed());
+
+  BazMR->failMaterialization();
+}
+
 } // namespace