diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index a6be8ef6d8bae..3c1319a5c625f 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -25,10 +25,6 @@ def LLVM_Dialect : Dialect {
   let name = "llvm";
   let cppNamespace = "::mlir::LLVM";
 
-  /// FIXME: at the moment this is a dependency of the translation to LLVM IR,
-  /// not really one of this dialect per-se.
-  let dependentDialects = ["omp::OpenMPDialect"];
-
   let hasRegionArgAttrVerify = 1;
   let hasOperationAttrVerify = 1;
   let extraClassDeclaration = [{
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index b87dde3078674..d28c7968c5aef 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -123,7 +123,9 @@ class ModuleTranslation {
 
   /// Builder for LLVM IR generation of OpenMP constructs.
   std::unique_ptr<llvm::OpenMPIRBuilder> ompBuilder;
-  /// Precomputed pointer to OpenMP dialect.
+  /// Precomputed pointer to OpenMP dialect. Note this can be nullptr if the
+  /// OpenMP dialect hasn't been loaded (it is always loaded if there are OpenMP
+  /// operations in the module though).
   const Dialect *ompDialect;
 
   /// Mappings between llvm.mlir.global definitions and corresponding globals.
diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
index ff6560305cb80..9827b86e8b241 100644
--- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
@@ -18,12 +18,10 @@ add_mlir_dialect_library(MLIRLLVMIR
   BitReader
   BitWriter
   Core
-  FrontendOpenMP
 
   LINK_LIBS PUBLIC
   MLIRCallInterfaces
   MLIRControlFlowInterfaces
-  MLIROpenMP
   MLIRIR
   MLIRSideEffectInterfaces
   MLIRSupport
diff --git a/mlir/lib/Target/CMakeLists.txt b/mlir/lib/Target/CMakeLists.txt
index 5ca335b4b4b50..ce67586e60dc9 100644
--- a/mlir/lib/Target/CMakeLists.txt
+++ b/mlir/lib/Target/CMakeLists.txt
@@ -51,6 +51,7 @@ add_mlir_translation_library(MLIRTargetLLVMIR
   IRReader
 
   LINK_LIBS PUBLIC
+  MLIROpenMP
   MLIRTargetLLVMIRModuleTranslation
   )
 
diff --git a/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp
index 89fb5b1e22140..baf9b5eba2c89 100644
--- a/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp
+++ b/mlir/lib/Target/LLVMIR/ConvertToLLVMIR.cpp
@@ -41,6 +41,8 @@ void registerToLLVMIRTranslation() {
         llvmModule->print(output, nullptr);
         return success();
       },
-      [](DialectRegistry &registry) { registry.insert<LLVM::LLVMDialect>(); });
+      [](DialectRegistry &registry) {
+        registry.insert<LLVM::LLVMDialect, omp::OpenMPDialect>();
+      });
 }
 } // namespace mlir
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 8b1b1b02aa118..5e393843fcf5e 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -302,7 +302,7 @@ ModuleTranslation::ModuleTranslation(Operation *module,
     : mlirModule(module), llvmModule(std::move(llvmModule)),
       debugTranslation(
           std::make_unique<DebugTranslation>(module, *this->llvmModule)),
-      ompDialect(module->getContext()->getOrLoadDialect<omp::OpenMPDialect>()),
+      ompDialect(module->getContext()->getLoadedDialect("omp")),
       typeTranslator(this->llvmModule->getContext()) {
   assert(satisfiesLLVMModule(mlirModule) &&
          "mlirModule should honor LLVM's module semantics.");
@@ -639,9 +639,8 @@ LogicalResult ModuleTranslation::convertOperation(Operation &opInst,
     return success();
   }
 
-  if (opInst.getDialect() == ompDialect) {
+  if (ompDialect && opInst.getDialect() == ompDialect)
     return convertOmpOperation(opInst, builder);
-  }
 
   return opInst.emitError("unsupported or non-LLVM operation: ")
          << opInst.getName();