diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index 383fd9d94661a..077e19d429e14 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -1434,6 +1434,20 @@ class OpenMPIRBuilder { /// \param Loop The loop to unroll. The loop will be invalidated. LLVM_ABI void unrollLoopFull(DebugLoc DL, CanonicalLoopInfo *Loop); + /// Interchange the order of the nested loops. + /// + /// @param DL Debug location for instructions added by interchange. + /// @param Loops Loops affected by loop interchange. The + /// CanonicalLoopInfo objects are invalidated by this + /// method, i.e. should not be used after interchange. + /// @param Permutation The new order in wich the \p Loops will be arranged. + /// + /// \returns A list of generated loops. Contains the same loops as the input + /// loop nest reordered. + LLVM_ABI std::vector + interchangeLoops(DebugLoc DL, ArrayRef Loops, + ArrayRef Permutation); + /// Fully or partially unroll a loop. How the loop is unrolled is determined /// using LLVM's LoopUnrollPass. /// diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index d149593795267..e5150df40e3d6 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -6783,6 +6783,76 @@ void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) { }); } +std::vector +OpenMPIRBuilder::interchangeLoops(DebugLoc DL, + ArrayRef Loops, + ArrayRef Permutation) { + assert(Permutation.size() == Loops.size() && + "The permutation must have every input loop"); + int NumLoops = Loops.size(); + assert(NumLoops >= 2 && "At least two loops to interchange required"); + + CanonicalLoopInfo *OutermostLoop = Loops.front(); + CanonicalLoopInfo *InnermostLoop = Loops.back(); + Function *F = OutermostLoop->getBody()->getParent(); + + // Loop control blocks that may become orphaned later. + SmallVector OldControlBBs; + for (CanonicalLoopInfo *Loop : Loops) { + Loop->collectControlBlocks(OldControlBBs); + } + + // Create the new loop nest structure + std::vector Result; + + BasicBlock *Enter = OutermostLoop->getPreheader(); + BasicBlock *Continue = OutermostLoop->getAfter(); + BasicBlock *OutroInsertBefore = InnermostLoop->getExit(); + + for (int i = 0; i < NumLoops; i++) { + int LoopIndex = Permutation[i] - 1; + assert(Loops[LoopIndex]->isValid() && + "All input loops must be valid canonical loops"); + + CanonicalLoopInfo *newLoop = createLoopSkeleton( + DL, Loops[LoopIndex]->getTripCount(), F, InnermostLoop->getBody(), + OutroInsertBefore, "interchange" + Twine(i)); + redirectTo(Enter, newLoop->getPreheader(), DL); + redirectTo(newLoop->getAfter(), Continue, DL); + + // Setup the position where the next loop connects to this loop. + Enter = newLoop->getBody(); + Continue = newLoop->getLatch(); + OutroInsertBefore = newLoop->getLatch(); + + Result.push_back(newLoop); + } + + // Append the original loop nest body into the generated loop nest body. + redirectTo(Enter, InnermostLoop->getBody(), DL); + redirectAllPredecessorsTo(InnermostLoop->getLatch(), Continue, DL); + + // Replace the original induction variable with the new induction variable + Builder.restoreIP(Result.back()->getBodyIP()); + for (int i = 0; i < NumLoops; ++i) { + int LoopIndex = Permutation[i] - 1; + Value *OrigIndVar = Loops[LoopIndex]->getIndVar(); + OrigIndVar->replaceAllUsesWith(Result[i]->getIndVar()); + } + + // Remove unused parts of the original loops. + removeUnusedBlocksFromParent(OldControlBBs); + + for (CanonicalLoopInfo *L : Loops) + L->invalidate(); + +#ifndef NDEBUG + for (CanonicalLoopInfo *GenL : Result) + GenL->assertOK(); +#endif + return Result; +} + void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop, Value *IfCond, ValueToValueMapTy &VMap, LoopAnalysis &LIA, LoopInfo &LI, Loop *L, diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td index 23c2fbdfd7368..72694f4edfe8d 100644 --- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td +++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauses.td @@ -1148,6 +1148,30 @@ class OpenMP_LooprangeClauseSkip< def OpenMP_LooprangeClause : OpenMP_LooprangeClauseSkip<>; +//===----------------------------------------------------------------------===// +// V6.0: `permutation` clause +//===----------------------------------------------------------------------===// + +class OpenMP_PermutationClauseSkip< + bit traits = false, bit arguments = false, bit assemblyFormat = false, + bit description = false, bit extraClassDeclaration = false> + : OpenMP_Clause { + let arguments = (ins OptionalAttr:$permutation); + + let optAssemblyFormat = [{ + `permutation` `(` $permutation `)` + }]; + + let description = [{ + The `permutation` clause contains the new order in which the loops affected + by a loop interchange are arranged. The clause contains a list of constant + positive numbers. + }]; +} + +def OpenMP_PermutationClause : OpenMP_PermutationClauseSkip<>; + //===----------------------------------------------------------------------===// // V5.2: [10.1.2] `num_threads` clause //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td index 88c8ab4f6f949..64743d3e7c365 100644 --- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td +++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td @@ -577,6 +577,25 @@ def FuseOp let hasVerifier = 1; } +//===----------------------------------------------------------------------===// +// OpenMP interchange operation +//===----------------------------------------------------------------------===// + +def InterchangeOp + : OpenMPTransformBase_Op<"interchange", + clauses = [OpenMP_PermutationClause]> { + let summary = "OpenMP interchange operation"; + let description = [{ + Represents the OpenMP interchange directive introduced in OpenMP 6.0. + + The construct takes a loop nest and rearranges the affected loops in the + order specified by the `permutation` clause. The `permutation clause must + be present and its elements known beforehand. + }]#clausesDescription; + + let hasVerifier = 1; +} + //===----------------------------------------------------------------------===// // 2.8.3 Workshare Construct //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp index 7cab929d583ca..9c8cedde76d6d 100644 --- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp +++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp @@ -3518,6 +3518,8 @@ void NewCliOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) { else return "fused"; }) + .Case( + [&](InterchangeOp op) -> std::string { return "interchanged"; }) .Case([&](TileOp op) -> std::string { auto [generateesFirst, generateesCount] = op.getGenerateesODSOperandIndexAndLength(); @@ -3812,7 +3814,8 @@ static ParseResult parseLoopTransformClis( /// (loop bounds are invariant in respect to the outer loops) /// /// TODO: Generalize for LoopTransformationInterface. -static LogicalResult checkApplyeesNesting(TileOp op) { +template +static LogicalResult checkApplyeesNesting(OpenMPTransform op) { // Collect the loops from the nest bool isOnlyCanonLoops = true; SmallVector canonLoops; @@ -3839,8 +3842,8 @@ static LogicalResult checkApplyeesNesting(TileOp op) { auto loop = canonLoops[i]; if (parentLoop.getOperation() != loop.getOperation()->getParentOp()) - return op.emitOpError() - << "tiled loop nest must be nested within each other"; + return op.emitOpError() << "OpenMP transformation loop nest must be " + "nested within each other"; parentIVs.insert(parentLoop.getInductionVar()); @@ -3868,10 +3871,12 @@ static LogicalResult checkApplyeesNesting(TileOp op) { return true; }(); if (!isPerfectlyNested) - return op.emitOpError() << "tiled loop nest must be perfectly nested"; + return op.emitOpError() + << "OpenMP transformation loop nest must be perfectly nested"; if (parentIVs.contains(loop.getTripCount())) - return op.emitOpError() << "tiled loop nest must be rectangular"; + return op.emitOpError() + << "OpenMP transformation loop nest must be rectangular"; } // TODO: The tile sizes must be computed before the loop, but checking this @@ -3965,6 +3970,68 @@ std::pair FuseOp::getGenerateesODSOperandIndexAndLength() { return getODSOperandIndexAndLength(odsIndex_generatees); } +//===----------------------------------------------------------------------===// +// InterchangeOp +//===----------------------------------------------------------------------===// + +static void printLoopTransformClis(OpAsmPrinter &p, InterchangeOp op, + OperandRange generatees, + OperandRange applyees) { + if (!generatees.empty()) + p << '(' << llvm::interleaved(generatees) << ')'; + + if (!applyees.empty()) + p << " <- (" << llvm::interleaved(applyees) << ')'; +} + +LogicalResult InterchangeOp::verify() { + if (getApplyees().size() < 2) + return emitOpError() << "must apply to at least two loops"; + + if (!getPermutation().has_value()) + return emitOpError() << "must have permutation attribute"; + + auto permutation = getPermutation().value(); + if (permutation.size() != getApplyees().size()) + return emitOpError() << "expecting the same number of permutation " + "attributes and applyees"; + + llvm::SmallVector found(permutation.size(), false); + for (auto &val : permutation) { + if (auto intAttr = llvm::dyn_cast(val)) { + int perm = intAttr.getInt(); + if (perm <= 0) + return emitOpError() + << "permutation attribute must be a positive integer"; + if ((size_t)perm - 1 < permutation.size()) + found[perm - 1] = true; + } else + return emitOpError() << "permutation attribute must be of integer type"; + } + for (bool b : found) { + if (!b) + return emitOpError() + << "every integer from 1 must appear in the permutation attribute"; + } + + if (!getGeneratees().empty() && + getApplyees().size() != getGeneratees().size()) + return emitOpError() + << "expecting the same number of generatees and applyees"; + + return checkApplyeesNesting(*this); +} + +std::pair +InterchangeOp::getApplyeesODSOperandIndexAndLength() { + return getODSOperandIndexAndLength(odsIndex_applyees); +} + +std::pair +InterchangeOp::getGenerateesODSOperandIndexAndLength() { + return getODSOperandIndexAndLength(odsIndex_generatees); +} + //===----------------------------------------------------------------------===// // Critical construct (2.17.1) //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index 42fdadaba0da1..d83ca745cef7b 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -4154,6 +4154,46 @@ static LogicalResult applyFuse(omp::FuseOp op, llvm::IRBuilderBase &builder, return success(); } +/// Apply a `#pragma omp interchange` / `!$omp interchange` transformation using +/// the OpenMPIRBuilder. +static LogicalResult +applyInterchange(omp::InterchangeOp op, llvm::IRBuilderBase &builder, + LLVM::ModuleTranslation &moduleTranslation) { + llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); + llvm::OpenMPIRBuilder::LocationDescription loc(builder); + + SmallVector translatedLoops; + + for (Value applyee : op.getApplyees()) { + llvm::CanonicalLoopInfo *consBuilderCLI = + moduleTranslation.lookupOMPLoop(applyee); + assert(applyee && "Canonical loop must already been translated"); + translatedLoops.push_back(consBuilderCLI); + } + + auto perm = op.getPermutation().value(); + SmallVector permutation; + for (auto &val : perm) { + auto intVal = llvm::dyn_cast(val); + assert(intVal && "permutation attributes must be integers"); + permutation.push_back(intVal.getInt()); + } + + auto generatedLoops = + ompBuilder->interchangeLoops(loc.DL, translatedLoops, permutation); + if (!op.getGeneratees().empty()) { + for (auto [mlirLoop, genLoop] : + zip_equal(op.getGeneratees(), generatedLoops)) + moduleTranslation.mapOmpLoop(mlirLoop, genLoop); + } + + // CLIs can only be consumed once + for (Value applyee : op.getApplyees()) + moduleTranslation.invalidateOmpLoop(applyee); + + return success(); +} + /// Convert an Atomic Ordering attribute to llvm::AtomicOrdering. static llvm::AtomicOrdering convertAtomicOrdering(std::optional ao) { @@ -7650,6 +7690,9 @@ LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation( .Case([&](omp::FuseOp op) { return applyFuse(op, builder, moduleTranslation); }) + .Case([&](omp::InterchangeOp op) { + return applyInterchange(op, builder, moduleTranslation); + }) .Case([&](omp::TargetAllocMemOp) { return convertTargetAllocMemOp(*op, builder, moduleTranslation); }) diff --git a/mlir/test/Dialect/OpenMP/cli-interchange.mlir b/mlir/test/Dialect/OpenMP/cli-interchange.mlir new file mode 100644 index 0000000000000..3fd2df2822b12 --- /dev/null +++ b/mlir/test/Dialect/OpenMP/cli-interchange.mlir @@ -0,0 +1,95 @@ +// RUN: mlir-opt %s | FileCheck %s --enable-var-scope +// RUN: mlir-opt %s | mlir-opt | FileCheck %s --enable-var-scope + + +// Raw syntax check (MLIR output is always pretty-printed) +// CHECK-LABEL: @omp_interchange_raw( +// CHECK-SAME: %[[tc1:.+]]: i32, %[[tc2:.+]]: i32) { +func.func @omp_interchange_raw(%tc1 : i32, %tc2 : i32) -> () { + // CHECK-NEXT: %canonloop = omp.new_cli + %cli_outer = "omp.new_cli" () : () -> (!omp.cli) + // CHECK-NEXT: %canonloop_d1 = omp.new_cli + %cli_inner = "omp.new_cli" () : () -> (!omp.cli) + // CHECK-NEXT: %interchanged = omp.new_cli + %interchange1 = "omp.new_cli" () : () -> (!omp.cli) + // CHECK-NEXT: %interchanged_0 = omp.new_cli + %interchange2 = "omp.new_cli" () : () -> (!omp.cli) + // CHECK-NEXT: omp.canonical_loop(%canonloop) %iv : i32 in range(%[[tc1]]) { + "omp.canonical_loop" (%tc1, %cli_outer) ({ + ^bb0(%iv: i32): + // CHECK-NEXT: omp.canonical_loop(%canonloop_d1) %iv_d1 : i32 in range(%[[tc2]]) { + "omp.canonical_loop" (%tc2, %cli_inner) ({ + ^bb0(%iv_d1: i32): + // CHECK: omp.terminator + omp.terminator + }) : (i32, !omp.cli) -> () + // CHECK: omp.terminator + omp.terminator + }) : (i32, !omp.cli) -> () + // CHECK: omp.interchange (%interchanged, %interchanged_0) <- (%canonloop, %canonloop_d1) + // CHECK-SAME: permutation([2 : i32, 1 : i32]) + "omp.interchange" (%interchange1, %interchange2, %cli_outer, %cli_inner) <{operandSegmentSizes = array, permutation = [2 : i32, 1 : i32]}> : (!omp.cli, !omp.cli, !omp.cli, !omp.cli) -> () + return +} + + +// Pretty syntax check +// CHECK-LABEL: @omp_interchange_pretty( +// CHECK-SAME: %[[tc1:.+]]: i32, %[[tc2:.+]]: i32) { +func.func @omp_interchange_pretty(%tc1 : i32, %tc2 : i32) -> () { + // CHECK-NEXT: %canonloop = omp.new_cli + %cli_outer = omp.new_cli + // CHECK-NEXT: %canonloop_d1 = omp.new_cli + %cli_inner = omp.new_cli + // CHECK-NEXT: %interchanged = omp.new_cli + %interchange1 = omp.new_cli + // CHECK-NEXT: %interchanged_0 = omp.new_cli + %interchange2 = omp.new_cli + // CHECK-NEXT: omp.canonical_loop(%canonloop) %iv : i32 in range(%[[tc1]]) { + omp.canonical_loop(%cli_outer) %iv_outer : i32 in range(%tc1) { + // CHECK-NEXT: omp.canonical_loop(%canonloop_d1) %iv_d1 : i32 in range(%[[tc2]]) { + omp.canonical_loop(%cli_inner) %iv_inner : i32 in range(%tc2) { + // CHECK: omp.terminator + omp.terminator + } + // CHECK: omp.terminator + omp.terminator + } + // CHECK: omp.interchange (%interchanged, %interchanged_0) <- (%canonloop, %canonloop_d1) + // CHECK-SAME: permutation([2 : i32, 1 : i32]) + omp.interchange (%interchange1, %interchange2) <- (%cli_outer, %cli_inner) permutation([2 : i32, 1 : i32]) + return +} + + +// Composition of multiple interchanges +// CHECK-LABEL: @omp_interchange_composition( +// CHECK-SAME: %[[tc1:.+]]: i32, %[[tc2:.+]]: i32) { +func.func @omp_interchange_composition(%tc1 : i32, %tc2 : i32) -> () { + %cli_outer = omp.new_cli + %cli_inner = omp.new_cli + %interchange1 = omp.new_cli + %interchange2 = omp.new_cli + %interchange3 = omp.new_cli + %interchange4 = omp.new_cli + + // CHECK: omp.canonical_loop(%canonloop) %iv : i32 in range(%[[tc1]]) { + omp.canonical_loop(%cli_outer) %iv_outer : i32 in range(%tc1) { + // CHECK: omp.canonical_loop(%canonloop_d1) %iv_d1 : i32 in range(%[[tc2]]) { + omp.canonical_loop(%cli_inner) %iv_inner : i32 in range(%tc2) { + // CHECK: omp.terminator + omp.terminator + } + // CHECK: omp.terminator + omp.terminator + } + // CHECK: omp.interchange (%interchanged, %interchanged_0) <- (%canonloop, %canonloop_d1) + // CHECK-SAME: permutation([2 : i32, 1 : i32]) + omp.interchange (%interchange1, %interchange2) <- (%cli_outer, %cli_inner) permutation([2 : i32, 1 : i32]) + + // CHECK: omp.interchange (%interchanged_1, %interchanged_2) <- (%interchanged, %interchanged_0) + // CHECK-SAME: permutation([1 : i32, 2 : i32]) + omp.interchange (%interchange3, %interchange4) <- (%interchange1, %interchange2) permutation([1 : i32, 2 : i32]) + return +} + diff --git a/mlir/test/Dialect/OpenMP/invalid-interchange.mlir b/mlir/test/Dialect/OpenMP/invalid-interchange.mlir new file mode 100644 index 0000000000000..d0be42a82f6c4 --- /dev/null +++ b/mlir/test/Dialect/OpenMP/invalid-interchange.mlir @@ -0,0 +1,201 @@ +// RUN: mlir-opt -split-input-file -verify-diagnostics %s + + +func.func @missing_permutation(%tc1 : i32, %tc2 : i32) { + %canonloop1 = omp.new_cli + %canonloop2 = omp.new_cli + omp.canonical_loop(%canonloop1) %iv1 : i32 in range(%tc1) { + omp.canonical_loop(%canonloop2) %iv2 : i32 in range(%tc2) { + omp.terminator + } + omp.terminator + } + + // expected-error@+1 {{'omp.interchange' op must have permutation attribute}} + omp.interchange <-(%canonloop1, %canonloop2) + + llvm.return +} + +// ----- + +func.func @no_loops(%tc1 : i32, %tc2 : i32) { + // expected-error@+1 {{'omp.interchange' op must apply to at least two loops}} + omp.interchange <-() permutation([2, 1]) + + return +} + +// ----- + +func.func @missing_loops(%tc1 : i32, %tc2 : i32) { + %canonloop1 = omp.new_cli + omp.canonical_loop(%canonloop1) %iv1 : i32 in range(%tc1) { + omp.terminator + } + + // expected-error@+1 {{'omp.interchange' op must apply to at least two loops}} + omp.interchange <-(%canonloop1) permutation([2, 1]) + + llvm.return +} + +// ----- + +func.func @wrong_permutation(%tc1 : i32, %tc2 : i32) { + %canonloop1 = omp.new_cli + %canonloop2 = omp.new_cli + omp.canonical_loop(%canonloop1) %iv1 : i32 in range(%tc1) { + omp.canonical_loop(%canonloop2) %iv2 : i32 in range(%tc2) { + omp.terminator + } + omp.terminator + } + + // expected-error@+1 {{'omp.interchange' op expecting the same number of permutation attributes and applyees}} + omp.interchange <-(%canonloop1, %canonloop2) permutation([1 : i32]) + + llvm.return +} + +// ----- + +func.func @insufficient_generatees(%tc1 : i32, %tc2 : i32) { + %canonloop1 = omp.new_cli + %canonloop2 = omp.new_cli + %canonloop3 = omp.new_cli + omp.canonical_loop(%canonloop1) %iv1 : i32 in range(%tc1) { + omp.canonical_loop(%canonloop2) %iv2 : i32 in range(%tc2) { + omp.terminator + } + omp.terminator + } + + // expected-error@+1 {{'omp.interchange' op expecting the same number of generatees and applyees}} + omp.interchange (%canonloop3) <- (%canonloop1, %canonloop2) permutation([1 : i32, 2 : i32]) + + return +} + +// ----- + +func.func @zero_attribute(%tc1 : i32, %tc2 : i32) { + %canonloop1 = omp.new_cli + %canonloop2 = omp.new_cli + %canonloop3 = omp.new_cli + %canonloop4 = omp.new_cli + omp.canonical_loop(%canonloop1) %iv1 : i32 in range(%tc1) { + omp.canonical_loop(%canonloop2) %iv2 : i32 in range(%tc2) { + omp.terminator + } + omp.terminator + } + + // expected-error@+1 {{'omp.interchange' op permutation attribute must be a positive integer}} + omp.interchange (%canonloop3, %canonloop4) <- (%canonloop1, %canonloop2) permutation([0 : i32, 2 : i32]) + + return +} + +// ----- + +func.func @float_attribute(%tc1 : i32, %tc2 : i32) { + %canonloop1 = omp.new_cli + %canonloop2 = omp.new_cli + %canonloop3 = omp.new_cli + %canonloop4 = omp.new_cli + omp.canonical_loop(%canonloop1) %iv1 : i32 in range(%tc1) { + omp.canonical_loop(%canonloop2) %iv2 : i32 in range(%tc2) { + omp.terminator + } + omp.terminator + } + + // expected-error@+1 {{'omp.interchange' op permutation attribute must be of integer type}} + omp.interchange (%canonloop3, %canonloop4) <- (%canonloop1, %canonloop2) permutation([3.14 : f32, 2 : i32]) + + return +} + +// ----- +func.func @missing_attribute(%tc1 : i32, %tc2 : i32) { + %canonloop1 = omp.new_cli + %canonloop2 = omp.new_cli + %canonloop3 = omp.new_cli + %canonloop4 = omp.new_cli + omp.canonical_loop(%canonloop1) %iv1 : i32 in range(%tc1) { + omp.canonical_loop(%canonloop2) %iv2 : i32 in range(%tc2) { + omp.terminator + } + omp.terminator + } + + // expected-error@+1 {{'omp.interchange' op every integer from 1 must appear in the permutation attribute}} + omp.interchange (%canonloop3, %canonloop4) <- (%canonloop1, %canonloop2) permutation([1 : i32, 3 : i32]) + + return +} + +// ----- + +func.func @missing_generator(%tc1 : i32, %tc2 : i32) { + // expected-error@+1 {{'omp.new_cli' op CLI has no generator}} + %canonloop1 = omp.new_cli + + // expected-note@+1 {{see consumer here: "omp.interchange"(%0) <{operandSegmentSizes = array, permutation = [1 : i32, 2 : i32]}> : (!omp.cli) -> ()}} + omp.interchange <-(%canonloop1) permutation([1 : i32, 2 : i32]) + + return +} + +// ----- + +func.func @not_perfectly_nested(%tc1 : i32, %tc2 : i32) { + %canonloop1 = omp.new_cli + %canonloop2 = omp.new_cli + omp.canonical_loop(%canonloop1) %iv1 : i32 in range(%tc1) { + %v = arith.constant 42 : i32 + omp.canonical_loop(%canonloop2) %iv2 : i32 in range(%tc2) { + omp.terminator + } + omp.terminator + } + + // expected-error@+1 {{'omp.interchange' op OpenMP transformation loop nest must be perfectly nested}} + omp.interchange <-(%canonloop1, %canonloop2) permutation([1 : i32, 2 : i32]) + + llvm.return +} + +// ----- + +func.func @non_nectangular(%tc1 : i32, %tc2 : i32) { + %canonloop1 = omp.new_cli + %canonloop2 = omp.new_cli + omp.canonical_loop(%canonloop1) %iv1 : i32 in range(%tc1) { + omp.canonical_loop(%canonloop2) %iv2 : i32 in range(%iv1) { + omp.terminator + } + omp.terminator + } + + // expected-error@+1 {{'omp.interchange' op OpenMP transformation loop nest must be rectangular}} + omp.interchange <-(%canonloop1, %canonloop2) permutation([1 : i32, 2 : i32]) + + llvm.return +} + +func.func @not_nested(%tc1 : i32, %tc2 : i32) -> () { + %cli_outer = omp.new_cli + %cli_inner = omp.new_cli + omp.canonical_loop(%cli_outer) %iv_outer : i32 in range(%tc1) { + omp.terminator + } + omp.canonical_loop(%cli_inner) %iv_inner : i32 in range(%tc2) { + omp.terminator + } + + // expected-error@+1 {{'omp.interchange' op OpenMP transformation loop nest must be nested within each other}} + omp.interchange <- (%cli_outer, %cli_inner) permutation([2 : i32, 1 : i32]) + return +} diff --git a/mlir/test/Dialect/OpenMP/invalid-tile.mlir b/mlir/test/Dialect/OpenMP/invalid-tile.mlir index 868796730ef2b..2ee57ed906c0e 100644 --- a/mlir/test/Dialect/OpenMP/invalid-tile.mlir +++ b/mlir/test/Dialect/OpenMP/invalid-tile.mlir @@ -93,7 +93,7 @@ func.func @not_nested(%tc : i32, %ts : i32) { omp.terminator } - // expected-error@+1 {{'omp.tile' op tiled loop nest must be nested within each other}} + // expected-error@+1 {{'omp.tile' op OpenMP transformation loop nest must be nested within each other}} omp.tile <-(%canonloop1, %canonloop2) sizes(%ts, %ts : i32, i32) llvm.return @@ -112,7 +112,7 @@ func.func @not_perfectly_nested(%tc : i32, %ts : i32) { omp.terminator } - // expected-error@+1 {{'omp.tile' op tiled loop nest must be perfectly nested}} + // expected-error@+1 {{'omp.tile' op OpenMP transformation loop nest must be perfectly nested}} omp.tile <-(%canonloop1, %canonloop2) sizes(%ts, %ts : i32, i32) llvm.return @@ -130,7 +130,7 @@ func.func @non_nectangular(%tc : i32, %ts : i32) { omp.terminator } - // expected-error@+1 {{'omp.tile' op tiled loop nest must be rectangular}} + // expected-error@+1 {{'omp.tile' op OpenMP transformation loop nest must be rectangular}} omp.tile <-(%canonloop1, %canonloop2) sizes(%ts, %ts : i32, i32) llvm.return diff --git a/mlir/test/Target/LLVMIR/openmp-cli-interchange01.mlir b/mlir/test/Target/LLVMIR/openmp-cli-interchange01.mlir new file mode 100644 index 0000000000000..260c27192529a --- /dev/null +++ b/mlir/test/Target/LLVMIR/openmp-cli-interchange01.mlir @@ -0,0 +1,127 @@ +// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s --enable-var-scope + + +llvm.func @interchange_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32) -> () { + %cli_outer = omp.new_cli + %cli_inner = omp.new_cli + omp.canonical_loop(%cli_outer) %iv1 : i32 in range(%tc1) { + omp.canonical_loop(%cli_inner) %iv2 : i32 in range(%tc2) { + %ptr = llvm.getelementptr inbounds %baseptr[%iv1] : (!llvm.ptr, i32) -> !llvm.ptr, f32 + %val = llvm.mlir.constant(42.0 : f32) : f32 + llvm.store %val, %ptr : f32, !llvm.ptr + omp.terminator + } + omp.terminator + } + omp.interchange <- (%cli_outer, %cli_inner) {permutation = [2 : i32, 1 : i32]} + llvm.return +} +// CHECK-LABEL: define void @interchange_loop( +// CHECK-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) { +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_PREHEADER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_PREHEADER]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_PREHEADER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_HEADER:.*]]: +// CHECK-NEXT: [[OMP_OMP_LOOP_IV:%.*]] = phi i32 [ [[OMP_OMP_LOOP_NEXT:%.*]], %[[OMP_OMP_LOOP_INC:.*]] ] +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_COND:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_COND]]: +// CHECK-NEXT: [[OMP_OMP_LOOP_CMP:%.*]] = icmp ult i32 [[OMP_INTERCHANGE1_IV:%.*]], [[TMP1]] +// CHECK-NEXT: br i1 [[OMP_OMP_LOOP_CMP]], label %[[OMP_OMP_LOOP_BODY:.*]], label %[[OMP_OMP_LOOP_EXIT:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_BODY]]: +// CHECK-NEXT: br label %[[OMP_LOOP_REGION:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_LOOP_REGION]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_PREHEADER1:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_PREHEADER1]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_HEADER2:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_HEADER2]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_COND3:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_COND3]]: +// CHECK-NEXT: [[OMP_OMP_LOOP_CMP9:%.*]] = icmp ult i32 [[OMP_INTERCHANGE0_IV:%.*]], [[TMP2]] +// CHECK-NEXT: br i1 [[OMP_OMP_LOOP_CMP9]], label %[[OMP_OMP_LOOP_BODY4:.*]], label %[[OMP_OMP_LOOP_EXIT6:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_PREHEADER]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_HEADER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_HEADER]]: +// CHECK-NEXT: [[OMP_INTERCHANGE0_IV]] = phi i32 [ 0, %[[OMP_INTERCHANGE0_PREHEADER]] ], [ [[OMP_INTERCHANGE0_NEXT:%.*]], %[[OMP_INTERCHANGE0_INC:.*]] ] +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_COND:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_COND]]: +// CHECK-NEXT: [[OMP_INTERCHANGE0_CMP:%.*]] = icmp ult i32 [[OMP_INTERCHANGE0_IV]], [[TMP2]] +// CHECK-NEXT: br i1 [[OMP_INTERCHANGE0_CMP]], label %[[OMP_INTERCHANGE0_BODY:.*]], label %[[OMP_INTERCHANGE0_EXIT:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_BODY]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_PREHEADER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_PREHEADER]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_HEADER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_HEADER]]: +// CHECK-NEXT: [[OMP_INTERCHANGE1_IV]] = phi i32 [ 0, %[[OMP_INTERCHANGE1_PREHEADER]] ], [ [[OMP_INTERCHANGE1_NEXT:%.*]], %[[OMP_INTERCHANGE1_INC:.*]] ] +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_COND:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_COND]]: +// CHECK-NEXT: [[OMP_INTERCHANGE1_CMP:%.*]] = icmp ult i32 [[OMP_INTERCHANGE1_IV]], [[TMP1]] +// CHECK-NEXT: br i1 [[OMP_INTERCHANGE1_CMP]], label %[[OMP_INTERCHANGE1_BODY:.*]], label %[[OMP_INTERCHANGE1_EXIT:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_BODY]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_BODY4]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_BODY4]]: +// CHECK-NEXT: br label %[[OMP_LOOP_REGION12:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_LOOP_REGION12]]: +// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i32 [[OMP_INTERCHANGE1_IV]] +// CHECK-NEXT: store float 4.200000e+01, ptr [[TMP4]], align 4 +// CHECK-NEXT: br label %[[OMP_REGION_CONT11:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_REGION_CONT11]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_INC]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_INC]]: +// CHECK-NEXT: [[OMP_INTERCHANGE1_NEXT]] = add nuw i32 [[OMP_INTERCHANGE1_IV]], 1 +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_HEADER]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_EXIT]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_AFTER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_AFTER]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_INC]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_INC]]: +// CHECK-NEXT: [[OMP_INTERCHANGE0_NEXT]] = add nuw i32 [[OMP_INTERCHANGE0_IV]], 1 +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_HEADER]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_EXIT]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_AFTER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_AFTER]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_AFTER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_EXIT6]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_AFTER7:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_AFTER7]]: +// CHECK-NEXT: br label %[[OMP_REGION_CONT:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_REGION_CONT]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_INC]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_INC]]: +// CHECK-NEXT: [[OMP_OMP_LOOP_NEXT]] = add nuw i32 [[OMP_INTERCHANGE1_IV]], 1 +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_HEADER]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_EXIT]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_AFTER]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_AFTER]]: +// CHECK-NEXT: ret void + diff --git a/mlir/test/Target/LLVMIR/openmp-cli-interchange02.mlir b/mlir/test/Target/LLVMIR/openmp-cli-interchange02.mlir new file mode 100644 index 0000000000000..a23c591ea5299 --- /dev/null +++ b/mlir/test/Target/LLVMIR/openmp-cli-interchange02.mlir @@ -0,0 +1,178 @@ +// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s --enable-var-scope + + +llvm.func @interchange_loop_composition(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32) -> () { + %cli_outer = omp.new_cli + %cli_inner = omp.new_cli + %interchanged1 = omp.new_cli + %interchanged2 = omp.new_cli + %interchanged3 = omp.new_cli + %interchanged4 = omp.new_cli + + omp.canonical_loop(%cli_outer) %iv1 : i32 in range(%tc1) { + omp.canonical_loop(%cli_inner) %iv2 : i32 in range(%tc2) { + %ptr = llvm.getelementptr inbounds %baseptr[%iv1] : (!llvm.ptr, i32) -> !llvm.ptr, f32 + %val = llvm.mlir.constant(42.0 : f32) : f32 + llvm.store %val, %ptr : f32, !llvm.ptr + omp.terminator + } + omp.terminator + } + omp.interchange (%interchanged1, %interchanged2) <- (%cli_outer, %cli_inner) {permutation = [2 : i32, 1 : i32]} + omp.interchange (%interchanged3, %interchanged4) <- (%interchanged1, %interchanged2) {permutation = [2 : i32, 1 : i32]} + llvm.return +} + + +// CHECK-LABEL: define void @interchange_loop_composition( +// CHECK-SAME: ptr [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) { +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_PREHEADER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_PREHEADER]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_PREHEADER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_HEADER:.*]]: +// CHECK-NEXT: [[OMP_OMP_LOOP_IV:%.*]] = phi i32 [ [[OMP_OMP_LOOP_NEXT:%.*]], %[[OMP_OMP_LOOP_INC:.*]] ] +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_COND:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_COND]]: +// CHECK-NEXT: [[OMP_OMP_LOOP_CMP:%.*]] = icmp ult i32 [[OMP_INTERCHANGE0_IV20:%.*]], [[TMP1]] +// CHECK-NEXT: br i1 [[OMP_OMP_LOOP_CMP]], label %[[OMP_OMP_LOOP_BODY:.*]], label %[[OMP_OMP_LOOP_EXIT:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_BODY]]: +// CHECK-NEXT: br label %[[OMP_LOOP_REGION:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_LOOP_REGION]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_PREHEADER1:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_PREHEADER1]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_HEADER2:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_HEADER2]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_COND3:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_COND3]]: +// CHECK-NEXT: [[OMP_OMP_LOOP_CMP9:%.*]] = icmp ult i32 [[OMP_INTERCHANGE1_IV30:%.*]], [[TMP2]] +// CHECK-NEXT: br i1 [[OMP_OMP_LOOP_CMP9]], label %[[OMP_OMP_LOOP_BODY4:.*]], label %[[OMP_OMP_LOOP_EXIT6:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_PREHEADER]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_PREHEADER13:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_HEADER:.*]]: +// CHECK-NEXT: [[OMP_INTERCHANGE0_IV:%.*]] = phi i32 [ [[OMP_INTERCHANGE0_NEXT:%.*]], %[[OMP_INTERCHANGE0_INC:.*]] ] +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_COND:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_COND]]: +// CHECK-NEXT: [[OMP_INTERCHANGE0_CMP:%.*]] = icmp ult i32 [[OMP_INTERCHANGE1_IV30]], [[TMP2]] +// CHECK-NEXT: br i1 [[OMP_INTERCHANGE0_CMP]], label %[[OMP_INTERCHANGE0_BODY:.*]], label %[[OMP_INTERCHANGE0_EXIT:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_BODY]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_PREHEADER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_PREHEADER]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_HEADER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_HEADER]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_COND:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_COND]]: +// CHECK-NEXT: [[OMP_INTERCHANGE1_CMP:%.*]] = icmp ult i32 [[OMP_INTERCHANGE0_IV20]], [[TMP1]] +// CHECK-NEXT: br i1 [[OMP_INTERCHANGE1_CMP]], label %[[OMP_INTERCHANGE1_BODY:.*]], label %[[OMP_INTERCHANGE1_EXIT:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_PREHEADER13]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_HEADER14:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_HEADER14]]: +// CHECK-NEXT: [[OMP_INTERCHANGE0_IV20]] = phi i32 [ 0, %[[OMP_INTERCHANGE0_PREHEADER13]] ], [ [[OMP_INTERCHANGE0_NEXT22:%.*]], %[[OMP_INTERCHANGE0_INC17:.*]] ] +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_COND15:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_COND15]]: +// CHECK-NEXT: [[OMP_INTERCHANGE0_CMP21:%.*]] = icmp ult i32 [[OMP_INTERCHANGE0_IV20]], [[TMP1]] +// CHECK-NEXT: br i1 [[OMP_INTERCHANGE0_CMP21]], label %[[OMP_INTERCHANGE0_BODY16:.*]], label %[[OMP_INTERCHANGE0_EXIT18:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_BODY16]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_PREHEADER23:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_PREHEADER23]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_HEADER24:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_HEADER24]]: +// CHECK-NEXT: [[OMP_INTERCHANGE1_IV30]] = phi i32 [ 0, %[[OMP_INTERCHANGE1_PREHEADER23]] ], [ [[OMP_INTERCHANGE1_NEXT32:%.*]], %[[OMP_INTERCHANGE1_INC27:.*]] ] +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_COND25:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_COND25]]: +// CHECK-NEXT: [[OMP_INTERCHANGE1_CMP31:%.*]] = icmp ult i32 [[OMP_INTERCHANGE1_IV30]], [[TMP2]] +// CHECK-NEXT: br i1 [[OMP_INTERCHANGE1_CMP31]], label %[[OMP_INTERCHANGE1_BODY26:.*]], label %[[OMP_INTERCHANGE1_EXIT28:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_BODY26]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_BODY]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_BODY]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_BODY4]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_BODY4]]: +// CHECK-NEXT: br label %[[OMP_LOOP_REGION12:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_LOOP_REGION12]]: +// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i32 [[OMP_INTERCHANGE0_IV20]] +// CHECK-NEXT: store float 4.200000e+01, ptr [[TMP4]], align 4 +// CHECK-NEXT: br label %[[OMP_REGION_CONT11:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_REGION_CONT11]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_INC27]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_INC27]]: +// CHECK-NEXT: [[OMP_INTERCHANGE1_NEXT32]] = add nuw i32 [[OMP_INTERCHANGE1_IV30]], 1 +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_HEADER24]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_EXIT28]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_AFTER29:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_AFTER29]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_INC17]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_INC17]]: +// CHECK-NEXT: [[OMP_INTERCHANGE0_NEXT22]] = add nuw i32 [[OMP_INTERCHANGE0_IV20]], 1 +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_HEADER14]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_EXIT18]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_AFTER19:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_AFTER19]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_AFTER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_EXIT]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE1_AFTER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE1_AFTER]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_INC]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_INC]]: +// CHECK-NEXT: [[OMP_INTERCHANGE0_NEXT]] = add nuw i32 [[OMP_INTERCHANGE1_IV30]], 1 +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_HEADER]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_EXIT]]: +// CHECK-NEXT: br label %[[OMP_INTERCHANGE0_AFTER]] +// CHECK-EMPTY: +// CHECK: [[OMP_INTERCHANGE0_AFTER]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_AFTER:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_EXIT6]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_AFTER7:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_AFTER7]]: +// CHECK-NEXT: br label %[[OMP_REGION_CONT:.*]] +// CHECK-EMPTY: +// CHECK: [[OMP_REGION_CONT]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_INC]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_INC]]: +// CHECK-NEXT: [[OMP_OMP_LOOP_NEXT]] = add nuw i32 [[OMP_INTERCHANGE0_IV20]], 1 +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_HEADER]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_EXIT]]: +// CHECK-NEXT: br label %[[OMP_OMP_LOOP_AFTER]] +// CHECK-EMPTY: +// CHECK: [[OMP_OMP_LOOP_AFTER]]: +// CHECK-NEXT: ret void +