diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index fb7a5116bad74..ac9fdd28b6e11 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -150,6 +150,7 @@ class LinearClauseProcessor { llvm::BasicBlock *linearFinalizationBB; llvm::BasicBlock *linearExitBB; llvm::BasicBlock *linearLastIterExitBB; + Value linearLoopIV; public: // Register type for the linear variables @@ -189,6 +190,26 @@ class LinearClauseProcessor { } } + // Find linear iteration variable and save it for later updates + void initLinearIV(omp::SimdOp simdOp) { + auto loopOp = cast(simdOp.getWrappedLoop()); + // NOTE iteration variables can only be linear in non-nested loops. + if (loopOp.getIVs().size() != 1) + return; + // The linear IV is the loop IV's store address. + BlockArgument arg = loopOp.getIVs().front(); + for (const Operation *user : arg.getUsers()) { + if (auto storeOp = dyn_cast(user)) { + for (Value linearVar : simdOp.getLinearVars()) { + if (linearVar == storeOp.getAddr()) { + linearLoopIV = linearVar; + break; + } + } + } + } + } + // Emit IR for updating Linear variables void updateLinearVar(llvm::IRBuilderBase &builder, llvm::BasicBlock *loopBody, llvm::Value *loopInductionVar) { @@ -229,6 +250,34 @@ class LinearClauseProcessor { } } + // Emit IR for updating linear iteration variables on loop exit + void updateLinearIV(llvm::IRBuilderBase &builder, + LLVM::ModuleTranslation &moduleTranslation) { + if (!linearLoopIV) + return; + llvm::Value *linearIV = moduleTranslation.lookupValue(linearLoopIV); + + // Find linearIV's index + size_t index; + for (index = 0; index < linearOrigVal.size(); index++) + if (linearIV == linearOrigVal[index]) + break; + if (index == linearOrigVal.size()) + return; + + // Add one more step to the linear iteration variable + llvm::Type *varType = linearVarTypes[index]; + llvm::Value *var = linearLoopBodyTemps[index]; + llvm::Value *step = linearSteps[index]; + if (!varType->isIntegerTy()) + llvm_unreachable("Linear iteration variable must be of integer type"); + + step = builder.CreateSExtOrTrunc(step, varType); + llvm::Value *val = builder.CreateLoad(varType, var); + llvm::Value *addInst = builder.CreateAdd(val, step); + builder.CreateStore(addInst, var); + } + // Linear variable finalization is conditional on the last logical iteration. // Create BB splits to manage the same. void splitLinearFiniBB(llvm::IRBuilderBase &builder, @@ -4138,6 +4187,7 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder, // Initialize linear variables and linear step LinearClauseProcessor linearClauseProcessor; + linearClauseProcessor.initLinearIV(simdOp); if (!simdOp.getLinearVars().empty()) { auto linearVarTypes = simdOp.getLinearVarTypes().value(); @@ -4247,6 +4297,7 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder, : nullptr, order, simdlen, safelen); + linearClauseProcessor.updateLinearIV(builder, moduleTranslation); linearClauseProcessor.emitStoresForLinearVar(builder); // Check if this SIMD loop contains ordered regions diff --git a/mlir/test/Target/LLVMIR/openmp-llvm.mlir b/mlir/test/Target/LLVMIR/openmp-llvm.mlir index f002a64a593a0..1ecf02b8c2652 100644 --- a/mlir/test/Target/LLVMIR/openmp-llvm.mlir +++ b/mlir/test/Target/LLVMIR/openmp-llvm.mlir @@ -743,6 +743,7 @@ llvm.func @simd_simple(%lb : i64, %ub : i64, %step : i64, %arg0: !llvm.ptr) { llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) { // CHECK-LABEL: @simd_linear +// CHECK-SAME: (i32 %{{.*}}, i32 %{{.*}}, i32 %[[STEP:.*]], ptr %[[X:.*]]) // CHECK: %[[LINEAR_VAR:.*]] = alloca i32, align 4 // CHECK: %[[LINEAR_RESULT:.*]] = alloca i32, align 4 @@ -758,8 +759,16 @@ llvm.func @simd_linear(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) { // CHECK: %[[MUL:.*]] = mul i32 %omp_loop.iv, {{.*}} // CHECK: %[[ADD:.*]] = add i32 %[[LOAD]], %[[MUL]] // CHECK: store i32 %[[ADD]], ptr %[[LINEAR_RESULT]], align 4, !llvm.access.group !1 + +// CHECK: omp.region.cont: +// CHECK: %[[VAL:.*]] = load i32, ptr %[[LINEAR_RESULT]] +// CHECK-NEXT: %[[ADD:.*]] = add i32 %[[VAL]], %[[STEP]] +// CHECK-NEXT: store i32 %[[ADD]], ptr %[[LINEAR_RESULT]] +// CHECK-NEXT: %[[LOAD:.*]] = load i32, ptr %[[LINEAR_RESULT]] +// CHECK-NEXT: store i32 %[[LOAD]], ptr %[[X]], align 4 omp.simd linear(%x : !llvm.ptr = %step : i32) { omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) { + llvm.store %iv, %x : i32, !llvm.ptr omp.yield } } {linear_var_types = [i32]} @@ -832,6 +841,36 @@ llvm.func @simd_linear_f64_var_i32_step(%lb : i32, %ub : i32, %x : !llvm.ptr) { // ----- +// Test the update of omp.simd linear iteration variables, when nested inside +// omp.wsloop. +llvm.func @wsloop_simd_linear(%x : !llvm.ptr) { + +// CHECK-LABEL: @wsloop_simd_linear + +// CHECK: omp.wsloop.region: +// CHECK: %[[LINEAR_VAR:.*]] = alloca i32 +// CHECK: %[[LINEAR_RESULT:.*]] = alloca i32 + +// CHECK: omp.region.cont2: +// CHECK: %[[VAL:.*]] = load i32, ptr %[[LINEAR_RESULT]] +// CHECK-NEXT: %[[ADD:.*]] = add i32 %[[VAL]], 25 +// CHECK-NEXT: store i32 %[[ADD]], ptr %[[LINEAR_RESULT]] + %lb = llvm.mlir.constant(1 : i32) : i32 + %ub = llvm.mlir.constant(100 : i32) : i32 + %step = llvm.mlir.constant(25 : i32) : i32 + omp.wsloop { + omp.simd linear(%x : !llvm.ptr = %step : i32) { + omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) { + llvm.store %iv, %x : i32, !llvm.ptr + omp.yield + } + } {linear_var_types = [i32], omp.composite} + } {omp.composite} + llvm.return +} + +// ----- + // CHECK-LABEL: @simd_simple_multiple llvm.func @simd_simple_multiple(%lb1 : i64, %ub1 : i64, %step1 : i64, %lb2 : i64, %ub2 : i64, %step2 : i64, %arg0: !llvm.ptr, %arg1: !llvm.ptr) { omp.simd {