diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp index d619d47fc2359..a2823fb3e27d4 100644 --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -1607,6 +1607,48 @@ mlir::Type getTypeFromIvTypeSize(fir::FirOpBuilder &builder, return builder.getIntegerType(ivTypeSize * 8); } +static void privatizeIv(Fortran::lower::AbstractConverter &converter, + const Fortran::semantics::Symbol &sym, + mlir::Location loc, + llvm::SmallVector &ivTypes, + llvm::SmallVector &ivLocs, + llvm::SmallVector &privateOperands, + llvm::SmallVector &ivPrivate, + llvm::SmallVector &privatizations, + bool isDoConcurrent = false) { + fir::FirOpBuilder &builder = converter.getFirOpBuilder(); + + mlir::Type ivTy = getTypeFromIvTypeSize(builder, sym); + ivTypes.push_back(ivTy); + ivLocs.push_back(loc); + mlir::Value ivValue = converter.getSymbolAddress(sym); + if (!ivValue && isDoConcurrent) { + // DO CONCURRENT induction variables are not mapped yet since they are local + // to the DO CONCURRENT scope. + mlir::OpBuilder::InsertPoint insPt = builder.saveInsertionPoint(); + builder.setInsertionPointToStart(builder.getAllocaBlock()); + ivValue = builder.createTemporaryAlloc(loc, ivTy, toStringRef(sym.name())); + builder.restoreInsertionPoint(insPt); + } + + std::string recipeName = + fir::getTypeAsString(ivValue.getType(), converter.getKindMap(), + Fortran::lower::privatizationRecipePrefix); + auto recipe = Fortran::lower::createOrGetPrivateRecipe( + builder, recipeName, loc, ivValue.getType()); + + std::stringstream asFortran; + auto op = createDataEntryOp( + builder, loc, ivValue, asFortran, {}, true, /*implicit=*/true, + mlir::acc::DataClause::acc_private, ivValue.getType()); + + privateOperands.push_back(op.getAccPtr()); + ivPrivate.push_back(op.getAccPtr()); + privatizations.push_back(mlir::SymbolRefAttr::get(builder.getContext(), + recipe.getSymName().str())); + converter.bindSymbol(sym, op.getAccPtr()); +} + static mlir::acc::LoopOp createLoopOp(Fortran::lower::AbstractConverter &converter, mlir::Location currentLocation, @@ -1640,68 +1682,86 @@ createLoopOp(Fortran::lower::AbstractConverter &converter, llvm::SmallVector ivLocs; llvm::SmallVector inclusiveBounds; - if (outerDoConstruct.IsDoConcurrent()) - TODO(currentLocation, "OpenACC loop with DO CONCURRENT"); - llvm::SmallVector locs; locs.push_back(currentLocation); // Location of the directive - - int64_t collapseValue = Fortran::lower::getCollapseValue(accClauseList); Fortran::lower::pft::Evaluation *crtEval = &eval.getFirstNestedEvaluation(); - for (unsigned i = 0; i < collapseValue; ++i) { - const Fortran::parser::LoopControl *loopControl; - if (i == 0) { - loopControl = &*outerDoConstruct.GetLoopControl(); - locs.push_back(converter.genLocation( - Fortran::parser::FindSourceLocation(outerDoConstruct))); - } else { - auto *doCons = crtEval->getIf(); - assert(doCons && "expect do construct"); - loopControl = &*doCons->GetLoopControl(); - locs.push_back( - converter.genLocation(Fortran::parser::FindSourceLocation(*doCons))); + bool isDoConcurrent = outerDoConstruct.IsDoConcurrent(); + if (isDoConcurrent) { + locs.push_back(converter.genLocation( + Fortran::parser::FindSourceLocation(outerDoConstruct))); + const Fortran::parser::LoopControl *loopControl = + &*outerDoConstruct.GetLoopControl(); + const auto &concurrent = + std::get(loopControl->u); + if (!std::get>(concurrent.t) + .empty()) + TODO(currentLocation, "DO CONCURRENT with locality spec"); + + const auto &concurrentHeader = + std::get(concurrent.t); + const auto &controls = + std::get>( + concurrentHeader.t); + for (const auto &control : controls) { + lowerbounds.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(std::get<1>(control.t)), stmtCtx))); + upperbounds.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(std::get<2>(control.t)), stmtCtx))); + if (const auto &expr = + std::get>( + control.t)) + steps.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(*expr), stmtCtx))); + else // If `step` is not present, assume it is `1`. + steps.push_back(builder.createIntegerConstant( + currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1)); + + const auto &name = std::get(control.t); + privatizeIv(converter, *name.symbol, currentLocation, ivTypes, ivLocs, + privateOperands, ivPrivate, privatizations, isDoConcurrent); + + inclusiveBounds.push_back(true); } + } else { + int64_t collapseValue = Fortran::lower::getCollapseValue(accClauseList); + for (unsigned i = 0; i < collapseValue; ++i) { + const Fortran::parser::LoopControl *loopControl; + if (i == 0) { + loopControl = &*outerDoConstruct.GetLoopControl(); + locs.push_back(converter.genLocation( + Fortran::parser::FindSourceLocation(outerDoConstruct))); + } else { + auto *doCons = crtEval->getIf(); + assert(doCons && "expect do construct"); + loopControl = &*doCons->GetLoopControl(); + locs.push_back(converter.genLocation( + Fortran::parser::FindSourceLocation(*doCons))); + } - const Fortran::parser::LoopControl::Bounds *bounds = - std::get_if(&loopControl->u); - assert(bounds && "Expected bounds on the loop construct"); - lowerbounds.push_back(fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(bounds->lower), stmtCtx))); - upperbounds.push_back(fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(bounds->upper), stmtCtx))); - if (bounds->step) - steps.push_back(fir::getBase(converter.genExprValue( - *Fortran::semantics::GetExpr(bounds->step), stmtCtx))); - else // If `step` is not present, assume it as `1`. - steps.push_back(builder.createIntegerConstant( - currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1)); - - Fortran::semantics::Symbol &ivSym = - bounds->name.thing.symbol->GetUltimate(); - - mlir::Type ivTy = getTypeFromIvTypeSize(builder, ivSym); - mlir::Value ivValue = converter.getSymbolAddress(ivSym); - ivTypes.push_back(ivTy); - ivLocs.push_back(currentLocation); - std::string recipeName = - fir::getTypeAsString(ivValue.getType(), converter.getKindMap(), - Fortran::lower::privatizationRecipePrefix); - auto recipe = Fortran::lower::createOrGetPrivateRecipe( - builder, recipeName, currentLocation, ivValue.getType()); - std::stringstream asFortran; - auto op = createDataEntryOp( - builder, currentLocation, ivValue, asFortran, {}, true, - /*implicit=*/true, mlir::acc::DataClause::acc_private, - ivValue.getType()); - - privateOperands.push_back(op.getAccPtr()); - ivPrivate.push_back(op.getAccPtr()); - privatizations.push_back(mlir::SymbolRefAttr::get( - builder.getContext(), recipe.getSymName().str())); - inclusiveBounds.push_back(true); - converter.bindSymbol(ivSym, op.getAccPtr()); - if (i < collapseValue - 1) - crtEval = &*std::next(crtEval->getNestedEvaluations().begin()); + const Fortran::parser::LoopControl::Bounds *bounds = + std::get_if(&loopControl->u); + assert(bounds && "Expected bounds on the loop construct"); + lowerbounds.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(bounds->lower), stmtCtx))); + upperbounds.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(bounds->upper), stmtCtx))); + if (bounds->step) + steps.push_back(fir::getBase(converter.genExprValue( + *Fortran::semantics::GetExpr(bounds->step), stmtCtx))); + else // If `step` is not present, assume it is `1`. + steps.push_back(builder.createIntegerConstant( + currentLocation, upperbounds[upperbounds.size() - 1].getType(), 1)); + + Fortran::semantics::Symbol &ivSym = + bounds->name.thing.symbol->GetUltimate(); + privatizeIv(converter, ivSym, currentLocation, ivTypes, ivLocs, + privateOperands, ivPrivate, privatizations); + + inclusiveBounds.push_back(true); + + if (i < collapseValue - 1) + crtEval = &*std::next(crtEval->getNestedEvaluations().begin()); + } } for (const Fortran::parser::AccClause &clause : accClauseList.v) { diff --git a/flang/test/Lower/OpenACC/acc-loop.f90 b/flang/test/Lower/OpenACC/acc-loop.f90 index 6bd4dd61bd893..28da9f99282ce 100644 --- a/flang/test/Lower/OpenACC/acc-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-loop.f90 @@ -306,3 +306,23 @@ program acc_loop ! CHECK: acc.loop gang([#acc.device_type, #acc.device_type]) end program + +subroutine sub1(i, j, k) + integer :: i,j,k + integer :: a(i,j,k) + !$acc parallel loop + do concurrent (i=1:10,j=1:100,k=1:200) + a(i,j,k) = a(i,j,k) + 1 + end do +end subroutine + +! CHECK: func.func @_QPsub1 +! CHECK: %[[DC_K:.*]] = fir.alloca i32 {bindc_name = "k"} +! CHECK: %[[DC_J:.*]] = fir.alloca i32 {bindc_name = "j"} +! CHECK: %[[DC_I:.*]] = fir.alloca i32 {bindc_name = "i"} +! CHECK: acc.parallel +! CHECK: %[[P_I:.*]] = acc.private varPtr(%[[DC_I]] : !fir.ref) -> !fir.ref {implicit = true, name = ""} +! CHECK: %[[P_J:.*]] = acc.private varPtr(%[[DC_J]] : !fir.ref) -> !fir.ref {implicit = true, name = ""} +! CHECK: %[[P_K:.*]] = acc.private varPtr(%[[DC_K]] : !fir.ref) -> !fir.ref {implicit = true, name = ""} +! CHECK: acc.loop private(@privatization_ref_i32 -> %[[P_I]] : !fir.ref, @privatization_ref_i32 -> %[[P_J]] : !fir.ref, @privatization_ref_i32 -> %[[P_K]] : !fir.ref) (%{{.*}} : i32, %{{.*}} : i32, %{{.*}} : i32) = (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32) to (%c10{{.*}}, %c100{{.*}}, %c200{{.*}} : i32, i32, i32) step (%c1{{.*}}, %c1{{.*}}, %c1{{.*}} : i32, i32, i32) +! CHECK: } attributes {inclusiveUpperbound = array}