diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp index b7e1a23476a45..b77ab6248c4d5 100644 --- a/flang/lib/Lower/OpenACC.cpp +++ b/flang/lib/Lower/OpenACC.cpp @@ -416,6 +416,24 @@ static void genDataExitOperations(fir::FirOpBuilder &builder, } } +template +static void genPrivateLikeInitRegion(mlir::OpBuilder &builder, RecipeOp recipe, + mlir::Type ty, mlir::Location loc) { + mlir::Value retVal = recipe.getInitRegion().front().getArgument(0); + if (auto refTy = mlir::dyn_cast_or_null(ty)) { + if (fir::isa_trivial(refTy.getEleTy())) + retVal = builder.create(loc, refTy.getEleTy()); + else if (auto seqTy = + mlir::dyn_cast_or_null(refTy.getEleTy())) { + if (seqTy.hasDynamicExtents()) + TODO(loc, "private recipe of array with dynamic extents"); + if (fir::isa_trivial(seqTy.getEleTy())) + retVal = builder.create(loc, seqTy); + } + } + builder.create(loc, retVal); +} + mlir::acc::PrivateRecipeOp Fortran::lower::createOrGetPrivateRecipe(mlir::OpBuilder &builder, llvm::StringRef recipeName, @@ -432,20 +450,8 @@ Fortran::lower::createOrGetPrivateRecipe(mlir::OpBuilder &builder, builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(), {ty}, {loc}); builder.setInsertionPointToEnd(&recipe.getInitRegion().back()); - - mlir::Value retVal = recipe.getInitRegion().front().getArgument(0); - if (auto refTy = mlir::dyn_cast_or_null(ty)) { - if (fir::isa_trivial(refTy.getEleTy())) - retVal = builder.create(loc, refTy.getEleTy()); - else if (auto seqTy = - mlir::dyn_cast_or_null(refTy.getEleTy())) { - if (seqTy.hasDynamicExtents()) - TODO(loc, "private recipe of array with dynamic extents"); - if (fir::isa_trivial(seqTy.getEleTy())) - retVal = builder.create(loc, seqTy); - } - } - builder.create(loc, retVal); + genPrivateLikeInitRegion(builder, recipe, ty, + loc); builder.restoreInsertionPoint(crtPos); return recipe; } @@ -466,15 +472,51 @@ mlir::acc::FirstprivateRecipeOp Fortran::lower::createOrGetFirstprivateRecipe( builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(), {ty}, {loc}); builder.setInsertionPointToEnd(&recipe.getInitRegion().back()); - builder.create( - loc, recipe.getInitRegion().front().getArgument(0)); + genPrivateLikeInitRegion(builder, recipe, ty, + loc); // Add empty copy region for firstprivate. TODO add copy sequence. builder.createBlock(&recipe.getCopyRegion(), recipe.getCopyRegion().end(), {ty, ty}, {loc, loc}); + builder.setInsertionPointToEnd(&recipe.getCopyRegion().back()); + if (auto refTy = mlir::dyn_cast_or_null(ty)) { + if (fir::isa_trivial(refTy.getEleTy())) { + mlir::Value initValue = builder.create( + loc, recipe.getCopyRegion().front().getArgument(0)); + builder.create( + loc, initValue, recipe.getCopyRegion().front().getArgument(1)); + } else if (auto seqTy = mlir::dyn_cast_or_null( + refTy.getEleTy())) { + if (seqTy.hasDynamicExtents()) + TODO(loc, "private recipe of array with dynamic extents"); + mlir::Type idxTy = builder.getIndexType(); + mlir::Type refTy = fir::ReferenceType::get(seqTy.getEleTy()); + mlir::Value arraySrc = recipe.getCopyRegion().front().getArgument(0); + mlir::Value arrayDst = recipe.getCopyRegion().front().getArgument(1); + llvm::SmallVector loops; + llvm::SmallVector ivs; + for (auto ext : llvm::reverse(seqTy.getShape())) { + auto lb = builder.create( + loc, idxTy, builder.getIntegerAttr(idxTy, 0)); + auto ub = builder.create( + loc, idxTy, builder.getIntegerAttr(idxTy, ext - 1)); + auto step = builder.create( + loc, idxTy, builder.getIntegerAttr(idxTy, 1)); + auto loop = builder.create(loc, lb, ub, step, + /*unordered=*/false); + builder.setInsertionPointToStart(loop.getBody()); + loops.push_back(loop); + ivs.push_back(loop.getInductionVar()); + } + auto addr1 = builder.create(loc, refTy, arraySrc, ivs); + auto addr2 = builder.create(loc, refTy, arrayDst, ivs); + auto loadedValue = builder.create(loc, addr1); + builder.create(loc, loadedValue, addr2); + builder.setInsertionPointAfter(loops[0]); + } + } builder.create(loc); - builder.restoreInsertionPoint(crtPos); return recipe; } diff --git a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 b/flang/test/Lower/OpenACC/acc-parallel-loop.f90 index 2e89327de30d4..64c256f85abef 100644 --- a/flang/test/Lower/OpenACC/acc-parallel-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-parallel-loop.f90 @@ -3,10 +3,20 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s ! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10xf32 : !fir.ref> init { -! CHECK: ^bb0(%arg0: !fir.ref>): -! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: ^bb0(%{{.*}}: !fir.ref>): +! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<10xf32> +! CHECK: acc.yield %[[ALLOCA]] : !fir.ref> ! CHECK: } copy { -! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): +! CHECK: ^bb0(%[[SRC:.*]]: !fir.ref>, %[[DST:.*]]: !fir.ref>): +! CHECK: %[[LB0:.*]] = arith.constant 0 : index +! CHECK: %[[UB0:.*]] = arith.constant 9 : index +! CHECK: %[[STEP0:.*]] = arith.constant 1 : index +! CHECK: fir.do_loop %[[IV0:.*]] = %[[LB0]] to %[[UB0]] step %[[STEP0]] { +! CHECK: %[[COORD0:.*]] = fir.coordinate_of %[[SRC]], %[[IV0]] : (!fir.ref>, index) -> !fir.ref +! CHECK: %[[COORD1:.*]] = fir.coordinate_of %[[DST]], %[[IV0]] : (!fir.ref>, index) -> !fir.ref +! CHECK: %[[LOAD:.*]] = fir.load %[[COORD0]] : !fir.ref +! CHECK: fir.store %[[LOAD]] to %[[COORD1]] : !fir.ref +! CHECK: } ! CHECK: acc.terminator ! CHECK: } diff --git a/flang/test/Lower/OpenACC/acc-parallel.f90 b/flang/test/Lower/OpenACC/acc-parallel.f90 index dbc6e6ccc46b0..c930a3342b4a6 100644 --- a/flang/test/Lower/OpenACC/acc-parallel.f90 +++ b/flang/test/Lower/OpenACC/acc-parallel.f90 @@ -3,8 +3,9 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s ! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10x10xf32 : !fir.ref> init { -! CHECK: ^bb0(%arg0: !fir.ref>): -! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: ^bb0(%{{.*}}: !fir.ref>): +! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<10x10xf32> +! CHECK: acc.yield %[[ALLOCA]] : !fir.ref> ! CHECK: } copy { ! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): ! CHECK: acc.terminator diff --git a/flang/test/Lower/OpenACC/acc-private.f90 b/flang/test/Lower/OpenACC/acc-private.f90 index ecee86585522f..f3162d14763ec 100644 --- a/flang/test/Lower/OpenACC/acc-private.f90 +++ b/flang/test/Lower/OpenACC/acc-private.f90 @@ -2,6 +2,35 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s +! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_100xf32 : !fir.ref> init { +! CHECK: ^bb0(%{{.*}}: !fir.ref>): +! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<100xf32> +! CHECK: acc.yield %[[ALLOCA]] : !fir.ref> +! CHECK: } copy { +! CHECK: ^bb0(%[[SRC:.*]]: !fir.ref>, %[[DST:.*]]: !fir.ref>): +! CHECK: %[[LB0:.*]] = arith.constant 0 : index +! CHECK: %[[UB0:.*]] = arith.constant 99 : index +! CHECK: %[[STEP1:.*]] = arith.constant 1 : index +! CHECK: fir.do_loop %[[IV0:.*]] = %c0 to %c99 step %c1 { +! CHECK: %[[COORD0:.*]] = fir.coordinate_of %[[SRC]], %[[IV0]] : (!fir.ref>, index) -> !fir.ref +! CHECK: %[[COORD1:.*]] = fir.coordinate_of %[[DST]], %[[IV0]] : (!fir.ref>, index) -> !fir.ref +! CHECK: %[[VALUE:.*]] = fir.load %[[COORD0]] : !fir.ref +! CHECK: fir.store %[[VALUE]] to %[[COORD1]] : !fir.ref +! CHECK: } +! CHECK: acc.terminator +! CHECK: } + +! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_i32 : !fir.ref init { +! CHECK: ^bb0(%{{.*}}: !fir.ref): +! CHECK: %[[ALLOCA:.*]] = fir.alloca i32 +! CHECK: acc.yield %[[ALLOCA]] : !fir.ref +! CHECK: } copy { +! CHECK: ^bb0(%[[SRC:.*]]: !fir.ref, %[[DST:.*]]: !fir.ref): +! CHECK: %[[VALUE:.*]] = fir.load %[[SRC]] : !fir.ref +! CHECK: fir.store %[[VALUE]] to %[[DST]] : !fir.ref +! CHECK: acc.terminator +! CHECK: } + ! CHECK-LABEL: acc.private.recipe @privatization_ref_50xf32 : !fir.ref> init { ! CHECK: ^bb0(%{{.*}}: !fir.ref>): ! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<50xf32> @@ -65,5 +94,27 @@ program acc_private ! CHECK: %[[B_PRIVATE:.*]] = acc.private varPtr(%[[B]] : !fir.ref>) bounds(%[[BOUND]]) -> !fir.ref> {name = "b(1:50)"} ! CHECK: acc.loop private(@privatization_ref_50xf32 -> %[[B_PRIVATE]] : !fir.ref>) + !$acc parallel loop firstprivate(c) + DO i = 1, n + c = i + a(i) = b(i) + c + END DO + +! CHECK: %[[FP_C:.*]] = acc.firstprivate varPtr(%[[C]] : !fir.ref) -> !fir.ref {name = "c"} +! CHECK: acc.parallel firstprivate(@firstprivatization_ref_i32 -> %[[FP_C]] : !fir.ref) +! CHECK: acc.yield + + !$acc parallel loop firstprivate(b) + DO i = 1, n + c = i + a(i) = b(i) + c + END DO + +! CHECK: %[[C1:.*]] = arith.constant 1 : index +! CHECK: %[[LB:.*]] = arith.constant 0 : index +! CHECK: %[[UB:.*]] = arith.subi %{{.*}}, %[[C1]] : index +! CHECK: %[[BOUND:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index) +! CHECK: %[[FP_B:.*]] = acc.firstprivate varPtr(%[[B]] : !fir.ref>) bounds(%[[BOUND]]) -> !fir.ref> {name = "b"} +! CHECK: acc.parallel firstprivate(@firstprivatization_ref_100xf32 -> %[[FP_B]] : !fir.ref>) end program diff --git a/flang/test/Lower/OpenACC/acc-serial-loop.f90 b/flang/test/Lower/OpenACC/acc-serial-loop.f90 index eb57ef28e4f06..53b05f8077ac6 100644 --- a/flang/test/Lower/OpenACC/acc-serial-loop.f90 +++ b/flang/test/Lower/OpenACC/acc-serial-loop.f90 @@ -3,8 +3,9 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s ! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10xf32 : !fir.ref> init { -! CHECK: ^bb0(%arg0: !fir.ref>): -! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: ^bb0(%{{.*}}: !fir.ref>): +! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<10xf32> +! CHECK: acc.yield %[[ALLOCA]] : !fir.ref> ! CHECK: } copy { ! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): ! CHECK: acc.terminator diff --git a/flang/test/Lower/OpenACC/acc-serial.f90 b/flang/test/Lower/OpenACC/acc-serial.f90 index cea0a54406724..283a75fcf7261 100644 --- a/flang/test/Lower/OpenACC/acc-serial.f90 +++ b/flang/test/Lower/OpenACC/acc-serial.f90 @@ -3,8 +3,9 @@ ! RUN: bbc -fopenacc -emit-fir %s -o - | FileCheck %s ! CHECK-LABEL: acc.firstprivate.recipe @firstprivatization_ref_10x10xf32 : !fir.ref> init { -! CHECK: ^bb0(%arg0: !fir.ref>): -! CHECK: acc.yield %arg0 : !fir.ref> +! CHECK: ^bb0(%{{.*}}: !fir.ref>): +! CHECK: %[[ALLOCA:.*]] = fir.alloca !fir.array<10x10xf32> +! CHECK: acc.yield %[[ALLOCA]] : !fir.ref> ! CHECK: } copy { ! CHECK: ^bb0(%arg0: !fir.ref>, %arg1: !fir.ref>): ! CHECK: acc.terminator