135 changes: 132 additions & 3 deletions clang/lib/CodeGen/CGStmtOpenMP.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1119,17 +1119,147 @@ static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
}

static Optional<llvm::AtomicRMWInst::BinOp>
getCompatibleAtomicRMWBinOp(ASTContext &Context, BinaryOperatorKind Op,
bool IsXLHSInRHSPart, LValue XLValue,
RValue ExprRValue) {
Optional<llvm::AtomicRMWInst::BinOp> RMWOp;
// Allow atomicrmw only if 'x' and 'expr' are integer values, lvalue for 'x'
// expression is simple and atomic is allowed for the given type for the
// target platform.
if (ExprRValue.isScalar() &&
ExprRValue.getScalarVal()->getType()->isIntegerTy() &&
XLValue.isSimple() &&
(isa<llvm::ConstantInt>(ExprRValue.getScalarVal()) ||
(ExprRValue.getScalarVal()->getType() ==
XLValue.getAddress()->getType()->getPointerElementType())) &&
Context.getTargetInfo().hasBuiltinAtomic(
Context.getTypeSize(XLValue.getType()),
Context.toBits(XLValue.getAlignment()))) {
switch (Op) {
case BO_Add:
RMWOp = llvm::AtomicRMWInst::Add;
break;
case BO_Sub:
if (IsXLHSInRHSPart) {
RMWOp = llvm::AtomicRMWInst::Sub;
}
break;
case BO_And:
RMWOp = llvm::AtomicRMWInst::And;
break;
case BO_Or:
RMWOp = llvm::AtomicRMWInst::Or;
break;
case BO_Xor:
RMWOp = llvm::AtomicRMWInst::Xor;
break;
case BO_Mul:
case BO_Div:
case BO_Rem:
case BO_Shl:
case BO_Shr:
break;
case BO_PtrMemD:
case BO_PtrMemI:
case BO_LT:
case BO_GT:
case BO_LE:
case BO_GE:
case BO_EQ:
case BO_NE:
case BO_LAnd:
case BO_LOr:
case BO_Assign:
case BO_MulAssign:
case BO_DivAssign:
case BO_RemAssign:
case BO_AddAssign:
case BO_SubAssign:
case BO_ShlAssign:
case BO_ShrAssign:
case BO_AndAssign:
case BO_XorAssign:
case BO_OrAssign:
case BO_Comma:
llvm_unreachable("Unexpected binary operation in 'atomic update'.");
}
}
return std::move(RMWOp);
}

static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
const Expr *X, const Expr *E,
const Expr *UE, bool IsXLHSInRHSPart,
SourceLocation Loc) {
assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
"Update expr in 'atomic update' must be a binary operator.");
auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
// Update expressions are allowed to have the following forms:
// x binop= expr; -> xrval + expr;
// x++, ++x -> xrval + 1;
// x--, --x -> xrval - 1;
// x = x binop expr; -> xrval binop expr
// x = expr Op x; - > expr binop xrval;
assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
LValue XLValue = CGF.EmitLValue(X);
RValue ExprRValue = CGF.EmitAnyExpr(E);
const auto &Op =
getCompatibleAtomicRMWBinOp(CGF.CGM.getContext(), BOUE->getOpcode(),
IsXLHSInRHSPart, XLValue, ExprRValue);
auto AO = IsSeqCst ? llvm::SequentiallyConsistent : llvm::Monotonic;
if (Op) {
auto *ExprVal = ExprRValue.getScalarVal();
if (auto *IC = dyn_cast<llvm::ConstantInt>(ExprVal)) {
ExprVal = CGF.Builder.CreateIntCast(
IC, XLValue.getAddress()->getType()->getPointerElementType(),
XLValue.getType()->hasSignedIntegerRepresentation());
}
CGF.Builder.CreateAtomicRMW(*Op, XLValue.getAddress(), ExprVal, AO);
} else {
auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
CodeGenFunction::OpaqueValueMapping MapExpr(
CGF, IsXLHSInRHSPart ? RHS : LHS, ExprRValue);
auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
if (XLValue.isGlobalReg()) {
// Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
// 'xrval'.
CodeGenFunction::OpaqueValueMapping MapX(
CGF, XRValExpr, CGF.EmitLoadOfLValue(XLValue, Loc));
CGF.EmitStoreThroughLValue(CGF.EmitAnyExpr(UE), XLValue);
} else {
// Perform compare-and-swap procedure.
CGF.EmitAtomicUpdate(
XLValue, AO, [&CGF, &UE, &XRValExpr](RValue XRVal) -> RValue {
CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRVal);
return CGF.EmitAnyExpr(UE);
}, /*IsVolatile=*/false);
}
}
// OpenMP, 2.12.6, atomic Construct
// Any atomic construct with a seq_cst clause forces the atomically
// performed operation to include an implicit flush operation without a
// list.
if (IsSeqCst)
CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
}

static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
bool IsSeqCst, const Expr *X, const Expr *V,
const Expr *E, SourceLocation Loc) {
const Expr *E, const Expr *UE,
bool IsXLHSInRHSPart, SourceLocation Loc) {
switch (Kind) {
case OMPC_read:
EmitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
break;
case OMPC_write:
EmitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
break;
case OMPC_unknown:
case OMPC_update:
EmitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
break;
case OMPC_capture:
llvm_unreachable("CodeGen for 'omp atomic clause' is not supported yet.");
case OMPC_if:
Expand All @@ -1156,7 +1286,6 @@ static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
case OMPC_untied:
case OMPC_threadprivate:
case OMPC_mergeable:
case OMPC_unknown:
llvm_unreachable("Clause is not allowed in 'omp atomic'.");
}
}
Expand All @@ -1179,7 +1308,7 @@ void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
InlinedOpenMPRegionScopeRAII Region(*this, S);

EmitOMPAtomicExpr(*this, Kind, IsSeqCst, S.getX(), S.getV(), S.getExpr(),
S.getLocStart());
S.getUpdateExpr(), S.isXLHSInRHSPart(), S.getLocStart());
}

void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &) {
Expand Down
6 changes: 5 additions & 1 deletion clang/lib/CodeGen/CodeGenFunction.h
Original file line number Diff line number Diff line change
Expand Up @@ -2123,12 +2123,16 @@ class CodeGenFunction : public CodeGenTypeCache {
void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,
bool IsVolatile, bool isInit);

std::pair<RValue, RValue> EmitAtomicCompareExchange(
std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(
LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
llvm::AtomicOrdering Success = llvm::SequentiallyConsistent,
llvm::AtomicOrdering Failure = llvm::SequentiallyConsistent,
bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());

void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,
const std::function<RValue(RValue)> &UpdateOp,
bool IsVolatile);

/// EmitToMemory - Change a scalar value from its value
/// representation to its in-memory representation.
llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);
Expand Down
94 changes: 60 additions & 34 deletions clang/lib/Sema/SemaOpenMP.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3282,21 +3282,24 @@ class OpenMPAtomicUpdateChecker {
Sema &SemaRef;
/// \brief A location for note diagnostics (when error is found).
SourceLocation NoteLoc;
/// \brief Atomic operation supposed to be performed on source expression.
BinaryOperatorKind OpKind;
/// \brief 'x' lvalue part of the source atomic expression.
Expr *X;
/// \brief 'x' rvalue part of the source atomic expression, used in the right
/// hand side of the expression. We need this to properly generate RHS part of
/// the source expression (x = x'rval' binop expr or x = expr binop x'rval').
Expr *XRVal;
/// \brief 'expr' rvalue part of the source atomic expression.
Expr *E;
/// \brief Helper expression of the form
/// 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' or
/// 'OpaqueValueExpr(expr) binop OpaqueValueExpr(x)'.
Expr *UpdateExpr;
/// \brief Is 'x' a LHS in a RHS part of full update expression. It is
/// important for non-associative operations.
bool IsXLHSInRHSPart;
BinaryOperatorKind Op;
SourceLocation OpLoc;

public:
OpenMPAtomicUpdateChecker(Sema &SemaRef)
: SemaRef(SemaRef), OpKind(BO_PtrMemD), X(nullptr), XRVal(nullptr),
E(nullptr) {}
: SemaRef(SemaRef), X(nullptr), E(nullptr), UpdateExpr(nullptr),
IsXLHSInRHSPart(false), Op(BO_PtrMemD) {}
/// \brief Check specified statement that it is suitable for 'atomic update'
/// constructs and extract 'x', 'expr' and Operation from the original
/// expression.
Expand All @@ -3306,13 +3309,16 @@ class OpenMPAtomicUpdateChecker {
bool checkStatement(Stmt *S, unsigned DiagId, unsigned NoteId);
/// \brief Return the 'x' lvalue part of the source atomic expression.
Expr *getX() const { return X; }
/// \brief Return the 'x' rvalue part of the source atomic expression, used in
/// the RHS part of the source expression.
Expr *getXRVal() const { return XRVal; }
/// \brief Return the 'expr' rvalue part of the source atomic expression.
Expr *getExpr() const { return E; }
/// \brief Return required atomic operation.
BinaryOperatorKind getOpKind() const {return OpKind;}
/// \brief Return the update expression used in calculation of the updated
/// value. Always has form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' or
/// 'OpaqueValueExpr(expr) binop OpaqueValueExpr(x)'.
Expr *getUpdateExpr() const { return UpdateExpr; }
/// \brief Return true if 'x' is LHS in RHS part of full update expression,
/// false otherwise.
bool isXLHSInRHSPart() const { return IsXLHSInRHSPart; }

private:
bool checkBinaryOperation(BinaryOperator *AtomicBinOp, unsigned DiagId,
unsigned NoteId);
Expand All @@ -3334,7 +3340,8 @@ bool OpenMPAtomicUpdateChecker::checkBinaryOperation(
if (AtomicInnerBinOp->isMultiplicativeOp() ||
AtomicInnerBinOp->isAdditiveOp() || AtomicInnerBinOp->isShiftOp() ||
AtomicInnerBinOp->isBitwiseOp()) {
OpKind = AtomicInnerBinOp->getOpcode();
Op = AtomicInnerBinOp->getOpcode();
OpLoc = AtomicInnerBinOp->getOperatorLoc();
auto *LHS = AtomicInnerBinOp->getLHS();
auto *RHS = AtomicInnerBinOp->getRHS();
llvm::FoldingSetNodeID XId, LHSId, RHSId;
Expand All @@ -3346,10 +3353,10 @@ bool OpenMPAtomicUpdateChecker::checkBinaryOperation(
/*Canonical=*/true);
if (XId == LHSId) {
E = RHS;
XRVal = LHS;
IsXLHSInRHSPart = true;
} else if (XId == RHSId) {
E = LHS;
XRVal = RHS;
IsXLHSInRHSPart = false;
} else {
ErrorLoc = AtomicInnerBinOp->getExprLoc();
ErrorRange = AtomicInnerBinOp->getSourceRange();
Expand Down Expand Up @@ -3381,7 +3388,7 @@ bool OpenMPAtomicUpdateChecker::checkBinaryOperation(
SemaRef.Diag(NoteLoc, NoteId) << ErrorFound << NoteRange;
return true;
} else if (SemaRef.CurContext->isDependentContext())
E = X = XRVal = nullptr;
E = X = UpdateExpr = nullptr;
return false;
}

Expand All @@ -3405,26 +3412,26 @@ bool OpenMPAtomicUpdateChecker::checkStatement(Stmt *S, unsigned DiagId,
if (auto *AtomicCompAssignOp = dyn_cast<CompoundAssignOperator>(
AtomicBody->IgnoreParenImpCasts())) {
// Check for Compound Assignment Operation
OpKind = BinaryOperator::getOpForCompoundAssignment(
Op = BinaryOperator::getOpForCompoundAssignment(
AtomicCompAssignOp->getOpcode());
X = AtomicCompAssignOp->getLHS();
XRVal = SemaRef.PerformImplicitConversion(
X, AtomicCompAssignOp->getComputationLHSType(),
Sema::AA_Casting, /*AllowExplicit=*/true).get();
OpLoc = AtomicCompAssignOp->getOperatorLoc();
E = AtomicCompAssignOp->getRHS();
X = AtomicCompAssignOp->getLHS();
IsXLHSInRHSPart = true;
} else if (auto *AtomicBinOp = dyn_cast<BinaryOperator>(
AtomicBody->IgnoreParenImpCasts())) {
// Check for Binary Operation
return checkBinaryOperation(AtomicBinOp, DiagId, NoteId);
if(checkBinaryOperation(AtomicBinOp, DiagId, NoteId))
return true;
} else if (auto *AtomicUnaryOp =
// Check for Binary Operation
dyn_cast<UnaryOperator>(AtomicBody->IgnoreParenImpCasts())) {
// Check for Unary Operation
if (AtomicUnaryOp->isIncrementDecrementOp()) {
OpKind = AtomicUnaryOp->isIncrementOp() ? BO_Add : BO_Sub;
XRVal = X = AtomicUnaryOp->getSubExpr();
E = SemaRef.ActOnIntegerConstant(AtomicUnaryOp->getOperatorLoc(), 1)
.get();
Op = AtomicUnaryOp->isIncrementOp() ? BO_Add : BO_Sub;
OpLoc = AtomicUnaryOp->getOperatorLoc();
X = AtomicUnaryOp->getSubExpr();
E = SemaRef.ActOnIntegerConstant(OpLoc, /*uint64_t Val=*/1).get();
IsXLHSInRHSPart = true;
} else {
ErrorFound = NotAnUnaryIncDecExpression;
ErrorLoc = AtomicUnaryOp->getExprLoc();
Expand Down Expand Up @@ -3452,7 +3459,26 @@ bool OpenMPAtomicUpdateChecker::checkStatement(Stmt *S, unsigned DiagId,
SemaRef.Diag(NoteLoc, NoteId) << ErrorFound << NoteRange;
return true;
} else if (SemaRef.CurContext->isDependentContext())
E = X = XRVal = nullptr;
E = X = UpdateExpr = nullptr;
if (E && X) {
// Build an update expression of form 'OpaqueValueExpr(x) binop
// OpaqueValueExpr(expr)' or 'OpaqueValueExpr(expr) binop
// OpaqueValueExpr(x)' and then cast it to the type of the 'x' expression.
auto *OVEX = new (SemaRef.getASTContext())
OpaqueValueExpr(X->getExprLoc(), X->getType(), VK_RValue);
auto *OVEExpr = new (SemaRef.getASTContext())
OpaqueValueExpr(E->getExprLoc(), E->getType(), VK_RValue);
auto Update =
SemaRef.CreateBuiltinBinOp(OpLoc, Op, IsXLHSInRHSPart ? OVEX : OVEExpr,
IsXLHSInRHSPart ? OVEExpr : OVEX);
if (Update.isInvalid())
return true;
Update = SemaRef.PerformImplicitConversion(Update.get(), X->getType(),
Sema::AA_Casting);
if (Update.isInvalid())
return true;
UpdateExpr = Update.get();
}
return false;
}

Expand Down Expand Up @@ -3490,11 +3516,11 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
if (auto *EWC = dyn_cast<ExprWithCleanups>(Body))
Body = EWC->getSubExpr();

BinaryOperatorKind OpKind = BO_PtrMemD;
Expr *X = nullptr;
Expr *XRVal = nullptr;
Expr *V = nullptr;
Expr *E = nullptr;
Expr *UE = nullptr;
bool IsXLHSInRHSPart = false;
// OpenMP [2.12.6, atomic Construct]
// In the next expressions:
// * x and v (as applicable) are both l-value expressions with scalar type.
Expand Down Expand Up @@ -3652,8 +3678,8 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
if (!CurContext->isDependentContext()) {
E = Checker.getExpr();
X = Checker.getX();
XRVal = Checker.getXRVal();
OpKind = Checker.getOpKind();
UE = Checker.getUpdateExpr();
IsXLHSInRHSPart = Checker.isXLHSInRHSPart();
}
} else if (AtomicKind == OMPC_capture) {
if (isa<Expr>(Body) && !isa<BinaryOperator>(Body)) {
Expand All @@ -3670,7 +3696,7 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
getCurFunction()->setHasBranchProtectedScope();

return OMPAtomicDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt,
OpKind, X, XRVal, V, E);
X, V, E, UE, IsXLHSInRHSPart);
}

StmtResult Sema::ActOnOpenMPTargetDirective(ArrayRef<OMPClause *> Clauses,
Expand Down
4 changes: 2 additions & 2 deletions clang/lib/Serialization/ASTReaderStmt.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2157,11 +2157,11 @@ void ASTStmtReader::VisitOMPAtomicDirective(OMPAtomicDirective *D) {
// The NumClauses field was read in ReadStmtFromStream.
++Idx;
VisitOMPExecutableDirective(D);
D->setOpKind(static_cast<BinaryOperatorKind>(Record[Idx++]));
D->setX(Reader.ReadSubExpr());
D->setXRVal(Reader.ReadSubExpr());
D->setV(Reader.ReadSubExpr());
D->setExpr(Reader.ReadSubExpr());
D->setUpdateExpr(Reader.ReadSubExpr());
D->IsXLHSInRHSPart = Record[Idx++] != 0;
}

void ASTStmtReader::VisitOMPTargetDirective(OMPTargetDirective *D) {
Expand Down
4 changes: 2 additions & 2 deletions clang/lib/Serialization/ASTWriterStmt.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2005,11 +2005,11 @@ void ASTStmtWriter::VisitOMPAtomicDirective(OMPAtomicDirective *D) {
VisitStmt(D);
Record.push_back(D->getNumClauses());
VisitOMPExecutableDirective(D);
Record.push_back(D->getOpKind());
Writer.AddStmt(D->getX());
Writer.AddStmt(D->getXRVal());
Writer.AddStmt(D->getV());
Writer.AddStmt(D->getExpr());
Writer.AddStmt(D->getUpdateExpr());
Record.push_back(D->isXLHSInRHSPart() ? 1 : 0);
Code = serialization::STMT_OMP_ATOMIC_DIRECTIVE;
}

Expand Down
34 changes: 29 additions & 5 deletions clang/test/OpenMP/atomic_codegen.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,22 @@ void parallel_atomic_ewc() {
// CHECK: invoke void @_ZN2StD1Ev(%struct.St* [[TEMP_ST_ADDR]])
#pragma omp atomic write
St().get() = b;
// CHECK: invoke void @_ZN2StC1Ev(%struct.St* [[TEMP_ST_ADDR:%.+]])
// CHECK: [[SCALAR_ADDR:%.+]] = invoke dereferenceable(4) i32* @_ZN2St3getEv(%struct.St* [[TEMP_ST_ADDR]])
// CHECK: [[B_VAL:%.+]] = load i32, i32* @b
// CHECK: [[OLD_VAL:%.+]] = load atomic i32, i32* [[SCALAR_ADDR]] monotonic,
// CHECK: br label %[[OMP_UPDATE:.+]]
// CHECK: [[OMP_UPDATE]]
// CHECK: [[OLD_PHI_VAL:%.+]] = phi i32 [ [[OLD_VAL]], %{{.+}} ], [ [[NEW_OLD_VAL:%.+]], %[[OMP_UPDATE]] ]
// CHECK: [[NEW_VAL:%.+]] = srem i32 [[OLD_PHI_VAL]], [[B_VAL]]
// CHECK: [[RES:%.+]] = cmpxchg i32* [[SCALAR_ADDR]], i32 [[OLD_PHI_VAL]], i32 [[NEW_VAL]] monotonic monotonic
// CHECK: [[NEW_OLD_VAL]] = extractvalue { i32, i1 } [[RES]], 0
// CHECK: [[COND:%.+]] = extractvalue { i32, i1 } [[RES]], 1
// CHECK: br i1 [[COND]], label %[[OMP_DONE:.+]], label %[[OMP_UPDATE]]
// CHECK: [[OMP_DONE]]
// CHECK: invoke void @_ZN2StD1Ev(%struct.St* [[TEMP_ST_ADDR]])
#pragma omp atomic
St().get() %= b;
}
}

Expand All @@ -50,11 +66,19 @@ void parallel_atomic() {
// TERM_DEBUG: unwind label %[[TERM_LPAD:.+]],
// TERM_DEBUG-NOT: __kmpc_global_thread_num
// TERM_DEBUG: store atomic i32 {{%.+}}, i32* @{{.+}} monotonic, {{.*}}!dbg [[WRITE_LOC:![0-9]+]]
// TERM_DEBUG: [[TERM_LPAD]]
// TERM_DEBUG: call void @__clang_call_terminate
// TERM_DEBUG: unreachable
a = foo();
#pragma omp atomic update
// TERM_DEBUG-NOT: __kmpc_global_thread_num
// TERM_DEBUG: invoke {{.*}}foo{{.*}}()
// TERM_DEBUG: unwind label %[[TERM_LPAD:.+]],
// TERM_DEBUG-NOT: __kmpc_global_thread_num
// TERM_DEBUG: atomicrmw add i32* @{{.+}}, i32 %{{.+}} monotonic, {{.*}}!dbg [[UPDATE_LOC:![0-9]+]]
a += foo();
}
// TERM_DEBUG: [[TERM_LPAD]]
// TERM_DEBUG: call void @__clang_call_terminate
// TERM_DEBUG: unreachable
}
// TERM_DEBUG-DAG: [[READ_LOC]] = !MDLocation(line: 41,
// TERM_DEBUG-DAG: [[WRITE_LOC]] = !MDLocation(line: 47,
// TERM_DEBUG-DAG: [[READ_LOC]] = !MDLocation(line: [[@LINE-25]],
// TERM_DEBUG-DAG: [[WRITE_LOC]] = !MDLocation(line: [[@LINE-20]],
// TERM_DEBUG-DAG: [[UPDATE_LOC]] = !MDLocation(line: [[@LINE-14]],
1,070 changes: 1,070 additions & 0 deletions clang/test/OpenMP/atomic_update_codegen.cpp

Large diffs are not rendered by default.