From 57e1b095244131e723f52f415bd18e7e0d6c1ab9 Mon Sep 17 00:00:00 2001 From: Sirui Mu Date: Sun, 19 Oct 2025 22:00:49 +0800 Subject: [PATCH] [CIR] Add support for atomic test-and-set and atomic clear This patch adds support for the following atomic builtin functions: - `__atomic_test_and_set` - `__atomic_clear` --- clang/include/clang/CIR/Dialect/IR/CIROps.td | 62 ++++++++++++++++ clang/lib/CIR/CodeGen/CIRGenAtomic.cpp | 26 +++++-- .../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp | 40 +++++++++++ clang/test/CIR/CodeGen/atomic.c | 70 +++++++++++++++++++ 4 files changed, 194 insertions(+), 4 deletions(-) diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td index e0163a4fecd5f..5a3dfec6e015c 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIROps.td +++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td @@ -4554,4 +4554,66 @@ def CIR_AtomicCmpXchgOp : CIR_Op<"atomic.cmpxchg", [ }]; } +def CIR_AtomicTestAndSetOp : CIR_Op<"atomic.test_and_set"> { + let summary = "Atomic test and set"; + let description = [{ + C/C++ atomic test and set operation. Implements the builtin function + `__atomic_test_and_set`. + + The operation takes as its only operand a pointer to an 8-bit signed + integer. The operation atomically set the integer to an implementation- + defined non-zero "set" value. The result of the operation is a boolean value + indicating whether the previous value of the integer was the "set" value. + + Example: + ```mlir + %res = cir.atomic.test_and_set seq_cst %ptr : !cir.ptr -> !cir.bool + ``` + }]; + + let arguments = (ins + Arg, "", [MemRead, MemWrite]>:$ptr, + Arg:$mem_order, + OptionalAttr:$alignment, + UnitAttr:$is_volatile + ); + + let results = (outs CIR_BoolType:$result); + + let assemblyFormat = [{ + $mem_order $ptr + (`volatile` $is_volatile^)? + `:` qualified(type($ptr)) `->` qualified(type($result)) attr-dict + }]; +} + +def CIR_AtomicClearOp : CIR_Op<"atomic.clear"> { + let summary = "Atomic clear"; + let description = [{ + C/C++ atomic clear operation. Implements the builtin function + `__atomic_clear`. + + The operation takes as its only operand a pointer to an 8-bit signed + integer. The operation atomically sets the integer to zero. + + Example: + ```mlir + cir.atomic.clear seq_cst %ptr : !cir.ptr + ``` + }]; + + let arguments = (ins + Arg, "", [MemRead, MemWrite]>:$ptr, + Arg:$mem_order, + OptionalAttr:$alignment, + UnitAttr:$is_volatile + ); + + let assemblyFormat = [{ + $mem_order $ptr + (`volatile` $is_volatile^)? + `:` qualified(type($ptr)) attr-dict + }]; +} + #endif // CLANG_CIR_DIALECT_IR_CIROPS_TD diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp index a9983f882e28c..67ca60c971e04 100644 --- a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp @@ -407,6 +407,23 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest, opName = cir::AtomicXchgOp::getOperationName(); break; + case AtomicExpr::AO__atomic_test_and_set: { + auto op = cir::AtomicTestAndSetOp::create( + builder, loc, ptr.getPointer(), order, + builder.getI64IntegerAttr(ptr.getAlignment().getQuantity()), + expr->isVolatile()); + builder.createStore(loc, op, dest); + return; + } + + case AtomicExpr::AO__atomic_clear: { + cir::AtomicClearOp::create( + builder, loc, ptr.getPointer(), order, + builder.getI64IntegerAttr(ptr.getAlignment().getQuantity()), + expr->isVolatile()); + return; + } + case AtomicExpr::AO__opencl_atomic_init: case AtomicExpr::AO__hip_atomic_compare_exchange_strong: @@ -502,10 +519,6 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest, case AtomicExpr::AO__c11_atomic_fetch_nand: case AtomicExpr::AO__atomic_fetch_nand: case AtomicExpr::AO__scoped_atomic_fetch_nand: - - case AtomicExpr::AO__atomic_test_and_set: - - case AtomicExpr::AO__atomic_clear: cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: expr op NYI"); return; } @@ -581,6 +594,8 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) { case AtomicExpr::AO__atomic_load_n: case AtomicExpr::AO__c11_atomic_load: + case AtomicExpr::AO__atomic_test_and_set: + case AtomicExpr::AO__atomic_clear: break; case AtomicExpr::AO__atomic_load: @@ -640,6 +655,9 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) { dest = atomics.castToAtomicIntPointer(dest); } else if (e->isCmpXChg()) { dest = createMemTemp(resultTy, getLoc(e->getSourceRange()), "cmpxchg.bool"); + } else if (e->getOp() == AtomicExpr::AO__atomic_test_and_set) { + dest = createMemTemp(resultTy, getLoc(e->getSourceRange()), + "test_and_set.bool"); } else if (!resultTy->isVoidType()) { dest = atomics.createTempAlloca(); if (shouldCastToIntPtrTy) diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index 0243bf120f396..b688184c70af0 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -730,6 +730,46 @@ mlir::LogicalResult CIRToLLVMAtomicXchgOpLowering::matchAndRewrite( return mlir::success(); } +mlir::LogicalResult CIRToLLVMAtomicTestAndSetOpLowering::matchAndRewrite( + cir::AtomicTestAndSetOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const { + assert(!cir::MissingFeatures::atomicSyncScopeID()); + + mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder()); + + auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), + rewriter.getI8Type(), 1); + auto rmw = mlir::LLVM::AtomicRMWOp::create( + rewriter, op.getLoc(), mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(), + one, llvmOrder, /*syncscope=*/llvm::StringRef(), + adaptor.getAlignment().value_or(0), op.getIsVolatile()); + + auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), + rewriter.getI8Type(), 0); + auto cmp = mlir::LLVM::ICmpOp::create( + rewriter, op.getLoc(), mlir::LLVM::ICmpPredicate::ne, rmw, zero); + + rewriter.replaceOp(op, cmp); + return mlir::success(); +} + +mlir::LogicalResult CIRToLLVMAtomicClearOpLowering::matchAndRewrite( + cir::AtomicClearOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const { + assert(!cir::MissingFeatures::atomicSyncScopeID()); + + mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder()); + auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), + rewriter.getI8Type(), 0); + auto store = mlir::LLVM::StoreOp::create( + rewriter, op.getLoc(), zero, adaptor.getPtr(), + adaptor.getAlignment().value_or(0), op.getIsVolatile(), + /*isNonTemporal=*/false, /*isInvariantGroup=*/false, llvmOrder); + + rewriter.replaceOp(op, store); + return mlir::success(); +} + mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite( cir::BitClrsbOp op, OpAdaptor adaptor, mlir::ConversionPatternRewriter &rewriter) const { diff --git a/clang/test/CIR/CodeGen/atomic.c b/clang/test/CIR/CodeGen/atomic.c index 440010a0b6938..cf202266f5716 100644 --- a/clang/test/CIR/CodeGen/atomic.c +++ b/clang/test/CIR/CodeGen/atomic.c @@ -514,3 +514,73 @@ void atomic_exchange_n(int *ptr, int value) { // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4 // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4 } + +void test_and_set(void *p) { + // CIR-LABEL: @test_and_set + // LLVM-LABEL: @test_and_set + // OGCG-LABEL: @test_and_set + + __atomic_test_and_set(p, __ATOMIC_SEQ_CST); + // CIR: %[[VOID_PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr>, !cir.ptr + // CIR-NEXT: %[[PTR:.+]] = cir.cast bitcast %[[VOID_PTR]] : !cir.ptr -> !cir.ptr + // CIR-NEXT: %[[RES:.+]] = cir.atomic.test_and_set seq_cst %[[PTR]] : !cir.ptr -> !cir.bool + // CIR-NEXT: cir.store align(1) %[[RES]], %{{.+}} : !cir.bool, !cir.ptr + + // LLVM: %[[PTR:.+]] = load ptr, ptr %{{.+}}, align 8 + // LLVM-NEXT: %[[RES:.+]] = atomicrmw xchg ptr %[[PTR]], i8 1 seq_cst, align 1 + // LLVM-NEXT: %{{.+}} = icmp ne i8 %[[RES]], 0 + + // OGCG: %[[PTR:.+]] = load ptr, ptr %{{.+}}, align 8 + // OGCG-NEXT: %[[RES:.+]] = atomicrmw xchg ptr %[[PTR]], i8 1 seq_cst, align 1 + // OGCG-NEXT: %{{.+}} = icmp ne i8 %[[RES]], 0 +} + +void test_and_set_volatile(volatile void *p) { + // CIR-LABEL: @test_and_set_volatile + // LLVM-LABEL: @test_and_set_volatile + // OGCG-LABEL: @test_and_set_volatile + + __atomic_test_and_set(p, __ATOMIC_SEQ_CST); + // CIR: %[[VOID_PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr>, !cir.ptr + // CIR-NEXT: %[[PTR:.+]] = cir.cast bitcast %[[VOID_PTR]] : !cir.ptr -> !cir.ptr + // CIR-NEXT: %[[RES:.+]] = cir.atomic.test_and_set seq_cst %[[PTR]] volatile : !cir.ptr -> !cir.bool + // CIR-NEXT: cir.store align(1) %[[RES]], %{{.+}} : !cir.bool, !cir.ptr + + // LLVM: %[[PTR:.+]] = load ptr, ptr %{{.+}}, align 8 + // LLVM-NEXT: %[[RES:.+]] = atomicrmw volatile xchg ptr %[[PTR]], i8 1 seq_cst, align 1 + // LLVM-NEXT: %{{.+}} = icmp ne i8 %[[RES]], 0 + + // OGCG: %[[PTR:.+]] = load ptr, ptr %{{.+}}, align 8 + // OGCG-NEXT: %[[RES:.+]] = atomicrmw volatile xchg ptr %[[PTR]], i8 1 seq_cst, align 1 + // OGCG-NEXT: %{{.+}} = icmp ne i8 %[[RES]], 0 +} + +void clear(void *p) { + // CIR-LABEL: @clear + // LLVM-LABEL: @clear + // OGCG-LABEL: @clear + + __atomic_clear(p, __ATOMIC_SEQ_CST); + // CIR: %[[VOID_PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr>, !cir.ptr + // CIR-NEXT: %[[PTR:.+]] = cir.cast bitcast %[[VOID_PTR]] : !cir.ptr -> !cir.ptr + // CIR: cir.atomic.clear seq_cst %[[PTR]] : !cir.ptr + + // LLVM: store atomic i8 0, ptr %{{.+}} seq_cst, align 1 + + // OGCG: store atomic i8 0, ptr %{{.+}} seq_cst, align 1 +} + +void clear_volatile(volatile void *p) { + // CIR-LABEL: @clear_volatile + // LLVM-LABEL: @clear_volatile + // OGCG-LABEL: @clear_volatile + + __atomic_clear(p, __ATOMIC_SEQ_CST); + // CIR: %[[VOID_PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr>, !cir.ptr + // CIR-NEXT: %[[PTR:.+]] = cir.cast bitcast %[[VOID_PTR]] : !cir.ptr -> !cir.ptr + // CIR: cir.atomic.clear seq_cst %[[PTR]] volatile : !cir.ptr + + // LLVM: store atomic volatile i8 0, ptr %{{.+}} seq_cst, align 1 + + // OGCG: store atomic volatile i8 0, ptr %{{.+}} seq_cst, align 1 +}