Skip to content

Commit

Permalink
[clang][CodeGen] Allow memcpy replace with trivial auto var init
Browse files Browse the repository at this point in the history 
When emitting the storage (or memory copy operations) for constant
initializers, the decision whether to split a constant structure or
array store into a sequence of field stores or to use `memcpy` is
based upon the optimization level and the size of the initializer.
In afe8b93, we extended this by
allowing constants to be split when the array (or struct) type does
not match the type of data the address to the object (constant) is
expected to contain. This may happen when `emitStoresForConstant` is
called by `EmitAutoVarInit`, as the element type of the address gets
shrunk. When this occurs, let the initializer be split into a bunch
of stores only under `-ftrivial-auto-var-init=pattern`.

Fixes: #84178.
  • Loading branch information
@antoniofrighetto @tstellar
antoniofrighetto authored and tstellar committed Mar 27, 2024
1 parent 2498e3a commit 98a15dd
Show file tree
Hide file tree
Showing 7 changed files with 56 additions and 63 deletions.
43 changes: 27 additions & 16 deletions clang/lib/CodeGen/CGDecl.cpp
View file
Expand Up @@ -1241,27 +1241,38 @@ static void emitStoresForConstant(CodeGenModule &CGM, const VarDecl &D,
return;
}

// If the initializer is small, use a handful of stores.
// If the initializer is small or trivialAutoVarInit is set, use a handful of
// stores.
bool IsTrivialAutoVarInitPattern =
CGM.getContext().getLangOpts().getTrivialAutoVarInit() ==
LangOptions::TrivialAutoVarInitKind::Pattern;
if (shouldSplitConstantStore(CGM, ConstantSize)) {
if (auto *STy = dyn_cast<llvm::StructType>(Ty)) {
const llvm::StructLayout *Layout =
CGM.getDataLayout().getStructLayout(STy);
for (unsigned i = 0; i != constant->getNumOperands(); i++) {
CharUnits CurOff = CharUnits::fromQuantity(Layout->getElementOffset(i));
Address EltPtr = Builder.CreateConstInBoundsByteGEP(
Loc.withElementType(CGM.Int8Ty), CurOff);
emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder,
constant->getAggregateElement(i), IsAutoInit);
if (STy == Loc.getElementType() ||
(STy != Loc.getElementType() && IsTrivialAutoVarInitPattern)) {
const llvm::StructLayout *Layout =
CGM.getDataLayout().getStructLayout(STy);
for (unsigned i = 0; i != constant->getNumOperands(); i++) {
CharUnits CurOff =
CharUnits::fromQuantity(Layout->getElementOffset(i));
Address EltPtr = Builder.CreateConstInBoundsByteGEP(
Loc.withElementType(CGM.Int8Ty), CurOff);
emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder,
constant->getAggregateElement(i), IsAutoInit);
}
return;
}
return;
} else if (auto *ATy = dyn_cast<llvm::ArrayType>(Ty)) {
for (unsigned i = 0; i != ATy->getNumElements(); i++) {
Address EltPtr = Builder.CreateConstGEP(
Loc.withElementType(ATy->getElementType()), i);
emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder,
constant->getAggregateElement(i), IsAutoInit);
if (ATy == Loc.getElementType() ||
(ATy != Loc.getElementType() && IsTrivialAutoVarInitPattern)) {
for (unsigned i = 0; i != ATy->getNumElements(); i++) {
Address EltPtr = Builder.CreateConstGEP(
Loc.withElementType(ATy->getElementType()), i);
emitStoresForConstant(CGM, D, EltPtr, isVolatile, Builder,
constant->getAggregateElement(i), IsAutoInit);
}
return;
}
return;
}
}

Expand Down
4 changes: 2 additions & 2 deletions clang/test/CodeGen/aapcs-align.cpp
View file
Expand Up @@ -134,8 +134,8 @@ void g6() {
f6m(1, 2, 3, 4, 5, s);
}
// CHECK: define{{.*}} void @g6
// CHECK: call void @f6(i32 noundef 1, [4 x i32] [i32 6, i32 7, i32 0, i32 0])
// CHECK: call void @f6m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [4 x i32] [i32 6, i32 7, i32 0, i32 0])
// CHECK: call void @f6(i32 noundef 1, [4 x i32] [i32 6, i32 7, i32 0, i32 undef])
// CHECK: call void @f6m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [4 x i32] [i32 6, i32 7, i32 0, i32 undef])
// CHECK: declare void @f6(i32 noundef, [4 x i32])
// CHECK: declare void @f6m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [4 x i32])
}
8 changes: 4 additions & 4 deletions clang/test/CodeGen/aapcs64-align.cpp
View file
Expand Up @@ -75,8 +75,8 @@ void g4() {
f4m(1, 2, 3, 4, 5, s);
}
// CHECK: define{{.*}} void @g4()
// CHECK: call void @f4(i32 noundef 1, [2 x i64] %{{.*}})
// CHECK: void @f4m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] %{{.*}})
// CHECK: call void @f4(i32 noundef 1, [2 x i64] [i64 30064771078, i64 0])
// CHECK: void @f4m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] [i64 30064771078, i64 0])
// CHECK: declare void @f4(i32 noundef, [2 x i64])
// CHECK: declare void @f4m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [2 x i64])

Expand All @@ -95,8 +95,8 @@ void f5m(int, int, int, int, int, P16);
f5m(1, 2, 3, 4, 5, s);
}
// CHECK: define{{.*}} void @g5()
// CHECK: call void @f5(i32 noundef 1, [2 x i64] %{{.*}})
// CHECK: void @f5m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] %{{.*}})
// CHECK: call void @f5(i32 noundef 1, [2 x i64] [i64 30064771078, i64 0])
// CHECK: void @f5m(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, [2 x i64] [i64 30064771078, i64 0])
// CHECK: declare void @f5(i32 noundef, [2 x i64])
// CHECK: declare void @f5m(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, [2 x i64])

Expand Down
24 changes: 6 additions & 18 deletions clang/test/CodeGen/attr-counted-by.c
View file
Expand Up @@ -1288,16 +1288,10 @@ int test14(int idx) {
// NO-SANITIZE-WITH-ATTR-LABEL: define dso_local i32 @test15(
// NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR4]] {
// NO-SANITIZE-WITH-ATTR-NEXT: entry:
// NO-SANITIZE-WITH-ATTR-NEXT: [[FOO:%.*]] = alloca [[STRUCT_ANON_8:%.*]], align 4
// NO-SANITIZE-WITH-ATTR-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR12]]
// NO-SANITIZE-WITH-ATTR-NEXT: store i32 1, ptr [[FOO]], align 4
// NO-SANITIZE-WITH-ATTR-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 4
// NO-SANITIZE-WITH-ATTR-NEXT: store i32 2, ptr [[TMP0]], align 4
// NO-SANITIZE-WITH-ATTR-NEXT: [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
// NO-SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANON_8]], ptr [[FOO]], i64 0, i32 2, i64 [[IDXPROM]]
// NO-SANITIZE-WITH-ATTR-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
// NO-SANITIZE-WITH-ATTR-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR12]]
// NO-SANITIZE-WITH-ATTR-NEXT: ret i32 [[TMP1]]
// NO-SANITIZE-WITH-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr getelementptr inbounds ([[STRUCT_ANON_8:%.*]], ptr @__const.test15.foo, i64 1, i32 0), i64 0, i64 [[IDXPROM]]
// NO-SANITIZE-WITH-ATTR-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
// NO-SANITIZE-WITH-ATTR-NEXT: ret i32 [[TMP0]]
//
// SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test15(
// SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
Expand All @@ -1315,16 +1309,10 @@ int test14(int idx) {
// NO-SANITIZE-WITHOUT-ATTR-LABEL: define dso_local i32 @test15(
// NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[IDX:%.*]]) local_unnamed_addr #[[ATTR1]] {
// NO-SANITIZE-WITHOUT-ATTR-NEXT: entry:
// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[FOO:%.*]] = alloca [[STRUCT_ANON_8:%.*]], align 4
// NO-SANITIZE-WITHOUT-ATTR-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR9]]
// NO-SANITIZE-WITHOUT-ATTR-NEXT: store i32 1, ptr [[FOO]], align 4
// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[FOO]], i64 4
// NO-SANITIZE-WITHOUT-ATTR-NEXT: store i32 2, ptr [[TMP0]], align 4
// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[IDXPROM:%.*]] = sext i32 [[IDX]] to i64
// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_ANON_8]], ptr [[FOO]], i64 0, i32 2, i64 [[IDXPROM]]
// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
// NO-SANITIZE-WITHOUT-ATTR-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[FOO]]) #[[ATTR9]]
// NO-SANITIZE-WITHOUT-ATTR-NEXT: ret i32 [[TMP1]]
// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [0 x i32], ptr getelementptr inbounds ([[STRUCT_ANON_8:%.*]], ptr @__const.test15.foo, i64 1, i32 0), i64 0, i64 [[IDXPROM]]
// NO-SANITIZE-WITHOUT-ATTR-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
// NO-SANITIZE-WITHOUT-ATTR-NEXT: ret i32 [[TMP0]]
//
int test15(int idx) {
struct {
Expand Down
27 changes: 15 additions & 12 deletions clang/test/CodeGenCXX/auto-var-init.cpp
View file
@@ -1,8 +1,8 @@
// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK,CHECK-O0
// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O0,PATTERN,PATTERN-O0
// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O1,PATTERN,PATTERN-O1
// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=PATTERN,PATTERN-O1
// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=zero %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O0,ZERO,ZERO-O0
// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=zero %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O1,ZERO,ZERO-O1
// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown -fblocks -ftrivial-auto-var-init=zero %s -O1 -emit-llvm -o - | FileCheck %s -check-prefixes=ZERO,ZERO-O1
// RUN: %clang_cc1 -std=c++14 -triple i386-unknown-unknown -fblocks -ftrivial-auto-var-init=pattern %s -emit-llvm -o - | FileCheck %s -check-prefixes=CHECK-O0,PATTERN,PATTERN-O0

#pragma clang diagnostic ignored "-Winaccessible-base"
Expand Down Expand Up @@ -1303,9 +1303,10 @@ TEST_CUSTOM(semivolatile, semivolatile, { 0x44444444, 0x44444444 });
// CHECK-O0: call void @llvm.memcpy
// CHECK-NOT: !annotation
// CHECK-O0: call void @{{.*}}used{{.*}}%custom)
// CHECK-O1: store i32 1145324612, ptr %custom, align 4
// CHECK-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 4
// CHECK-O1-NEXT: store i32 1145324612, ptr %[[I]], align 4
// PATTERN-O1: store i32 1145324612, ptr %custom, align 4
// PATTERN-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 4
// PATTERN-O1-NEXT: store i32 1145324612, ptr %[[I]], align 4
// ZERO-O1: store i64 4919131752989213764, ptr %custom, align 8
// CHECK-NOT: !annotation

TEST_UNINIT(semivolatileinit, semivolatileinit);
Expand Down Expand Up @@ -1418,7 +1419,8 @@ TEST_CUSTOM(matching, matching, { .f = 0xf00f });
// CHECK-O0: call void @llvm.memcpy
// CHECK-NOT: !annotation
// CHECK-O0: call void @{{.*}}used{{.*}}%custom)
// CHECK-O1: store float 6.145500e+04, ptr {{.*}}, align 4
// PATTERN-O1: store float 6.145500e+04, ptr {{.*}}, align 4
// ZERO-O1: store i32 1198526208, ptr %custom, align 4
// CHECK-NOT: !annotation

TEST_UNINIT(matchingreverse, matchingreverse);
Expand All @@ -1445,7 +1447,8 @@ TEST_CUSTOM(matchingreverse, matchingreverse, { .i = 0xf00f });
// CHECK-O0: call void @llvm.memcpy
// CHECK-NOT: !annotation
// CHECK-O0: call void @{{.*}}used{{.*}}%custom)
// CHECK-O1: store i32 61455, ptr %custom, align 4
// PATTERN-O1: store i32 61455, ptr %custom, align 4
// ZERO-O1: store i32 61455, ptr %custom, align 4
// CHECK-NOT: !annotation

TEST_UNINIT(unmatched, unmatched);
Expand All @@ -1471,7 +1474,8 @@ TEST_CUSTOM(unmatched, unmatched, { .i = 0x3badbeef });
// CHECK-O0: call void @llvm.memcpy
// CHECK-NOT: !annotation
// CHECK-O0: call void @{{.*}}used{{.*}}%custom)
// CHECK-O1: store i32 1001242351, ptr {{.*}}, align 4
// PATTERN-O1: store i32 1001242351, ptr {{.*}}, align 4
// ZERO-O1: store i32 1001242351, ptr {{.*}}, align 4
// CHECK-NOT: !annotation

TEST_UNINIT(unmatchedreverse, unmatchedreverse);
Expand Down Expand Up @@ -1504,9 +1508,7 @@ TEST_CUSTOM(unmatchedreverse, unmatchedreverse, { .c = 42 });
// PATTERN-O1-NEXT: store i8 -86, ptr %[[I]], align {{.*}}
// PATTERN-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 3
// PATTERN-O1-NEXT: store i8 -86, ptr %[[I]], align {{.*}}
// ZERO-O1: store i8 42, ptr {{.*}}, align 4
// ZERO-O1-NEXT: %[[I:[^ ]*]] = getelementptr inbounds i8, ptr %custom, i64 1
// ZERO-O1-NEXT: call void @llvm.memset.{{.*}}({{.*}}, i8 0, i64 3, {{.*}})
// ZERO-O1: store i32 42, ptr {{.*}}, align 4

TEST_UNINIT(unmatchedfp, unmatchedfp);
// CHECK-LABEL: @test_unmatchedfp_uninit()
Expand All @@ -1531,7 +1533,8 @@ TEST_CUSTOM(unmatchedfp, unmatchedfp, { .d = 3.1415926535897932384626433 });
// CHECK-O0: call void @llvm.memcpy
// CHECK-NOT: !annotation
// CHECK-O0: call void @{{.*}}used{{.*}}%custom)
// CHECK-O1: store double 0x400921FB54442D18, ptr %custom, align 8
// PATTERN-O1: store double 0x400921FB54442D18, ptr %custom, align 8
// ZERO-O1: store i64 4614256656552045848, ptr %custom, align 8
// CHECK-NOT: !annotation

TEST_UNINIT(emptyenum, emptyenum);
Expand Down
9 changes: 1 addition & 8 deletions clang/test/CodeGenOpenCL/amdgpu-printf.cl
View file
Expand Up @@ -30,14 +30,7 @@ __kernel void test_printf_int(int i) {
// CHECK-NEXT: [[S:%.*]] = alloca [4 x i8], align 1, addrspace(5)
// CHECK-NEXT: store i32 [[I:%.*]], ptr addrspace(5) [[I_ADDR]], align 4, !tbaa [[TBAA8]]
// CHECK-NEXT: call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[S]]) #[[ATTR5:[0-9]+]]
// CHECK-NEXT: [[LOC0:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 0
// CHECK-NEXT: store i8 102, ptr addrspace(5) [[LOC0]], align 1
// CHECK-NEXT: [[LOC1:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 1
// CHECK-NEXT: store i8 111, ptr addrspace(5) [[LOC1]], align 1
// CHECK-NEXT: [[LOC2:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 2
// CHECK-NEXT: store i8 111, ptr addrspace(5) [[LOC2]], align 1
// CHECK-NEXT: [[LOC3:%.*]] = getelementptr i8, ptr addrspace(5) [[S]], i64 3
// CHECK-NEXT: store i8 0, ptr addrspace(5) [[LOC3]], align 1
// CHECK-NEXT: call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) align 1 [[S]], ptr addrspace(4) align 1 @__const.test_printf_str_int.s, i64 4, i1 false)
// CHECK-NEXT: [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i8], ptr addrspace(5) [[S]], i64 0, i64 0
// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(5) [[I_ADDR]], align 4, !tbaa [[TBAA8]]
// CHECK-NEXT: [[CALL:%.*]] = call i32 (ptr addrspace(4), ...) @printf(ptr addrspace(4) noundef @.str.2, ptr addrspace(5) noundef [[ARRAYDECAY]], i32 noundef [[TMP2]]) #[[ATTR4]]
Expand Down
4 changes: 1 addition & 3 deletions clang/test/OpenMP/bug54082.c
View file
Expand Up @@ -69,9 +69,7 @@ void foo() {
// CHECK-NEXT: [[X_TRAITS:%.*]] = alloca [1 x %struct.omp_alloctrait_t], align 16
// CHECK-NEXT: [[X_ALLOC:%.*]] = alloca i64, align 8
// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[X_TRAITS]]) #[[ATTR5:[0-9]+]]
// CHECK-NEXT: store i32 2, ptr [[X_TRAITS]], align 16
// CHECK-NEXT: [[LOC0:%.*]] = getelementptr inbounds i8, ptr [[X_TRAITS]], i64 8
// CHECK-NEXT: store i64 64, ptr [[LOC0]], align 8
// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(16) [[X_TRAITS]], ptr noundef nonnull align 16 dereferenceable(16) @__const.foo.x_traits, i64 16, i1 false)
// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[X_ALLOC]]) #[[ATTR5]]
// CHECK-NEXT: [[CALL:%.*]] = call i64 @omp_init_allocator(i64 noundef 0, i32 noundef 1, ptr noundef nonnull [[X_TRAITS]]) #[[ATTR5]]
// CHECK-NEXT: store i64 [[CALL]], ptr [[X_ALLOC]], align 8, !tbaa [[TBAA3:![0-9]+]]
Expand Down

0 comments on commit 98a15dd

Please sign in to comment.