Skip to content

Commit

Permalink
[AArch64][SVE] Codegen dup_lane for dup(vector_extract)
Browse files Browse the repository at this point in the history 
Differential Revision: https://reviews.llvm.org/D99324
  • Loading branch information
@junparser
junparser committed Mar 30, 2021
1 parent b0db2db commit 1af373c
Show file tree
Hide file tree
Showing 3 changed files with 151 additions and 1 deletion.
24 changes: 24 additions & 0 deletions llvm/lib/Target/AArch64/SVEInstrFormats.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1019,6 +1019,30 @@ multiclass sve_int_perm_dup_i<string asm> {
(!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>;
def : InstAlias<"mov $Zd, $Qn",
(!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>;

// Duplicate extracted element of vector into all vector elements
def : Pat<(nxv16i8 (AArch64dup (i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)))),
(!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>;
def : Pat<(nxv8i16 (AArch64dup (i32 (vector_extract (nxv8i16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))),
(!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>;
def : Pat<(nxv4i32 (AArch64dup (i32 (vector_extract (nxv4i32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))),
(!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>;
def : Pat<(nxv2i64 (AArch64dup (i64 (vector_extract (nxv2i64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))),
(!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>;
def : Pat<(nxv8f16 (AArch64dup (f16 (vector_extract (nxv8f16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))),
(!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>;
def : Pat<(nxv8bf16 (AArch64dup (bf16 (vector_extract (nxv8bf16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))),
(!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>;
def : Pat<(nxv4f16 (AArch64dup (f16 (vector_extract (nxv4f16 ZPR:$vec), sve_elm_idx_extdup_s:$index)))),
(!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>;
def : Pat<(nxv2f16 (AArch64dup (f16 (vector_extract (nxv2f16 ZPR:$vec), sve_elm_idx_extdup_d:$index)))),
(!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>;
def : Pat<(nxv4f32 (AArch64dup (f32 (vector_extract (nxv4f32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))),
(!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>;
def : Pat<(nxv2f32 (AArch64dup (f32 (vector_extract (nxv2f32 ZPR:$vec), sve_elm_idx_extdup_d:$index)))),
(!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>;
def : Pat<(nxv2f64 (AArch64dup (f64 (vector_extract (nxv2f64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))),
(!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>;
}

class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty,
Expand Down
126 changes: 126 additions & 0 deletions llvm/test/CodeGen/AArch64/aarch64-dup-extract-scalable.ll
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple aarch64-none-linux-gnu -mattr=+sve | FileCheck %s

define <vscale x 16 x i8> @dup_extract_i8(<vscale x 16 x i8> %data) {
; CHECK-LABEL: dup_extract_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.b, z0.b[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 16 x i8> %data, i8 1
%.splatinsert = insertelement <vscale x 16 x i8> poison, i8 %1, i32 0
%.splat = shufflevector <vscale x 16 x i8> %.splatinsert, <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
ret <vscale x 16 x i8> %.splat
}

define <vscale x 8 x i16> @dup_extract_i16(<vscale x 8 x i16> %data) {
; CHECK-LABEL: dup_extract_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.h, z0.h[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 8 x i16> %data, i16 1
%.splatinsert = insertelement <vscale x 8 x i16> poison, i16 %1, i32 0
%.splat = shufflevector <vscale x 8 x i16> %.splatinsert, <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
ret <vscale x 8 x i16> %.splat
}

define <vscale x 4 x i32> @dup_extract_i32(<vscale x 4 x i32> %data) {
; CHECK-LABEL: dup_extract_i32:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.s, z0.s[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 4 x i32> %data, i32 1
%.splatinsert = insertelement <vscale x 4 x i32> poison, i32 %1, i32 0
%.splat = shufflevector <vscale x 4 x i32> %.splatinsert, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
ret <vscale x 4 x i32> %.splat
}

define <vscale x 2 x i64> @dup_extract_i64(<vscale x 2 x i64> %data) {
; CHECK-LABEL: dup_extract_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.d, z0.d[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 2 x i64> %data, i64 1
%.splatinsert = insertelement <vscale x 2 x i64> poison, i64 %1, i32 0
%.splat = shufflevector <vscale x 2 x i64> %.splatinsert, <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
ret <vscale x 2 x i64> %.splat
}

define <vscale x 8 x half> @dup_extract_f16(<vscale x 8 x half> %data) {
; CHECK-LABEL: dup_extract_f16:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.h, z0.h[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 8 x half> %data, i16 1
%.splatinsert = insertelement <vscale x 8 x half> poison, half %1, i32 0
%.splat = shufflevector <vscale x 8 x half> %.splatinsert, <vscale x 8 x half> poison, <vscale x 8 x i32> zeroinitializer
ret <vscale x 8 x half> %.splat
}

define <vscale x 4 x half> @dup_extract_f16_4(<vscale x 4 x half> %data) {
; CHECK-LABEL: dup_extract_f16_4:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.s, z0.s[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 4 x half> %data, i16 1
%.splatinsert = insertelement <vscale x 4 x half> poison, half %1, i32 0
%.splat = shufflevector <vscale x 4 x half> %.splatinsert, <vscale x 4 x half> poison, <vscale x 4 x i32> zeroinitializer
ret <vscale x 4 x half> %.splat
}

define <vscale x 2 x half> @dup_extract_f16_2(<vscale x 2 x half> %data) {
; CHECK-LABEL: dup_extract_f16_2:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.d, z0.d[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 2 x half> %data, i16 1
%.splatinsert = insertelement <vscale x 2 x half> poison, half %1, i32 0
%.splat = shufflevector <vscale x 2 x half> %.splatinsert, <vscale x 2 x half> poison, <vscale x 2 x i32> zeroinitializer
ret <vscale x 2 x half> %.splat
}

define <vscale x 8 x bfloat> @dup_extract_bf16(<vscale x 8 x bfloat> %data) #0 {
; CHECK-LABEL: dup_extract_bf16:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.h, z0.h[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 8 x bfloat> %data, i16 1
%.splatinsert = insertelement <vscale x 8 x bfloat> poison, bfloat %1, i32 0
%.splat = shufflevector <vscale x 8 x bfloat> %.splatinsert, <vscale x 8 x bfloat> poison, <vscale x 8 x i32> zeroinitializer
ret <vscale x 8 x bfloat> %.splat
}

define <vscale x 4 x float> @dup_extract_f32(<vscale x 4 x float> %data) {
; CHECK-LABEL: dup_extract_f32:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.s, z0.s[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 4 x float> %data, i32 1
%.splatinsert = insertelement <vscale x 4 x float> poison, float %1, i32 0
%.splat = shufflevector <vscale x 4 x float> %.splatinsert, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
ret <vscale x 4 x float> %.splat
}

define <vscale x 2 x float> @dup_extract_f32_2(<vscale x 2 x float> %data) {
; CHECK-LABEL: dup_extract_f32_2:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.d, z0.d[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 2 x float> %data, i32 1
%.splatinsert = insertelement <vscale x 2 x float> poison, float %1, i32 0
%.splat = shufflevector <vscale x 2 x float> %.splatinsert, <vscale x 2 x float> poison, <vscale x 2 x i32> zeroinitializer
ret <vscale x 2 x float> %.splat
}

define <vscale x 2 x double> @dup_extract_f64(<vscale x 2 x double> %data) {
; CHECK-LABEL: dup_extract_f64:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z0.d, z0.d[1]
; CHECK-NEXT: ret
%1 = extractelement <vscale x 2 x double> %data, i64 1
%.splatinsert = insertelement <vscale x 2 x double> poison, double %1, i32 0
%.splat = shufflevector <vscale x 2 x double> %.splatinsert, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
ret <vscale x 2 x double> %.splat
}

; +bf16 is required for the bfloat version.
attributes #0 = { "target-features"="+sve,+bf16" }
2 changes: 1 addition & 1 deletion llvm/test/CodeGen/AArch64/sve-ld-post-inc.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ define <vscale x 2 x double> @test_post_ld1_dup(double* %a, double** %ptr, i64 %
; CHECK: // %bb.0:
; CHECK-NEXT: ldr d0, [x0]
; CHECK-NEXT: add x8, x0, x2, lsl #3
; CHECK-NEXT: mov z0.d, d0
; CHECK-NEXT: str x8, [x1]
; CHECK-NEXT: mov z0.d, d0
; CHECK-NEXT: ret
%load = load double, double* %a
%dup = call <vscale x 2 x double> @llvm.aarch64.sve.dup.x.nxv2f64(double %load)
Expand Down

0 comments on commit 1af373c

Please sign in to comment.