diff --git a/llvm/test/CodeGen/X86/bitcnt-big-integer.ll b/llvm/test/CodeGen/X86/bitcnt-big-integer.ll index 5b3b27a3d61de..0fd555991ae29 100644 --- a/llvm/test/CodeGen/X86/bitcnt-big-integer.ll +++ b/llvm/test/CodeGen/X86/bitcnt-big-integer.ll @@ -1,8 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX2 -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F -; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 -mattr=+avx512vpopcntdq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512POPCNT +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefixes=AVX512,AVX512F +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512,AVX512VL +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 -mattr=+avx512vpopcntdq | FileCheck %s --check-prefixes=AVX512,AVX512POPCNT ; ; CTPOP @@ -16,6 +17,14 @@ define i32 @test_ctpop_i128(i128 %a0) nounwind { ; CHECK-NEXT: addl %ecx, %eax ; CHECK-NEXT: # kill: def $eax killed $eax killed $rax ; CHECK-NEXT: retq +; +; AVX512-LABEL: test_ctpop_i128: +; AVX512: # %bb.0: +; AVX512-NEXT: popcntq %rsi, %rcx +; AVX512-NEXT: popcntq %rdi, %rax +; AVX512-NEXT: addl %ecx, %eax +; AVX512-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512-NEXT: retq %cnt = call i128 @llvm.ctpop.i128(i128 %a0) %res = trunc i128 %cnt to i32 ret i32 %res @@ -29,6 +38,14 @@ define i32 @load_ctpop_i128(ptr %p0) nounwind { ; CHECK-NEXT: addl %ecx, %eax ; CHECK-NEXT: # kill: def $eax killed $eax killed $rax ; CHECK-NEXT: retq +; +; AVX512-LABEL: load_ctpop_i128: +; AVX512: # %bb.0: +; AVX512-NEXT: popcntq 8(%rdi), %rcx +; AVX512-NEXT: popcntq (%rdi), %rax +; AVX512-NEXT: addl %ecx, %eax +; AVX512-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512-NEXT: retq %a0 = load i128, ptr %p0 %cnt = call i128 @llvm.ctpop.i128(i128 %a0) %res = trunc i128 %cnt to i32 @@ -50,6 +67,48 @@ define i32 @test_ctpop_i256(i256 %a0) nounwind { ; CHECK-NEXT: addl %ecx, %eax ; CHECK-NEXT: # kill: def $eax killed $eax killed $rax ; CHECK-NEXT: retq +; +; AVX512F-LABEL: test_ctpop_i256: +; AVX512F: # %bb.0: +; AVX512F-NEXT: popcntq %rcx, %rax +; AVX512F-NEXT: popcntq %rdx, %rcx +; AVX512F-NEXT: addl %eax, %ecx +; AVX512F-NEXT: popcntq %rsi, %rdx +; AVX512F-NEXT: popcntq %rdi, %rax +; AVX512F-NEXT: addl %edx, %eax +; AVX512F-NEXT: addl %ecx, %eax +; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: test_ctpop_i256: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: popcntq %rcx, %rax +; AVX512VL-NEXT: xorl %ecx, %ecx +; AVX512VL-NEXT: popcntq %rdx, %rcx +; AVX512VL-NEXT: addl %eax, %ecx +; AVX512VL-NEXT: xorl %edx, %edx +; AVX512VL-NEXT: popcntq %rsi, %rdx +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq %rdi, %rax +; AVX512VL-NEXT: addl %edx, %eax +; AVX512VL-NEXT: addl %ecx, %eax +; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: test_ctpop_i256: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: popcntq %rcx, %rax +; AVX512POPCNT-NEXT: xorl %ecx, %ecx +; AVX512POPCNT-NEXT: popcntq %rdx, %rcx +; AVX512POPCNT-NEXT: addl %eax, %ecx +; AVX512POPCNT-NEXT: xorl %edx, %edx +; AVX512POPCNT-NEXT: popcntq %rsi, %rdx +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq %rdi, %rax +; AVX512POPCNT-NEXT: addl %edx, %eax +; AVX512POPCNT-NEXT: addl %ecx, %eax +; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512POPCNT-NEXT: retq %cnt = call i256 @llvm.ctpop.i256(i256 %a0) %res = trunc i256 %cnt to i32 ret i32 %res @@ -81,18 +140,43 @@ define i32 @load_ctpop_i256(ptr %p0) nounwind { ; AVX2-NEXT: # kill: def $eax killed $eax killed $rax ; AVX2-NEXT: retq ; -; AVX512-LABEL: load_ctpop_i256: -; AVX512: # %bb.0: -; AVX512-NEXT: popcntq 24(%rdi), %rax -; AVX512-NEXT: popcntq 16(%rdi), %rcx -; AVX512-NEXT: addl %eax, %ecx -; AVX512-NEXT: popcntq 8(%rdi), %rdx -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq (%rdi), %rax -; AVX512-NEXT: addl %edx, %eax -; AVX512-NEXT: addl %ecx, %eax -; AVX512-NEXT: # kill: def $eax killed $eax killed $rax -; AVX512-NEXT: retq +; AVX512F-LABEL: load_ctpop_i256: +; AVX512F: # %bb.0: +; AVX512F-NEXT: popcntq 24(%rdi), %rax +; AVX512F-NEXT: popcntq 16(%rdi), %rcx +; AVX512F-NEXT: addl %eax, %ecx +; AVX512F-NEXT: popcntq 8(%rdi), %rdx +; AVX512F-NEXT: popcntq (%rdi), %rax +; AVX512F-NEXT: addl %edx, %eax +; AVX512F-NEXT: addl %ecx, %eax +; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: load_ctpop_i256: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: popcntq 24(%rdi), %rax +; AVX512VL-NEXT: popcntq 16(%rdi), %rcx +; AVX512VL-NEXT: addl %eax, %ecx +; AVX512VL-NEXT: popcntq 8(%rdi), %rdx +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq (%rdi), %rax +; AVX512VL-NEXT: addl %edx, %eax +; AVX512VL-NEXT: addl %ecx, %eax +; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: load_ctpop_i256: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rax +; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rcx +; AVX512POPCNT-NEXT: addl %eax, %ecx +; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rdx +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq (%rdi), %rax +; AVX512POPCNT-NEXT: addl %edx, %eax +; AVX512POPCNT-NEXT: addl %ecx, %eax +; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512POPCNT-NEXT: retq %a0 = load i256, ptr %p0 %cnt = call i256 @llvm.ctpop.i256(i256 %a0) %res = trunc i256 %cnt to i32 @@ -124,6 +208,76 @@ define i32 @test_ctpop_i512(i512 %a0) nounwind { ; CHECK-NEXT: addl %r8d, %eax ; CHECK-NEXT: # kill: def $eax killed $eax killed $rax ; CHECK-NEXT: retq +; +; AVX512F-LABEL: test_ctpop_i512: +; AVX512F: # %bb.0: +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 +; AVX512F-NEXT: addl %eax, %r10d +; AVX512F-NEXT: popcntq %r9, %rax +; AVX512F-NEXT: popcntq %r8, %r8 +; AVX512F-NEXT: addl %eax, %r8d +; AVX512F-NEXT: addl %r10d, %r8d +; AVX512F-NEXT: popcntq %rcx, %rax +; AVX512F-NEXT: popcntq %rdx, %rcx +; AVX512F-NEXT: addl %eax, %ecx +; AVX512F-NEXT: popcntq %rsi, %rdx +; AVX512F-NEXT: popcntq %rdi, %rax +; AVX512F-NEXT: addl %edx, %eax +; AVX512F-NEXT: addl %ecx, %eax +; AVX512F-NEXT: addl %r8d, %eax +; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: test_ctpop_i512: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 +; AVX512VL-NEXT: addl %eax, %r10d +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq %r9, %rax +; AVX512VL-NEXT: popcntq %r8, %r8 +; AVX512VL-NEXT: addl %eax, %r8d +; AVX512VL-NEXT: addl %r10d, %r8d +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq %rcx, %rax +; AVX512VL-NEXT: xorl %ecx, %ecx +; AVX512VL-NEXT: popcntq %rdx, %rcx +; AVX512VL-NEXT: addl %eax, %ecx +; AVX512VL-NEXT: xorl %edx, %edx +; AVX512VL-NEXT: popcntq %rsi, %rdx +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq %rdi, %rax +; AVX512VL-NEXT: addl %edx, %eax +; AVX512VL-NEXT: addl %ecx, %eax +; AVX512VL-NEXT: addl %r8d, %eax +; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: test_ctpop_i512: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 +; AVX512POPCNT-NEXT: addl %eax, %r10d +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq %r9, %rax +; AVX512POPCNT-NEXT: popcntq %r8, %r8 +; AVX512POPCNT-NEXT: addl %eax, %r8d +; AVX512POPCNT-NEXT: addl %r10d, %r8d +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq %rcx, %rax +; AVX512POPCNT-NEXT: xorl %ecx, %ecx +; AVX512POPCNT-NEXT: popcntq %rdx, %rcx +; AVX512POPCNT-NEXT: addl %eax, %ecx +; AVX512POPCNT-NEXT: xorl %edx, %edx +; AVX512POPCNT-NEXT: popcntq %rsi, %rdx +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq %rdi, %rax +; AVX512POPCNT-NEXT: addl %edx, %eax +; AVX512POPCNT-NEXT: addl %ecx, %eax +; AVX512POPCNT-NEXT: addl %r8d, %eax +; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512POPCNT-NEXT: retq %cnt = call i512 @llvm.ctpop.i512(i512 %a0) %res = trunc i512 %cnt to i32 ret i32 %res @@ -177,29 +331,73 @@ define i32 @load_ctpop_i512(ptr %p0) nounwind { ; AVX2-NEXT: # kill: def $eax killed $eax killed $rax ; AVX2-NEXT: retq ; -; AVX512-LABEL: load_ctpop_i512: -; AVX512: # %bb.0: -; AVX512-NEXT: popcntq 56(%rdi), %rax -; AVX512-NEXT: popcntq 48(%rdi), %rcx -; AVX512-NEXT: addl %eax, %ecx -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq 40(%rdi), %rax -; AVX512-NEXT: popcntq 32(%rdi), %rdx -; AVX512-NEXT: addl %eax, %edx -; AVX512-NEXT: addl %ecx, %edx -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq 24(%rdi), %rax -; AVX512-NEXT: xorl %ecx, %ecx -; AVX512-NEXT: popcntq 16(%rdi), %rcx -; AVX512-NEXT: popcntq 8(%rdi), %rsi -; AVX512-NEXT: addl %eax, %ecx -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq (%rdi), %rax -; AVX512-NEXT: addl %esi, %eax -; AVX512-NEXT: addl %ecx, %eax -; AVX512-NEXT: addl %edx, %eax -; AVX512-NEXT: # kill: def $eax killed $eax killed $rax -; AVX512-NEXT: retq +; AVX512F-LABEL: load_ctpop_i512: +; AVX512F: # %bb.0: +; AVX512F-NEXT: popcntq 56(%rdi), %rax +; AVX512F-NEXT: popcntq 48(%rdi), %rcx +; AVX512F-NEXT: addl %eax, %ecx +; AVX512F-NEXT: popcntq 40(%rdi), %rax +; AVX512F-NEXT: popcntq 32(%rdi), %rdx +; AVX512F-NEXT: addl %eax, %edx +; AVX512F-NEXT: addl %ecx, %edx +; AVX512F-NEXT: popcntq 24(%rdi), %rcx +; AVX512F-NEXT: popcntq 16(%rdi), %rsi +; AVX512F-NEXT: popcntq 8(%rdi), %r8 +; AVX512F-NEXT: popcntq (%rdi), %rax +; AVX512F-NEXT: addl %ecx, %esi +; AVX512F-NEXT: addl %r8d, %eax +; AVX512F-NEXT: addl %esi, %eax +; AVX512F-NEXT: addl %edx, %eax +; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: load_ctpop_i512: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: popcntq 56(%rdi), %rax +; AVX512VL-NEXT: popcntq 48(%rdi), %rcx +; AVX512VL-NEXT: addl %eax, %ecx +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq 40(%rdi), %rax +; AVX512VL-NEXT: popcntq 32(%rdi), %rdx +; AVX512VL-NEXT: addl %eax, %edx +; AVX512VL-NEXT: addl %ecx, %edx +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq 24(%rdi), %rax +; AVX512VL-NEXT: xorl %ecx, %ecx +; AVX512VL-NEXT: popcntq 16(%rdi), %rcx +; AVX512VL-NEXT: popcntq 8(%rdi), %rsi +; AVX512VL-NEXT: addl %eax, %ecx +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq (%rdi), %rax +; AVX512VL-NEXT: addl %esi, %eax +; AVX512VL-NEXT: addl %ecx, %eax +; AVX512VL-NEXT: addl %edx, %eax +; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: load_ctpop_i512: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: popcntq 56(%rdi), %rax +; AVX512POPCNT-NEXT: popcntq 48(%rdi), %rcx +; AVX512POPCNT-NEXT: addl %eax, %ecx +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq 40(%rdi), %rax +; AVX512POPCNT-NEXT: popcntq 32(%rdi), %rdx +; AVX512POPCNT-NEXT: addl %eax, %edx +; AVX512POPCNT-NEXT: addl %ecx, %edx +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rax +; AVX512POPCNT-NEXT: xorl %ecx, %ecx +; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rcx +; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rsi +; AVX512POPCNT-NEXT: addl %eax, %ecx +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq (%rdi), %rax +; AVX512POPCNT-NEXT: addl %esi, %eax +; AVX512POPCNT-NEXT: addl %ecx, %eax +; AVX512POPCNT-NEXT: addl %edx, %eax +; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512POPCNT-NEXT: retq %a0 = load i512, ptr %p0 %cnt = call i512 @llvm.ctpop.i512(i512 %a0) %res = trunc i512 %cnt to i32 @@ -309,57 +507,149 @@ define i32 @test_ctpop_i1024(i1024 %a0) nounwind { ; AVX2-NEXT: popq %r14 ; AVX2-NEXT: retq ; -; AVX512-LABEL: test_ctpop_i1024: -; AVX512: # %bb.0: -; AVX512-NEXT: pushq %r14 -; AVX512-NEXT: pushq %rbx -; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %rax -; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 -; AVX512-NEXT: addl %eax, %r10d -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %rax -; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %r11 -; AVX512-NEXT: addl %eax, %r11d -; AVX512-NEXT: addl %r10d, %r11d -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %rax -; AVX512-NEXT: xorl %ebx, %ebx -; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx -; AVX512-NEXT: xorl %r14d, %r14d -; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %r14 -; AVX512-NEXT: addl %eax, %ebx -; AVX512-NEXT: xorl %r10d, %r10d -; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 -; AVX512-NEXT: addl %r14d, %r10d -; AVX512-NEXT: addl %ebx, %r10d -; AVX512-NEXT: addl %r11d, %r10d -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %rax -; AVX512-NEXT: xorl %r11d, %r11d -; AVX512-NEXT: popcntq {{[0-9]+}}(%rsp), %r11 -; AVX512-NEXT: addl %eax, %r11d -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq %r9, %rax -; AVX512-NEXT: popcntq %r8, %r8 -; AVX512-NEXT: addl %eax, %r8d -; AVX512-NEXT: addl %r11d, %r8d -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq %rcx, %rax -; AVX512-NEXT: xorl %ecx, %ecx -; AVX512-NEXT: popcntq %rdx, %rcx -; AVX512-NEXT: addl %eax, %ecx -; AVX512-NEXT: xorl %edx, %edx -; AVX512-NEXT: popcntq %rsi, %rdx -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq %rdi, %rax -; AVX512-NEXT: addl %edx, %eax -; AVX512-NEXT: addl %ecx, %eax -; AVX512-NEXT: addl %r8d, %eax -; AVX512-NEXT: addl %r10d, %eax -; AVX512-NEXT: # kill: def $eax killed $eax killed $rax -; AVX512-NEXT: popq %rbx -; AVX512-NEXT: popq %r14 -; AVX512-NEXT: retq +; AVX512F-LABEL: test_ctpop_i1024: +; AVX512F: # %bb.0: +; AVX512F-NEXT: pushq %r14 +; AVX512F-NEXT: pushq %rbx +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 +; AVX512F-NEXT: addl %eax, %r10d +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r11 +; AVX512F-NEXT: addl %eax, %r11d +; AVX512F-NEXT: addl %r10d, %r11d +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r14 +; AVX512F-NEXT: addl %eax, %ebx +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 +; AVX512F-NEXT: addl %r14d, %r10d +; AVX512F-NEXT: addl %ebx, %r10d +; AVX512F-NEXT: addl %r11d, %r10d +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512F-NEXT: popcntq {{[0-9]+}}(%rsp), %r11 +; AVX512F-NEXT: addl %eax, %r11d +; AVX512F-NEXT: popcntq %r9, %rax +; AVX512F-NEXT: popcntq %r8, %r8 +; AVX512F-NEXT: addl %eax, %r8d +; AVX512F-NEXT: addl %r11d, %r8d +; AVX512F-NEXT: popcntq %rcx, %rax +; AVX512F-NEXT: popcntq %rdx, %rcx +; AVX512F-NEXT: addl %eax, %ecx +; AVX512F-NEXT: popcntq %rsi, %rdx +; AVX512F-NEXT: popcntq %rdi, %rax +; AVX512F-NEXT: addl %edx, %eax +; AVX512F-NEXT: addl %ecx, %eax +; AVX512F-NEXT: addl %r8d, %eax +; AVX512F-NEXT: addl %r10d, %eax +; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512F-NEXT: popq %rbx +; AVX512F-NEXT: popq %r14 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: test_ctpop_i1024: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: pushq %r14 +; AVX512VL-NEXT: pushq %rbx +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 +; AVX512VL-NEXT: addl %eax, %r10d +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r11 +; AVX512VL-NEXT: addl %eax, %r11d +; AVX512VL-NEXT: addl %r10d, %r11d +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512VL-NEXT: xorl %ebx, %ebx +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx +; AVX512VL-NEXT: xorl %r14d, %r14d +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r14 +; AVX512VL-NEXT: addl %eax, %ebx +; AVX512VL-NEXT: xorl %r10d, %r10d +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 +; AVX512VL-NEXT: addl %r14d, %r10d +; AVX512VL-NEXT: addl %ebx, %r10d +; AVX512VL-NEXT: addl %r11d, %r10d +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512VL-NEXT: xorl %r11d, %r11d +; AVX512VL-NEXT: popcntq {{[0-9]+}}(%rsp), %r11 +; AVX512VL-NEXT: addl %eax, %r11d +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq %r9, %rax +; AVX512VL-NEXT: popcntq %r8, %r8 +; AVX512VL-NEXT: addl %eax, %r8d +; AVX512VL-NEXT: addl %r11d, %r8d +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq %rcx, %rax +; AVX512VL-NEXT: xorl %ecx, %ecx +; AVX512VL-NEXT: popcntq %rdx, %rcx +; AVX512VL-NEXT: addl %eax, %ecx +; AVX512VL-NEXT: xorl %edx, %edx +; AVX512VL-NEXT: popcntq %rsi, %rdx +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq %rdi, %rax +; AVX512VL-NEXT: addl %edx, %eax +; AVX512VL-NEXT: addl %ecx, %eax +; AVX512VL-NEXT: addl %r8d, %eax +; AVX512VL-NEXT: addl %r10d, %eax +; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512VL-NEXT: popq %rbx +; AVX512VL-NEXT: popq %r14 +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: test_ctpop_i1024: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: pushq %r14 +; AVX512POPCNT-NEXT: pushq %rbx +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 +; AVX512POPCNT-NEXT: addl %eax, %r10d +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r11 +; AVX512POPCNT-NEXT: addl %eax, %r11d +; AVX512POPCNT-NEXT: addl %r10d, %r11d +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512POPCNT-NEXT: xorl %ebx, %ebx +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx +; AVX512POPCNT-NEXT: xorl %r14d, %r14d +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r14 +; AVX512POPCNT-NEXT: addl %eax, %ebx +; AVX512POPCNT-NEXT: xorl %r10d, %r10d +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10 +; AVX512POPCNT-NEXT: addl %r14d, %r10d +; AVX512POPCNT-NEXT: addl %ebx, %r10d +; AVX512POPCNT-NEXT: addl %r11d, %r10d +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax +; AVX512POPCNT-NEXT: xorl %r11d, %r11d +; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r11 +; AVX512POPCNT-NEXT: addl %eax, %r11d +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq %r9, %rax +; AVX512POPCNT-NEXT: popcntq %r8, %r8 +; AVX512POPCNT-NEXT: addl %eax, %r8d +; AVX512POPCNT-NEXT: addl %r11d, %r8d +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq %rcx, %rax +; AVX512POPCNT-NEXT: xorl %ecx, %ecx +; AVX512POPCNT-NEXT: popcntq %rdx, %rcx +; AVX512POPCNT-NEXT: addl %eax, %ecx +; AVX512POPCNT-NEXT: xorl %edx, %edx +; AVX512POPCNT-NEXT: popcntq %rsi, %rdx +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq %rdi, %rax +; AVX512POPCNT-NEXT: addl %edx, %eax +; AVX512POPCNT-NEXT: addl %ecx, %eax +; AVX512POPCNT-NEXT: addl %r8d, %eax +; AVX512POPCNT-NEXT: addl %r10d, %eax +; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512POPCNT-NEXT: popq %rbx +; AVX512POPCNT-NEXT: popq %r14 +; AVX512POPCNT-NEXT: retq %cnt = call i1024 @llvm.ctpop.i1024(i1024 %a0) %res = trunc i1024 %cnt to i32 ret i32 %res @@ -460,52 +750,135 @@ define i32 @load_ctpop_i1024(ptr %p0) nounwind { ; AVX2-NEXT: # kill: def $eax killed $eax killed $rax ; AVX2-NEXT: retq ; -; AVX512-LABEL: load_ctpop_i1024: -; AVX512: # %bb.0: -; AVX512-NEXT: popcntq 120(%rdi), %rax -; AVX512-NEXT: popcntq 112(%rdi), %rcx -; AVX512-NEXT: addl %eax, %ecx -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq 104(%rdi), %rax -; AVX512-NEXT: popcntq 96(%rdi), %rdx -; AVX512-NEXT: addl %eax, %edx -; AVX512-NEXT: addl %ecx, %edx -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq 88(%rdi), %rax -; AVX512-NEXT: popcntq 80(%rdi), %rsi -; AVX512-NEXT: popcntq 72(%rdi), %r8 -; AVX512-NEXT: addl %eax, %esi -; AVX512-NEXT: xorl %ecx, %ecx -; AVX512-NEXT: popcntq 64(%rdi), %rcx -; AVX512-NEXT: addl %r8d, %ecx -; AVX512-NEXT: addl %esi, %ecx -; AVX512-NEXT: addl %edx, %ecx -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq 56(%rdi), %rax -; AVX512-NEXT: xorl %edx, %edx -; AVX512-NEXT: popcntq 48(%rdi), %rdx -; AVX512-NEXT: xorl %esi, %esi -; AVX512-NEXT: popcntq 40(%rdi), %rsi -; AVX512-NEXT: addl %eax, %edx -; AVX512-NEXT: xorl %r8d, %r8d -; AVX512-NEXT: popcntq 32(%rdi), %r8 -; AVX512-NEXT: addl %esi, %r8d -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq 24(%rdi), %rax -; AVX512-NEXT: addl %edx, %r8d -; AVX512-NEXT: xorl %edx, %edx -; AVX512-NEXT: popcntq 16(%rdi), %rdx -; AVX512-NEXT: addl %eax, %edx -; AVX512-NEXT: xorl %esi, %esi -; AVX512-NEXT: popcntq 8(%rdi), %rsi -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: popcntq (%rdi), %rax -; AVX512-NEXT: addl %esi, %eax -; AVX512-NEXT: addl %edx, %eax -; AVX512-NEXT: addl %r8d, %eax -; AVX512-NEXT: addl %ecx, %eax -; AVX512-NEXT: # kill: def $eax killed $eax killed $rax -; AVX512-NEXT: retq +; AVX512F-LABEL: load_ctpop_i1024: +; AVX512F: # %bb.0: +; AVX512F-NEXT: popcntq 120(%rdi), %rax +; AVX512F-NEXT: popcntq 112(%rdi), %rcx +; AVX512F-NEXT: addl %eax, %ecx +; AVX512F-NEXT: popcntq 104(%rdi), %rax +; AVX512F-NEXT: popcntq 96(%rdi), %rdx +; AVX512F-NEXT: addl %eax, %edx +; AVX512F-NEXT: addl %ecx, %edx +; AVX512F-NEXT: popcntq 88(%rdi), %rax +; AVX512F-NEXT: popcntq 80(%rdi), %rsi +; AVX512F-NEXT: popcntq 72(%rdi), %r8 +; AVX512F-NEXT: popcntq 64(%rdi), %rcx +; AVX512F-NEXT: addl %eax, %esi +; AVX512F-NEXT: addl %r8d, %ecx +; AVX512F-NEXT: addl %esi, %ecx +; AVX512F-NEXT: addl %edx, %ecx +; AVX512F-NEXT: popcntq 56(%rdi), %rax +; AVX512F-NEXT: popcntq 48(%rdi), %rdx +; AVX512F-NEXT: popcntq 40(%rdi), %rsi +; AVX512F-NEXT: popcntq 32(%rdi), %r8 +; AVX512F-NEXT: addl %eax, %edx +; AVX512F-NEXT: addl %esi, %r8d +; AVX512F-NEXT: popcntq 24(%rdi), %rax +; AVX512F-NEXT: addl %edx, %r8d +; AVX512F-NEXT: popcntq 16(%rdi), %rdx +; AVX512F-NEXT: addl %eax, %edx +; AVX512F-NEXT: popcntq 8(%rdi), %rsi +; AVX512F-NEXT: popcntq (%rdi), %rax +; AVX512F-NEXT: addl %esi, %eax +; AVX512F-NEXT: addl %edx, %eax +; AVX512F-NEXT: addl %r8d, %eax +; AVX512F-NEXT: addl %ecx, %eax +; AVX512F-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: load_ctpop_i1024: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: popcntq 120(%rdi), %rax +; AVX512VL-NEXT: popcntq 112(%rdi), %rcx +; AVX512VL-NEXT: addl %eax, %ecx +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq 104(%rdi), %rax +; AVX512VL-NEXT: popcntq 96(%rdi), %rdx +; AVX512VL-NEXT: addl %eax, %edx +; AVX512VL-NEXT: addl %ecx, %edx +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq 88(%rdi), %rax +; AVX512VL-NEXT: popcntq 80(%rdi), %rsi +; AVX512VL-NEXT: popcntq 72(%rdi), %r8 +; AVX512VL-NEXT: addl %eax, %esi +; AVX512VL-NEXT: xorl %ecx, %ecx +; AVX512VL-NEXT: popcntq 64(%rdi), %rcx +; AVX512VL-NEXT: addl %r8d, %ecx +; AVX512VL-NEXT: addl %esi, %ecx +; AVX512VL-NEXT: addl %edx, %ecx +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq 56(%rdi), %rax +; AVX512VL-NEXT: xorl %edx, %edx +; AVX512VL-NEXT: popcntq 48(%rdi), %rdx +; AVX512VL-NEXT: xorl %esi, %esi +; AVX512VL-NEXT: popcntq 40(%rdi), %rsi +; AVX512VL-NEXT: addl %eax, %edx +; AVX512VL-NEXT: xorl %r8d, %r8d +; AVX512VL-NEXT: popcntq 32(%rdi), %r8 +; AVX512VL-NEXT: addl %esi, %r8d +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq 24(%rdi), %rax +; AVX512VL-NEXT: addl %edx, %r8d +; AVX512VL-NEXT: xorl %edx, %edx +; AVX512VL-NEXT: popcntq 16(%rdi), %rdx +; AVX512VL-NEXT: addl %eax, %edx +; AVX512VL-NEXT: xorl %esi, %esi +; AVX512VL-NEXT: popcntq 8(%rdi), %rsi +; AVX512VL-NEXT: xorl %eax, %eax +; AVX512VL-NEXT: popcntq (%rdi), %rax +; AVX512VL-NEXT: addl %esi, %eax +; AVX512VL-NEXT: addl %edx, %eax +; AVX512VL-NEXT: addl %r8d, %eax +; AVX512VL-NEXT: addl %ecx, %eax +; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: load_ctpop_i1024: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: popcntq 120(%rdi), %rax +; AVX512POPCNT-NEXT: popcntq 112(%rdi), %rcx +; AVX512POPCNT-NEXT: addl %eax, %ecx +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq 104(%rdi), %rax +; AVX512POPCNT-NEXT: popcntq 96(%rdi), %rdx +; AVX512POPCNT-NEXT: addl %eax, %edx +; AVX512POPCNT-NEXT: addl %ecx, %edx +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq 88(%rdi), %rax +; AVX512POPCNT-NEXT: popcntq 80(%rdi), %rsi +; AVX512POPCNT-NEXT: popcntq 72(%rdi), %r8 +; AVX512POPCNT-NEXT: addl %eax, %esi +; AVX512POPCNT-NEXT: xorl %ecx, %ecx +; AVX512POPCNT-NEXT: popcntq 64(%rdi), %rcx +; AVX512POPCNT-NEXT: addl %r8d, %ecx +; AVX512POPCNT-NEXT: addl %esi, %ecx +; AVX512POPCNT-NEXT: addl %edx, %ecx +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq 56(%rdi), %rax +; AVX512POPCNT-NEXT: xorl %edx, %edx +; AVX512POPCNT-NEXT: popcntq 48(%rdi), %rdx +; AVX512POPCNT-NEXT: xorl %esi, %esi +; AVX512POPCNT-NEXT: popcntq 40(%rdi), %rsi +; AVX512POPCNT-NEXT: addl %eax, %edx +; AVX512POPCNT-NEXT: xorl %r8d, %r8d +; AVX512POPCNT-NEXT: popcntq 32(%rdi), %r8 +; AVX512POPCNT-NEXT: addl %esi, %r8d +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rax +; AVX512POPCNT-NEXT: addl %edx, %r8d +; AVX512POPCNT-NEXT: xorl %edx, %edx +; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rdx +; AVX512POPCNT-NEXT: addl %eax, %edx +; AVX512POPCNT-NEXT: xorl %esi, %esi +; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rsi +; AVX512POPCNT-NEXT: xorl %eax, %eax +; AVX512POPCNT-NEXT: popcntq (%rdi), %rax +; AVX512POPCNT-NEXT: addl %esi, %eax +; AVX512POPCNT-NEXT: addl %edx, %eax +; AVX512POPCNT-NEXT: addl %r8d, %eax +; AVX512POPCNT-NEXT: addl %ecx, %eax +; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax +; AVX512POPCNT-NEXT: retq %a0 = load i1024, ptr %p0 %cnt = call i1024 @llvm.ctpop.i1024(i1024 %a0) %res = trunc i1024 %cnt to i32 @@ -710,17 +1083,42 @@ define i32 @load_ctlz_i256(ptr %p0) nounwind { ; AVX2-NEXT: # kill: def $eax killed $eax killed $rax ; AVX2-NEXT: retq ; -; AVX512-LABEL: load_ctlz_i256: -; AVX512: # %bb.0: -; AVX512-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0] -; AVX512-NEXT: vplzcntq %ymm0, %ymm1 -; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 -; AVX512-NEXT: vptestmq %ymm0, %ymm0, %k1 -; AVX512-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256] -; AVX512-NEXT: vpcompressq %ymm1, %ymm0 {%k1} -; AVX512-NEXT: vmovd %xmm0, %eax -; AVX512-NEXT: vzeroupper -; AVX512-NEXT: retq +; AVX512F-LABEL: load_ctlz_i256: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256] +; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = mem[3,2,1,0] +; AVX512F-NEXT: vplzcntq %zmm1, %zmm2 +; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 +; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0 +; AVX512F-NEXT: kshiftlw $12, %k0, %k0 +; AVX512F-NEXT: kshiftrw $12, %k0, %k1 +; AVX512F-NEXT: vpcompressq %zmm2, %zmm0 {%k1} +; AVX512F-NEXT: vmovd %xmm0, %eax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: load_ctlz_i256: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0] +; AVX512VL-NEXT: vplzcntq %ymm0, %ymm1 +; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 +; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256] +; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: load_ctlz_i256: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0] +; AVX512POPCNT-NEXT: vplzcntq %ymm0, %ymm1 +; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 +; AVX512POPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1 +; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256] +; AVX512POPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} +; AVX512POPCNT-NEXT: vmovd %xmm0, %eax +; AVX512POPCNT-NEXT: vzeroupper +; AVX512POPCNT-NEXT: retq %a0 = load i256, ptr %p0 %cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 0) %res = trunc i256 %cnt to i32 @@ -834,29 +1232,76 @@ define i32 @test_ctlz_i512(i512 %a0) nounwind { ; AVX2-NEXT: popq %r15 ; AVX2-NEXT: retq ; -; AVX512-LABEL: test_ctlz_i512: -; AVX512: # %bb.0: -; AVX512-NEXT: vmovq %rdi, %xmm0 -; AVX512-NEXT: vmovq %rsi, %xmm1 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512-NEXT: vmovq %rdx, %xmm1 -; AVX512-NEXT: vmovq %rcx, %xmm2 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 -; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] -; AVX512-NEXT: vmovq %r8, %xmm2 -; AVX512-NEXT: vmovq %r9, %xmm3 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 -; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; AVX512-NEXT: vplzcntq %zmm0, %zmm1 -; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 -; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1 -; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] -; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} -; AVX512-NEXT: vmovd %xmm0, %eax -; AVX512-NEXT: vzeroupper -; AVX512-NEXT: retq +; AVX512F-LABEL: test_ctlz_i512: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vmovq %rdi, %xmm0 +; AVX512F-NEXT: vmovq %rsi, %xmm1 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512F-NEXT: vmovq %rdx, %xmm1 +; AVX512F-NEXT: vmovq %rcx, %xmm2 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512F-NEXT: vmovq %r8, %xmm2 +; AVX512F-NEXT: vmovq %r9, %xmm3 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512] +; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1} +; AVX512F-NEXT: vmovd %xmm1, %eax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: test_ctlz_i512: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovq %rdi, %xmm0 +; AVX512VL-NEXT: vmovq %rsi, %xmm1 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512VL-NEXT: vmovq %rdx, %xmm1 +; AVX512VL-NEXT: vmovq %rcx, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512VL-NEXT: vmovq %r8, %xmm2 +; AVX512VL-NEXT: vmovq %r9, %xmm3 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: test_ctlz_i512: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: vmovq %rdi, %xmm0 +; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1 +; AVX512POPCNT-NEXT: vmovq %rcx, %xmm2 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512POPCNT-NEXT: vmovq %r8, %xmm2 +; AVX512POPCNT-NEXT: vmovq %r9, %xmm3 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512POPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 +; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512POPCNT-NEXT: vmovd %xmm0, %eax +; AVX512POPCNT-NEXT: vzeroupper +; AVX512POPCNT-NEXT: retq %cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 0) %res = trunc i512 %cnt to i32 ret i32 %res @@ -978,18 +1423,43 @@ define i32 @load_ctlz_i512(ptr %p0) nounwind { ; AVX2-NEXT: popq %r15 ; AVX2-NEXT: retq ; -; AVX512-LABEL: load_ctlz_i512: -; AVX512: # %bb.0: -; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] -; AVX512-NEXT: vpermq (%rdi), %zmm0, %zmm0 -; AVX512-NEXT: vplzcntq %zmm0, %zmm1 -; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 -; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1 -; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] -; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} -; AVX512-NEXT: vmovd %xmm0, %eax -; AVX512-NEXT: vzeroupper -; AVX512-NEXT: retq +; AVX512F-LABEL: load_ctlz_i512: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512] +; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1} +; AVX512F-NEXT: vmovd %xmm1, %eax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: load_ctlz_i512: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: load_ctlz_i512: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 +; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512POPCNT-NEXT: vmovd %xmm0, %eax +; AVX512POPCNT-NEXT: vzeroupper +; AVX512POPCNT-NEXT: retq %a0 = load i512, ptr %p0 %cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 0) %res = trunc i512 %cnt to i32 @@ -1247,54 +1717,151 @@ define i32 @test_ctlz_i1024(i1024 %a0) nounwind { ; AVX2-NEXT: popq %rbp ; AVX2-NEXT: retq ; -; AVX512-LABEL: test_ctlz_i1024: -; AVX512: # %bb.0: -; AVX512-NEXT: pushq %r14 -; AVX512-NEXT: pushq %rbx -; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx -; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10 -; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11 -; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r14 -; AVX512-NEXT: vmovq %rdi, %xmm0 -; AVX512-NEXT: vmovq %rsi, %xmm1 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512-NEXT: vmovq %rdx, %xmm1 -; AVX512-NEXT: vmovq %rcx, %xmm2 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 -; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] -; AVX512-NEXT: vmovq %r8, %xmm2 -; AVX512-NEXT: vmovq %r9, %xmm3 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 -; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; AVX512-NEXT: vplzcntq %zmm0, %zmm1 -; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448] -; AVX512-NEXT: vpaddq %zmm2, %zmm1, %zmm1 -; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1 -; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] -; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} -; AVX512-NEXT: vmovd %xmm0, %ecx -; AVX512-NEXT: addl $512, %ecx # imm = 0x200 -; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] -; AVX512-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0 -; AVX512-NEXT: vplzcntq %zmm0, %zmm1 -; AVX512-NEXT: vpaddq %zmm2, %zmm1, %zmm1 -; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1 -; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} -; AVX512-NEXT: vmovd %xmm0, %eax -; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r14 -; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r11 -; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %rbx -; AVX512-NEXT: orq %r14, %r11 -; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r10 -; AVX512-NEXT: orq %rbx, %r10 -; AVX512-NEXT: orq %r11, %r10 -; AVX512-NEXT: cmovel %ecx, %eax -; AVX512-NEXT: popq %rbx -; AVX512-NEXT: popq %r14 -; AVX512-NEXT: vzeroupper -; AVX512-NEXT: retq +; AVX512F-LABEL: test_ctlz_i1024: +; AVX512F: # %bb.0: +; AVX512F-NEXT: pushq %r14 +; AVX512F-NEXT: pushq %rbx +; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rbx +; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10 +; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r11 +; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r14 +; AVX512F-NEXT: vmovq %rdi, %xmm0 +; AVX512F-NEXT: vmovq %rsi, %xmm1 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512F-NEXT: vmovq %rdx, %xmm1 +; AVX512F-NEXT: vmovq %rcx, %xmm2 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1] +; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512F-NEXT: vmovq %r8, %xmm1 +; AVX512F-NEXT: vmovq %r9, %xmm3 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0] +; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 +; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512F-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448] +; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512F-NEXT: vmovd %xmm0, %ecx +; AVX512F-NEXT: addl $512, %ecx # imm = 0x200 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512F-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0 +; AVX512F-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r14 +; AVX512F-NEXT: vmovd %xmm0, %eax +; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r11 +; AVX512F-NEXT: orq %r14, %r11 +; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rbx +; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r10 +; AVX512F-NEXT: orq %rbx, %r10 +; AVX512F-NEXT: orq %r11, %r10 +; AVX512F-NEXT: cmovel %ecx, %eax +; AVX512F-NEXT: popq %rbx +; AVX512F-NEXT: popq %r14 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: test_ctlz_i1024: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: pushq %r14 +; AVX512VL-NEXT: pushq %rbx +; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rbx +; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10 +; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r11 +; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r14 +; AVX512VL-NEXT: vmovq %rdi, %xmm0 +; AVX512VL-NEXT: vmovq %rsi, %xmm1 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512VL-NEXT: vmovq %rdx, %xmm1 +; AVX512VL-NEXT: vmovq %rcx, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512VL-NEXT: vmovq %r8, %xmm2 +; AVX512VL-NEXT: vmovq %r9, %xmm3 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448] +; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: vmovd %xmm0, %ecx +; AVX512VL-NEXT: addl $512, %ecx # imm = 0x200 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512VL-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r14 +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r11 +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rbx +; AVX512VL-NEXT: orq %r14, %r11 +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r10 +; AVX512VL-NEXT: orq %rbx, %r10 +; AVX512VL-NEXT: orq %r11, %r10 +; AVX512VL-NEXT: cmovel %ecx, %eax +; AVX512VL-NEXT: popq %rbx +; AVX512VL-NEXT: popq %r14 +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: test_ctlz_i1024: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: pushq %r14 +; AVX512POPCNT-NEXT: pushq %rbx +; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %rbx +; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r10 +; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r11 +; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r14 +; AVX512POPCNT-NEXT: vmovq %rdi, %xmm0 +; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1 +; AVX512POPCNT-NEXT: vmovq %rcx, %xmm2 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512POPCNT-NEXT: vmovq %r8, %xmm2 +; AVX512POPCNT-NEXT: vmovq %r9, %xmm3 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512POPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448] +; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512POPCNT-NEXT: vmovd %xmm0, %ecx +; AVX512POPCNT-NEXT: addl $512, %ecx # imm = 0x200 +; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512POPCNT-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0 +; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512POPCNT-NEXT: vmovd %xmm0, %eax +; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r14 +; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r11 +; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rbx +; AVX512POPCNT-NEXT: orq %r14, %r11 +; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r10 +; AVX512POPCNT-NEXT: orq %rbx, %r10 +; AVX512POPCNT-NEXT: orq %r11, %r10 +; AVX512POPCNT-NEXT: cmovel %ecx, %eax +; AVX512POPCNT-NEXT: popq %rbx +; AVX512POPCNT-NEXT: popq %r14 +; AVX512POPCNT-NEXT: vzeroupper +; AVX512POPCNT-NEXT: retq %cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 0) %res = trunc i1024 %cnt to i32 ret i32 %res @@ -1560,38 +2127,103 @@ define i32 @load_ctlz_i1024(ptr %p0) nounwind { ; AVX2-NEXT: popq %rbp ; AVX2-NEXT: retq ; -; AVX512-LABEL: load_ctlz_i1024: -; AVX512: # %bb.0: -; AVX512-NEXT: movq 80(%rdi), %rsi -; AVX512-NEXT: movq 64(%rdi), %rcx -; AVX512-NEXT: movq 72(%rdi), %rdx -; AVX512-NEXT: movq 88(%rdi), %r8 -; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] -; AVX512-NEXT: vpermq 64(%rdi), %zmm0, %zmm1 -; AVX512-NEXT: vplzcntq %zmm1, %zmm2 -; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448] -; AVX512-NEXT: vpaddq %zmm3, %zmm2, %zmm2 -; AVX512-NEXT: vptestmq %zmm1, %zmm1, %k1 -; AVX512-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z} -; AVX512-NEXT: vmovd %xmm1, %r9d -; AVX512-NEXT: vpermq (%rdi), %zmm0, %zmm0 -; AVX512-NEXT: vplzcntq %zmm0, %zmm1 -; AVX512-NEXT: vpaddq %zmm3, %zmm1, %zmm1 -; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1 -; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] -; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} -; AVX512-NEXT: vmovd %xmm0, %eax -; AVX512-NEXT: addl $512, %eax # imm = 0x200 -; AVX512-NEXT: orq 120(%rdi), %r8 -; AVX512-NEXT: orq 104(%rdi), %rdx -; AVX512-NEXT: orq 112(%rdi), %rsi -; AVX512-NEXT: orq %r8, %rdx -; AVX512-NEXT: orq 96(%rdi), %rcx -; AVX512-NEXT: orq %rsi, %rcx -; AVX512-NEXT: orq %rdx, %rcx -; AVX512-NEXT: cmovnel %r9d, %eax -; AVX512-NEXT: vzeroupper -; AVX512-NEXT: retq +; AVX512F-LABEL: load_ctlz_i1024: +; AVX512F: # %bb.0: +; AVX512F-NEXT: movq 80(%rdi), %rsi +; AVX512F-NEXT: movq 64(%rdi), %rcx +; AVX512F-NEXT: movq 72(%rdi), %rdx +; AVX512F-NEXT: movq 88(%rdi), %r8 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512F-NEXT: vpermq 64(%rdi), %zmm0, %zmm1 +; AVX512F-NEXT: vplzcntq %zmm1, %zmm2 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448] +; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2 +; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z} +; AVX512F-NEXT: vmovd %xmm1, %r9d +; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512F-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512F-NEXT: vpaddq %zmm3, %zmm1, %zmm1 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512F-NEXT: vmovd %xmm0, %eax +; AVX512F-NEXT: orq 120(%rdi), %r8 +; AVX512F-NEXT: addl $512, %eax # imm = 0x200 +; AVX512F-NEXT: orq 104(%rdi), %rdx +; AVX512F-NEXT: orq %r8, %rdx +; AVX512F-NEXT: orq 112(%rdi), %rsi +; AVX512F-NEXT: orq 96(%rdi), %rcx +; AVX512F-NEXT: orq %rsi, %rcx +; AVX512F-NEXT: orq %rdx, %rcx +; AVX512F-NEXT: cmovnel %r9d, %eax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: load_ctlz_i1024: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: movq 80(%rdi), %rsi +; AVX512VL-NEXT: movq 64(%rdi), %rcx +; AVX512VL-NEXT: movq 72(%rdi), %rdx +; AVX512VL-NEXT: movq 88(%rdi), %r8 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512VL-NEXT: vpermq 64(%rdi), %zmm0, %zmm1 +; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448] +; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2 +; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm1, %r9d +; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512VL-NEXT: vpaddq %zmm3, %zmm1, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: addl $512, %eax # imm = 0x200 +; AVX512VL-NEXT: orq 120(%rdi), %r8 +; AVX512VL-NEXT: orq 104(%rdi), %rdx +; AVX512VL-NEXT: orq 112(%rdi), %rsi +; AVX512VL-NEXT: orq %r8, %rdx +; AVX512VL-NEXT: orq 96(%rdi), %rcx +; AVX512VL-NEXT: orq %rsi, %rcx +; AVX512VL-NEXT: orq %rdx, %rcx +; AVX512VL-NEXT: cmovnel %r9d, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: load_ctlz_i1024: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: movq 80(%rdi), %rsi +; AVX512POPCNT-NEXT: movq 64(%rdi), %rcx +; AVX512POPCNT-NEXT: movq 72(%rdi), %rdx +; AVX512POPCNT-NEXT: movq 88(%rdi), %r8 +; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512POPCNT-NEXT: vpermq 64(%rdi), %zmm0, %zmm1 +; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2 +; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448] +; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2 +; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z} +; AVX512POPCNT-NEXT: vmovd %xmm1, %r9d +; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm1, %zmm1 +; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512POPCNT-NEXT: vmovd %xmm0, %eax +; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200 +; AVX512POPCNT-NEXT: orq 120(%rdi), %r8 +; AVX512POPCNT-NEXT: orq 104(%rdi), %rdx +; AVX512POPCNT-NEXT: orq 112(%rdi), %rsi +; AVX512POPCNT-NEXT: orq %r8, %rdx +; AVX512POPCNT-NEXT: orq 96(%rdi), %rcx +; AVX512POPCNT-NEXT: orq %rsi, %rcx +; AVX512POPCNT-NEXT: orq %rdx, %rcx +; AVX512POPCNT-NEXT: cmovnel %r9d, %eax +; AVX512POPCNT-NEXT: vzeroupper +; AVX512POPCNT-NEXT: retq %a0 = load i1024, ptr %p0 %cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 0) %res = trunc i1024 %cnt to i32 @@ -1792,16 +2424,39 @@ define i32 @load_ctlz_undef_i256(ptr %p0) nounwind { ; AVX2-NEXT: # kill: def $eax killed $eax killed $rax ; AVX2-NEXT: retq ; -; AVX512-LABEL: load_ctlz_undef_i256: -; AVX512: # %bb.0: -; AVX512-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0] -; AVX512-NEXT: vptestmq %ymm0, %ymm0, %k1 -; AVX512-NEXT: vplzcntq %ymm0, %ymm0 -; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 -; AVX512-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z} -; AVX512-NEXT: vmovd %xmm0, %eax -; AVX512-NEXT: vzeroupper -; AVX512-NEXT: retq +; AVX512F-LABEL: load_ctlz_undef_i256: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0] +; AVX512F-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kshiftlw $12, %k0, %k0 +; AVX512F-NEXT: kshiftrw $12, %k0, %k1 +; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512F-NEXT: vmovd %xmm0, %eax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: load_ctlz_undef_i256: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0] +; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1 +; AVX512VL-NEXT: vplzcntq %ymm0, %ymm0 +; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 +; AVX512VL-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: load_ctlz_undef_i256: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0] +; AVX512POPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1 +; AVX512POPCNT-NEXT: vplzcntq %ymm0, %ymm0 +; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 +; AVX512POPCNT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z} +; AVX512POPCNT-NEXT: vmovd %xmm0, %eax +; AVX512POPCNT-NEXT: vzeroupper +; AVX512POPCNT-NEXT: retq %a0 = load i256, ptr %p0 %cnt = call i256 @llvm.ctlz.i256(i256 %a0, i1 -1) %res = trunc i256 %cnt to i32 @@ -1914,28 +2569,73 @@ define i32 @test_ctlz_undef_i512(i512 %a0) nounwind { ; AVX2-NEXT: popq %r15 ; AVX2-NEXT: retq ; -; AVX512-LABEL: test_ctlz_undef_i512: -; AVX512: # %bb.0: -; AVX512-NEXT: vmovq %rdi, %xmm0 -; AVX512-NEXT: vmovq %rsi, %xmm1 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512-NEXT: vmovq %rdx, %xmm1 -; AVX512-NEXT: vmovq %rcx, %xmm2 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 -; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] -; AVX512-NEXT: vmovq %r8, %xmm2 -; AVX512-NEXT: vmovq %r9, %xmm3 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 -; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1 -; AVX512-NEXT: vplzcntq %zmm0, %zmm0 -; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 -; AVX512-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} -; AVX512-NEXT: vmovd %xmm0, %eax -; AVX512-NEXT: vzeroupper -; AVX512-NEXT: retq +; AVX512F-LABEL: test_ctlz_undef_i512: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vmovq %rdi, %xmm0 +; AVX512F-NEXT: vmovq %rsi, %xmm1 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512F-NEXT: vmovq %rdx, %xmm1 +; AVX512F-NEXT: vmovq %rcx, %xmm2 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1] +; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512F-NEXT: vmovq %r8, %xmm1 +; AVX512F-NEXT: vmovq %r9, %xmm3 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0] +; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1 +; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} +; AVX512F-NEXT: vmovd %xmm0, %eax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: test_ctlz_undef_i512: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovq %rdi, %xmm0 +; AVX512VL-NEXT: vmovq %rsi, %xmm1 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512VL-NEXT: vmovq %rdx, %xmm1 +; AVX512VL-NEXT: vmovq %rcx, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512VL-NEXT: vmovq %r8, %xmm2 +; AVX512VL-NEXT: vmovq %r9, %xmm3 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 +; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: test_ctlz_undef_i512: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: vmovq %rdi, %xmm0 +; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1 +; AVX512POPCNT-NEXT: vmovq %rcx, %xmm2 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512POPCNT-NEXT: vmovq %r8, %xmm2 +; AVX512POPCNT-NEXT: vmovq %r9, %xmm3 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512POPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 +; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} +; AVX512POPCNT-NEXT: vmovd %xmm0, %eax +; AVX512POPCNT-NEXT: vzeroupper +; AVX512POPCNT-NEXT: retq %cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 -1) %res = trunc i512 %cnt to i32 ret i32 %res @@ -2056,17 +2756,40 @@ define i32 @load_ctlz_undef_i512(ptr %p0) nounwind { ; AVX2-NEXT: popq %r15 ; AVX2-NEXT: retq ; -; AVX512-LABEL: load_ctlz_undef_i512: -; AVX512: # %bb.0: -; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] -; AVX512-NEXT: vpermq (%rdi), %zmm0, %zmm0 -; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1 -; AVX512-NEXT: vplzcntq %zmm0, %zmm0 -; AVX512-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 -; AVX512-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} -; AVX512-NEXT: vmovd %xmm0, %eax -; AVX512-NEXT: vzeroupper -; AVX512-NEXT: retq +; AVX512F-LABEL: load_ctlz_undef_i512: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 +; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} +; AVX512F-NEXT: vmovd %xmm0, %eax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: load_ctlz_undef_i512: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 +; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: load_ctlz_undef_i512: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 +; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} +; AVX512POPCNT-NEXT: vmovd %xmm0, %eax +; AVX512POPCNT-NEXT: vzeroupper +; AVX512POPCNT-NEXT: retq %a0 = load i512, ptr %p0 %cnt = call i512 @llvm.ctlz.i512(i512 %a0, i1 -1) %res = trunc i512 %cnt to i32 @@ -2326,53 +3049,148 @@ define i32 @test_ctlz_undef_i1024(i1024 %a0) nounwind { ; AVX2-NEXT: popq %rbp ; AVX2-NEXT: retq ; -; AVX512-LABEL: test_ctlz_undef_i1024: -; AVX512: # %bb.0: -; AVX512-NEXT: pushq %r14 -; AVX512-NEXT: pushq %rbx -; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx -; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10 -; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11 -; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r14 -; AVX512-NEXT: vmovq %rdi, %xmm0 -; AVX512-NEXT: vmovq %rsi, %xmm1 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] -; AVX512-NEXT: vmovq %rdx, %xmm1 -; AVX512-NEXT: vmovq %rcx, %xmm2 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 -; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] -; AVX512-NEXT: vmovq %r8, %xmm2 -; AVX512-NEXT: vmovq %r9, %xmm3 -; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 -; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; AVX512-NEXT: vplzcntq %zmm0, %zmm1 -; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448] -; AVX512-NEXT: vpaddq %zmm2, %zmm1, %zmm1 -; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1 -; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} -; AVX512-NEXT: vmovd %xmm0, %ecx -; AVX512-NEXT: addl $512, %ecx # imm = 0x200 -; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] -; AVX512-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0 -; AVX512-NEXT: vplzcntq %zmm0, %zmm1 -; AVX512-NEXT: vpaddq %zmm2, %zmm1, %zmm1 -; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1 -; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} -; AVX512-NEXT: vmovd %xmm0, %eax -; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r14 -; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r11 -; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %rbx -; AVX512-NEXT: orq %r14, %r11 -; AVX512-NEXT: orq {{[0-9]+}}(%rsp), %r10 -; AVX512-NEXT: orq %rbx, %r10 -; AVX512-NEXT: orq %r11, %r10 -; AVX512-NEXT: cmovel %ecx, %eax -; AVX512-NEXT: popq %rbx -; AVX512-NEXT: popq %r14 -; AVX512-NEXT: vzeroupper -; AVX512-NEXT: retq +; AVX512F-LABEL: test_ctlz_undef_i1024: +; AVX512F: # %bb.0: +; AVX512F-NEXT: pushq %r14 +; AVX512F-NEXT: pushq %rbx +; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rbx +; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10 +; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r11 +; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r14 +; AVX512F-NEXT: vmovq %rdi, %xmm0 +; AVX512F-NEXT: vmovq %rsi, %xmm1 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512F-NEXT: vmovq %rdx, %xmm1 +; AVX512F-NEXT: vmovq %rcx, %xmm2 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512F-NEXT: vmovq %r8, %xmm2 +; AVX512F-NEXT: vmovq %r9, %xmm3 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512F-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448] +; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512F-NEXT: vmovd %xmm0, %ecx +; AVX512F-NEXT: addl $512, %ecx # imm = 0x200 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512F-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0 +; AVX512F-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r14 +; AVX512F-NEXT: vmovd %xmm0, %eax +; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r11 +; AVX512F-NEXT: orq %r14, %r11 +; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rbx +; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r10 +; AVX512F-NEXT: orq %rbx, %r10 +; AVX512F-NEXT: orq %r11, %r10 +; AVX512F-NEXT: cmovel %ecx, %eax +; AVX512F-NEXT: popq %rbx +; AVX512F-NEXT: popq %r14 +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: test_ctlz_undef_i1024: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: pushq %r14 +; AVX512VL-NEXT: pushq %rbx +; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rbx +; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10 +; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r11 +; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r14 +; AVX512VL-NEXT: vmovq %rdi, %xmm0 +; AVX512VL-NEXT: vmovq %rsi, %xmm1 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512VL-NEXT: vmovq %rdx, %xmm1 +; AVX512VL-NEXT: vmovq %rcx, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512VL-NEXT: vmovq %r8, %xmm2 +; AVX512VL-NEXT: vmovq %r9, %xmm3 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448] +; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %ecx +; AVX512VL-NEXT: addl $512, %ecx # imm = 0x200 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512VL-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r14 +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r11 +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rbx +; AVX512VL-NEXT: orq %r14, %r11 +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r10 +; AVX512VL-NEXT: orq %rbx, %r10 +; AVX512VL-NEXT: orq %r11, %r10 +; AVX512VL-NEXT: cmovel %ecx, %eax +; AVX512VL-NEXT: popq %rbx +; AVX512VL-NEXT: popq %r14 +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: test_ctlz_undef_i1024: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: pushq %r14 +; AVX512POPCNT-NEXT: pushq %rbx +; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %rbx +; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r10 +; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r11 +; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r14 +; AVX512POPCNT-NEXT: vmovq %rdi, %xmm0 +; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1 +; AVX512POPCNT-NEXT: vmovq %rcx, %xmm2 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512POPCNT-NEXT: vmovq %r8, %xmm2 +; AVX512POPCNT-NEXT: vmovq %r9, %xmm3 +; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512POPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 +; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448] +; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512POPCNT-NEXT: vmovd %xmm0, %ecx +; AVX512POPCNT-NEXT: addl $512, %ecx # imm = 0x200 +; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512POPCNT-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0 +; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1 +; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512POPCNT-NEXT: vmovd %xmm0, %eax +; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r14 +; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r11 +; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rbx +; AVX512POPCNT-NEXT: orq %r14, %r11 +; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r10 +; AVX512POPCNT-NEXT: orq %rbx, %r10 +; AVX512POPCNT-NEXT: orq %r11, %r10 +; AVX512POPCNT-NEXT: cmovel %ecx, %eax +; AVX512POPCNT-NEXT: popq %rbx +; AVX512POPCNT-NEXT: popq %r14 +; AVX512POPCNT-NEXT: vzeroupper +; AVX512POPCNT-NEXT: retq %cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 -1) %res = trunc i1024 %cnt to i32 ret i32 %res @@ -2638,37 +3456,100 @@ define i32 @load_ctlz_undef_i1024(ptr %p0) nounwind { ; AVX2-NEXT: popq %rbp ; AVX2-NEXT: retq ; -; AVX512-LABEL: load_ctlz_undef_i1024: -; AVX512: # %bb.0: -; AVX512-NEXT: movq 80(%rdi), %rsi -; AVX512-NEXT: movq 64(%rdi), %rcx -; AVX512-NEXT: movq 72(%rdi), %rdx -; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] -; AVX512-NEXT: vpermq 64(%rdi), %zmm0, %zmm1 -; AVX512-NEXT: movq 88(%rdi), %r8 -; AVX512-NEXT: vplzcntq %zmm1, %zmm2 -; AVX512-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448] -; AVX512-NEXT: vpaddq %zmm3, %zmm2, %zmm2 -; AVX512-NEXT: vptestmq %zmm1, %zmm1, %k1 -; AVX512-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z} -; AVX512-NEXT: vmovd %xmm1, %r9d -; AVX512-NEXT: vpermq (%rdi), %zmm0, %zmm0 -; AVX512-NEXT: vplzcntq %zmm0, %zmm1 -; AVX512-NEXT: vpaddq %zmm3, %zmm1, %zmm1 -; AVX512-NEXT: vptestmq %zmm0, %zmm0, %k1 -; AVX512-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} -; AVX512-NEXT: vmovd %xmm0, %eax -; AVX512-NEXT: addl $512, %eax # imm = 0x200 -; AVX512-NEXT: orq 120(%rdi), %r8 -; AVX512-NEXT: orq 104(%rdi), %rdx -; AVX512-NEXT: orq 112(%rdi), %rsi -; AVX512-NEXT: orq %r8, %rdx -; AVX512-NEXT: orq 96(%rdi), %rcx -; AVX512-NEXT: orq %rsi, %rcx -; AVX512-NEXT: orq %rdx, %rcx -; AVX512-NEXT: cmovnel %r9d, %eax -; AVX512-NEXT: vzeroupper -; AVX512-NEXT: retq +; AVX512F-LABEL: load_ctlz_undef_i1024: +; AVX512F: # %bb.0: +; AVX512F-NEXT: movq 80(%rdi), %rsi +; AVX512F-NEXT: movq 64(%rdi), %rcx +; AVX512F-NEXT: movq 72(%rdi), %rdx +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512F-NEXT: vpermq 64(%rdi), %zmm0, %zmm1 +; AVX512F-NEXT: movq 88(%rdi), %r8 +; AVX512F-NEXT: vplzcntq %zmm1, %zmm2 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448] +; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2 +; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z} +; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512F-NEXT: vmovd %xmm1, %r9d +; AVX512F-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512F-NEXT: vpaddq %zmm3, %zmm1, %zmm1 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512F-NEXT: vmovd %xmm0, %eax +; AVX512F-NEXT: orq 120(%rdi), %r8 +; AVX512F-NEXT: addl $512, %eax # imm = 0x200 +; AVX512F-NEXT: orq 104(%rdi), %rdx +; AVX512F-NEXT: orq %r8, %rdx +; AVX512F-NEXT: orq 112(%rdi), %rsi +; AVX512F-NEXT: orq 96(%rdi), %rcx +; AVX512F-NEXT: orq %rsi, %rcx +; AVX512F-NEXT: orq %rdx, %rcx +; AVX512F-NEXT: cmovnel %r9d, %eax +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: load_ctlz_undef_i1024: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: movq 80(%rdi), %rsi +; AVX512VL-NEXT: movq 64(%rdi), %rcx +; AVX512VL-NEXT: movq 72(%rdi), %rdx +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512VL-NEXT: vpermq 64(%rdi), %zmm0, %zmm1 +; AVX512VL-NEXT: movq 88(%rdi), %r8 +; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448] +; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2 +; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm1, %r9d +; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512VL-NEXT: vpaddq %zmm3, %zmm1, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: addl $512, %eax # imm = 0x200 +; AVX512VL-NEXT: orq 120(%rdi), %r8 +; AVX512VL-NEXT: orq 104(%rdi), %rdx +; AVX512VL-NEXT: orq 112(%rdi), %rsi +; AVX512VL-NEXT: orq %r8, %rdx +; AVX512VL-NEXT: orq 96(%rdi), %rcx +; AVX512VL-NEXT: orq %rsi, %rcx +; AVX512VL-NEXT: orq %rdx, %rcx +; AVX512VL-NEXT: cmovnel %r9d, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; +; AVX512POPCNT-LABEL: load_ctlz_undef_i1024: +; AVX512POPCNT: # %bb.0: +; AVX512POPCNT-NEXT: movq 80(%rdi), %rsi +; AVX512POPCNT-NEXT: movq 64(%rdi), %rcx +; AVX512POPCNT-NEXT: movq 72(%rdi), %rdx +; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0] +; AVX512POPCNT-NEXT: vpermq 64(%rdi), %zmm0, %zmm1 +; AVX512POPCNT-NEXT: movq 88(%rdi), %r8 +; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2 +; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448] +; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2 +; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z} +; AVX512POPCNT-NEXT: vmovd %xmm1, %r9d +; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0 +; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1 +; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm1, %zmm1 +; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512POPCNT-NEXT: vmovd %xmm0, %eax +; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200 +; AVX512POPCNT-NEXT: orq 120(%rdi), %r8 +; AVX512POPCNT-NEXT: orq 104(%rdi), %rdx +; AVX512POPCNT-NEXT: orq 112(%rdi), %rsi +; AVX512POPCNT-NEXT: orq %r8, %rdx +; AVX512POPCNT-NEXT: orq 96(%rdi), %rcx +; AVX512POPCNT-NEXT: orq %rsi, %rcx +; AVX512POPCNT-NEXT: orq %rdx, %rcx +; AVX512POPCNT-NEXT: cmovnel %r9d, %eax +; AVX512POPCNT-NEXT: vzeroupper +; AVX512POPCNT-NEXT: retq %a0 = load i1024, ptr %p0 %cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 -1) %res = trunc i1024 %cnt to i32 @@ -2864,19 +3745,36 @@ define i32 @load_cttz_i256(ptr %p0) nounwind { ; AVX512F-LABEL: load_cttz_i256: ; AVX512F: # %bb.0: ; AVX512F-NEXT: vmovdqu (%rdi), %ymm0 -; AVX512F-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1 -; AVX512F-NEXT: vpaddq %ymm1, %ymm0, %ymm1 -; AVX512F-NEXT: vpandn %ymm1, %ymm0, %ymm1 -; AVX512F-NEXT: vplzcntq %ymm1, %ymm1 -; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256] -; AVX512F-NEXT: vpsubq %ymm1, %ymm2, %ymm1 -; AVX512F-NEXT: vptestmq %ymm0, %ymm0, %k1 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256] -; AVX512F-NEXT: vpcompressq %ymm1, %ymm0 {%k1} -; AVX512F-NEXT: vmovd %xmm0, %eax -; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [256,256,256,256] +; AVX512F-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2 +; AVX512F-NEXT: vpaddq %ymm2, %ymm0, %ymm2 +; AVX512F-NEXT: vpandn %ymm2, %ymm0, %ymm2 +; AVX512F-NEXT: vplzcntq %zmm2, %zmm2 +; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [64,128,192,256] +; AVX512F-NEXT: vpsubq %ymm2, %ymm3, %ymm2 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kshiftlw $12, %k0, %k0 +; AVX512F-NEXT: kshiftrw $12, %k0, %k1 +; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} +; AVX512F-NEXT: vmovd %xmm1, %eax ; AVX512F-NEXT: retq ; +; AVX512VL-LABEL: load_cttz_i256: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovdqu (%rdi), %ymm0 +; AVX512VL-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1 +; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm1 +; AVX512VL-NEXT: vpandn %ymm1, %ymm0, %ymm1 +; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1 +; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256] +; AVX512VL-NEXT: vpsubq %ymm1, %ymm2, %ymm1 +; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256] +; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; ; AVX512POPCNT-LABEL: load_cttz_i256: ; AVX512POPCNT: # %bb.0: ; AVX512POPCNT-NEXT: vmovdqu (%rdi), %ymm0 @@ -3012,9 +3910,35 @@ define i32 @test_cttz_i512(i512 %a0) nounwind { ; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] ; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} ; AVX512F-NEXT: vmovd %xmm0, %eax -; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq ; +; AVX512VL-LABEL: test_cttz_i512: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovq %rcx, %xmm0 +; AVX512VL-NEXT: vmovq %rdx, %xmm1 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512VL-NEXT: vmovq %rsi, %xmm1 +; AVX512VL-NEXT: vmovq %rdi, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512VL-NEXT: vmovq %r9, %xmm1 +; AVX512VL-NEXT: vmovq %r8, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1 +; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 = -1 +; AVX512VL-NEXT: vpaddq %zmm1, %zmm0, %zmm1 +; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1 +; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512] +; AVX512VL-NEXT: vpsubq %zmm1, %zmm2, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; ; AVX512POPCNT-LABEL: test_cttz_i512: ; AVX512POPCNT: # %bb.0: ; AVX512POPCNT-NEXT: vmovq %rcx, %xmm0 @@ -3166,9 +4090,24 @@ define i32 @load_cttz_i512(ptr %p0) nounwind { ; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] ; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} ; AVX512F-NEXT: vmovd %xmm0, %eax -; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq ; +; AVX512VL-LABEL: load_cttz_i512: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0 +; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 = -1 +; AVX512VL-NEXT: vpaddq %zmm1, %zmm0, %zmm1 +; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1 +; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512] +; AVX512VL-NEXT: vpsubq %zmm1, %zmm2, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; ; AVX512POPCNT-LABEL: load_cttz_i512: ; AVX512POPCNT: # %bb.0: ; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm0 @@ -3428,12 +4367,12 @@ define i32 @test_cttz_i1024(i1024 %a0) nounwind { ; AVX512F-NEXT: vmovq %rsi, %xmm1 ; AVX512F-NEXT: vmovq %rdi, %xmm2 ; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] -; AVX512F-NEXT: vmovq %r9, %xmm2 -; AVX512F-NEXT: vmovq %r8, %xmm3 -; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] -; AVX512F-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm2, %ymm2 ; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 -; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 +; AVX512F-NEXT: vmovq %r9, %xmm1 +; AVX512F-NEXT: vmovq %r8, %xmm2 +; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512F-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1 +; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 ; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1 ; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1 ; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm3 @@ -3461,9 +4400,52 @@ define i32 @test_cttz_i1024(i1024 %a0) nounwind { ; AVX512F-NEXT: orq %rdi, %rdx ; AVX512F-NEXT: orq %rcx, %rdx ; AVX512F-NEXT: cmovnel %r10d, %eax -; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq ; +; AVX512VL-LABEL: test_cttz_i1024: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovq %rcx, %xmm0 +; AVX512VL-NEXT: vmovq %rdx, %xmm1 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512VL-NEXT: vmovq %rsi, %xmm1 +; AVX512VL-NEXT: vmovq %rdi, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vmovq %r9, %xmm2 +; AVX512VL-NEXT: vmovq %r8, %xmm3 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0] +; AVX512VL-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm2, %ymm2 +; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 +; AVX512VL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1 +; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1 +; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm3 +; AVX512VL-NEXT: vpandnq %zmm3, %zmm0, %zmm3 +; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512] +; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %r10d +; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm0 +; AVX512VL-NEXT: vpandnq %zmm0, %zmm1, %zmm0 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512VL-NEXT: vpsubq %zmm0, %zmm4, %zmm0 +; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512] +; AVX512VL-NEXT: vpcompressq %zmm0, %zmm1 {%k1} +; AVX512VL-NEXT: vmovd %xmm1, %eax +; AVX512VL-NEXT: addl $512, %eax # imm = 0x200 +; AVX512VL-NEXT: orq %r9, %rsi +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rcx +; AVX512VL-NEXT: orq %rsi, %rcx +; AVX512VL-NEXT: orq %r8, %rdi +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rdx +; AVX512VL-NEXT: orq %rdi, %rdx +; AVX512VL-NEXT: orq %rcx, %rdx +; AVX512VL-NEXT: cmovnel %r10d, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; ; AVX512POPCNT-LABEL: test_cttz_i1024: ; AVX512POPCNT: # %bb.0: ; AVX512POPCNT-NEXT: vmovq %rcx, %xmm0 @@ -3765,9 +4747,9 @@ define i32 @load_cttz_i1024(ptr %p0) nounwind { ; AVX512F-NEXT: movq 24(%rdi), %rsi ; AVX512F-NEXT: orq 56(%rdi), %rsi ; AVX512F-NEXT: orq 40(%rdi), %rdx -; AVX512F-NEXT: orq %rsi, %rdx ; AVX512F-NEXT: orq 48(%rdi), %rax ; AVX512F-NEXT: orq 32(%rdi), %rcx +; AVX512F-NEXT: orq %rsi, %rdx ; AVX512F-NEXT: orq %rax, %rcx ; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1 ; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm3 @@ -3789,9 +4771,45 @@ define i32 @load_cttz_i1024(ptr %p0) nounwind { ; AVX512F-NEXT: addl $512, %eax # imm = 0x200 ; AVX512F-NEXT: orq %rdx, %rcx ; AVX512F-NEXT: cmovnel %esi, %eax -; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq ; +; AVX512VL-LABEL: load_cttz_i1024: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0 +; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm1 +; AVX512VL-NEXT: movq 16(%rdi), %rax +; AVX512VL-NEXT: movq (%rdi), %rcx +; AVX512VL-NEXT: movq 8(%rdi), %rdx +; AVX512VL-NEXT: movq 24(%rdi), %rsi +; AVX512VL-NEXT: orq 56(%rdi), %rsi +; AVX512VL-NEXT: orq 40(%rdi), %rdx +; AVX512VL-NEXT: orq %rsi, %rdx +; AVX512VL-NEXT: orq 48(%rdi), %rax +; AVX512VL-NEXT: orq 32(%rdi), %rcx +; AVX512VL-NEXT: orq %rax, %rcx +; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1 +; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm3 +; AVX512VL-NEXT: vpandnq %zmm3, %zmm1, %zmm3 +; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512] +; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3 +; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512VL-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm1, %esi +; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm1 +; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1 +; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1 +; AVX512VL-NEXT: vpsubq %zmm1, %zmm4, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512] +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: addl $512, %eax # imm = 0x200 +; AVX512VL-NEXT: orq %rdx, %rcx +; AVX512VL-NEXT: cmovnel %esi, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; ; AVX512POPCNT-LABEL: load_cttz_i1024: ; AVX512POPCNT: # %bb.0: ; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0 @@ -4022,15 +5040,31 @@ define i32 @load_cttz_undef_i256(ptr %p0) nounwind { ; AVX512F-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpaddq %ymm1, %ymm0, %ymm1 ; AVX512F-NEXT: vpandn %ymm1, %ymm0, %ymm1 +; AVX512F-NEXT: vplzcntq %zmm1, %zmm1 ; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256] -; AVX512F-NEXT: vplzcntq %ymm1, %ymm1 ; AVX512F-NEXT: vpsubq %ymm1, %ymm2, %ymm1 -; AVX512F-NEXT: vptestmq %ymm0, %ymm0, %k1 -; AVX512F-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z} +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kshiftlw $12, %k0, %k0 +; AVX512F-NEXT: kshiftrw $12, %k0, %k1 +; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} ; AVX512F-NEXT: vmovd %xmm0, %eax -; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq ; +; AVX512VL-LABEL: load_cttz_undef_i256: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovdqu (%rdi), %ymm0 +; AVX512VL-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1 +; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm1 +; AVX512VL-NEXT: vpandn %ymm1, %ymm0, %ymm1 +; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm2 = [64,128,192,256] +; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1 +; AVX512VL-NEXT: vpsubq %ymm1, %ymm2, %ymm1 +; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1 +; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; ; AVX512POPCNT-LABEL: load_cttz_undef_i256: ; AVX512POPCNT: # %bb.0: ; AVX512POPCNT-NEXT: vmovdqu (%rdi), %ymm0 @@ -4157,15 +5191,40 @@ define i32 @test_cttz_undef_i512(i512 %a0) nounwind { ; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1 ; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1 ; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1 -; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512] ; AVX512F-NEXT: vplzcntq %zmm1, %zmm1 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512] ; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm1 ; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 ; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} ; AVX512F-NEXT: vmovd %xmm0, %eax -; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq ; +; AVX512VL-LABEL: test_cttz_undef_i512: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovq %rcx, %xmm0 +; AVX512VL-NEXT: vmovq %rdx, %xmm1 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512VL-NEXT: vmovq %rsi, %xmm1 +; AVX512VL-NEXT: vmovq %rdi, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512VL-NEXT: vmovq %r9, %xmm1 +; AVX512VL-NEXT: vmovq %r8, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1 +; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 = -1 +; AVX512VL-NEXT: vpaddq %zmm1, %zmm0, %zmm1 +; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512] +; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1 +; AVX512VL-NEXT: vpsubq %zmm1, %zmm2, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; ; AVX512POPCNT-LABEL: test_cttz_undef_i512: ; AVX512POPCNT: # %bb.0: ; AVX512POPCNT-NEXT: vmovq %rcx, %xmm0 @@ -4308,15 +5367,29 @@ define i32 @load_cttz_undef_i512(ptr %p0) nounwind { ; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1 ; AVX512F-NEXT: vpaddq %zmm1, %zmm0, %zmm1 ; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1 -; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512] ; AVX512F-NEXT: vplzcntq %zmm1, %zmm1 +; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512] ; AVX512F-NEXT: vpsubq %zmm1, %zmm2, %zmm1 ; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 ; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} ; AVX512F-NEXT: vmovd %xmm0, %eax -; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq ; +; AVX512VL-LABEL: load_cttz_undef_i512: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0 +; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 = -1 +; AVX512VL-NEXT: vpaddq %zmm1, %zmm0, %zmm1 +; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [64,128,192,256,320,384,448,512] +; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1 +; AVX512VL-NEXT: vpsubq %zmm1, %zmm2, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; ; AVX512POPCNT-LABEL: load_cttz_undef_i512: ; AVX512POPCNT: # %bb.0: ; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm0 @@ -4605,9 +5678,51 @@ define i32 @test_cttz_undef_i1024(i1024 %a0) nounwind { ; AVX512F-NEXT: orq %rdi, %rdx ; AVX512F-NEXT: orq %rcx, %rdx ; AVX512F-NEXT: cmovnel %r10d, %eax -; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq ; +; AVX512VL-LABEL: test_cttz_undef_i1024: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovq %rcx, %xmm0 +; AVX512VL-NEXT: vmovq %rdx, %xmm1 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0] +; AVX512VL-NEXT: vmovq %rsi, %xmm1 +; AVX512VL-NEXT: vmovq %rdi, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0 +; AVX512VL-NEXT: vmovq %r9, %xmm1 +; AVX512VL-NEXT: vmovq %r8, %xmm2 +; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0] +; AVX512VL-NEXT: vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1 +; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; AVX512VL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm1 +; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1 +; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm3 +; AVX512VL-NEXT: vpandnq %zmm3, %zmm0, %zmm3 +; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512] +; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpcompressq %zmm3, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %r10d +; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm0 +; AVX512VL-NEXT: vpandnq %zmm0, %zmm1, %zmm0 +; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0 +; AVX512VL-NEXT: vpsubq %zmm0, %zmm4, %zmm0 +; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: addl $512, %eax # imm = 0x200 +; AVX512VL-NEXT: orq %r9, %rsi +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rcx +; AVX512VL-NEXT: orq %rsi, %rcx +; AVX512VL-NEXT: orq %r8, %rdi +; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rdx +; AVX512VL-NEXT: orq %rdi, %rdx +; AVX512VL-NEXT: orq %rcx, %rdx +; AVX512VL-NEXT: cmovnel %r10d, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; ; AVX512POPCNT-LABEL: test_cttz_undef_i1024: ; AVX512POPCNT: # %bb.0: ; AVX512POPCNT-NEXT: vmovq %rcx, %xmm0 @@ -4908,8 +6023,8 @@ define i32 @load_cttz_undef_i1024(ptr %p0) nounwind { ; AVX512F-NEXT: orq 56(%rdi), %rsi ; AVX512F-NEXT: orq 40(%rdi), %rdx ; AVX512F-NEXT: orq 48(%rdi), %rax -; AVX512F-NEXT: orq 32(%rdi), %rcx ; AVX512F-NEXT: orq %rsi, %rdx +; AVX512F-NEXT: orq 32(%rdi), %rcx ; AVX512F-NEXT: orq %rax, %rcx ; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1 ; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm3 @@ -4930,9 +6045,44 @@ define i32 @load_cttz_undef_i1024(ptr %p0) nounwind { ; AVX512F-NEXT: addl $512, %eax # imm = 0x200 ; AVX512F-NEXT: orq %rdx, %rcx ; AVX512F-NEXT: cmovnel %esi, %eax -; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq ; +; AVX512VL-LABEL: load_cttz_undef_i1024: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0 +; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm1 +; AVX512VL-NEXT: movq 16(%rdi), %rax +; AVX512VL-NEXT: movq (%rdi), %rcx +; AVX512VL-NEXT: movq 8(%rdi), %rdx +; AVX512VL-NEXT: movq 24(%rdi), %rsi +; AVX512VL-NEXT: orq 56(%rdi), %rsi +; AVX512VL-NEXT: orq 40(%rdi), %rdx +; AVX512VL-NEXT: orq 48(%rdi), %rax +; AVX512VL-NEXT: orq 32(%rdi), %rcx +; AVX512VL-NEXT: orq %rsi, %rdx +; AVX512VL-NEXT: orq %rax, %rcx +; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1 +; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm3 +; AVX512VL-NEXT: vpandnq %zmm3, %zmm1, %zmm3 +; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3 +; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512] +; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3 +; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1 +; AVX512VL-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm1, %esi +; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm1 +; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1 +; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1 +; AVX512VL-NEXT: vpsubq %zmm1, %zmm4, %zmm1 +; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z} +; AVX512VL-NEXT: vmovd %xmm0, %eax +; AVX512VL-NEXT: addl $512, %eax # imm = 0x200 +; AVX512VL-NEXT: orq %rdx, %rcx +; AVX512VL-NEXT: cmovnel %esi, %eax +; AVX512VL-NEXT: vzeroupper +; AVX512VL-NEXT: retq +; ; AVX512POPCNT-LABEL: load_cttz_undef_i1024: ; AVX512POPCNT: # %bb.0: ; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0