Look more aggressively for `vpgatherdd` opportunities

[Validark]

(Zig Version)
LLVM version:

define dso_local <16 x i32> @foo(ptr nonnull align 4 %0, <16 x i8> %1, <16 x i32> %2) local_unnamed_addr {
Entry:
  %3 = ptrtoint ptr %0 to i64
  %4 = insertelement <1 x i64> poison, i64 %3, i64 0
  %5 = shufflevector <1 x i64> %4, <1 x i64> poison, <16 x i32> zeroinitializer
  %6 = zext <16 x i32> %2 to <16 x i64>
  %7 = add nuw <16 x i64> %5, %6
  %8 = inttoptr <16 x i64> %7 to <16 x ptr>
  %9 = tail call fastcc <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> %8, i32 4, <16 x i1> splat (i1 true), <16 x i32> poison)
  ret <16 x i32> %9
}

declare fastcc <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr>, i32 immarg, <16 x i1>, <16 x i32>) #1

Emit:

foo:
        vpmovzxdq       zmm0, ymm1
        vextracti64x4   ymm1, zmm1, 1
        kxnorw  k1, k0, k0
        kxnorw  k2, k0, k0
        vpxor   xmm2, xmm2, xmm2
        vpxor   xmm3, xmm3, xmm3
        vpmovzxdq       zmm1, ymm1
        vpgatherqd      ymm2 {k1}, dword ptr [rdi + zmm0]
        vpgatherqd      ymm3 {k2}, dword ptr [rdi + zmm1]
        vinserti64x4    zmm0, zmm2, ymm3, 1
        ret

Should be:

bar:
        vxor    xmm0, xmm0, xmm0
        kxnorw  k1, k0, k0
        vpgatherdd      zmm0 {k1}, dword ptr [rdi + zmm1]
        ret

[llvmbot]

@llvm/issue-subscribers-backend-x86

Author: Niles Salter (Validark)

[(Zig Version)](https://zig.godbo.lt/#z:OYLghAFBqd5QCxAYwPYBMCmBRdBLAF1QCcAaPECAMzwBtMA7AQwFtMQByARg9KtQYEAysib0QXACx8BBAKoBnTAAUAHpwAMvAFYTStJg1AAvPMFJL6yAngGVG6AMKpaAVxYMQAVgDMpBwAyeAyYAHLuAEaYxCCSAJykAA6oCoS2DM5uHt5%2Byak2AkEh4SxRMfEWmFYFDEIETMQEme6evpXV6XUNBEVhkdGxCQr1jc3ZbcPdvSVlgwCUFqiuxMjsHGgMwwDUAGoB2KFbAKQ%2BACJbXABsJwBCRxoAghvb2MdnW64%2BAEy39w9/mFUyUaWyoDFBqFQEHUxy8NwAVEcvKdsKQtlsAPogLYAAR2mGsJAgVzRrgAHAstqgqFQlAQFNi8QSiMQIHsDmjsHM5rj8YTWezQpyeUcAOx3R7o9HPAhbRIEYgM3nMomC0mpYyYEXvHEKRIGAgQHHBAgAMWIqBYygV0O5tylx0lUuImAIy3BwCYBAQ0SN8uI5stAElBBB/Qpjl8blSaXSFHafBL/qLTn8/mCtp7vb7w4y%2BSy2fshbCEUiUdzlfzCxytlzjuK/g6XW7iODhsRXNZ60mHQ7AQRouCMzijl8vrRaAA3FgAOhYTAUAGtMOgZ1mfcRR18II3e3umVW1SXEciuWiAPTnuUKhS7vcOz5fC9XsRmBhsQR3%2B/og8Fo%2BuLhKUvLZ5yXL9v1/VUi2FZ85QXYYEA7cCpR5UQJw2ScIBnJhXCIHlIIFaDazmX4nXRMVThnEcxwnac5wXZdV3XaItx3Mje3DUhkJ/V9gAYAB5KgIDPbjcT1A1iQWUTXAYLAaBCdAuPYkjEzTFM00eDMIgaaFsSRUtTzRLFKwLEkPgpNFqVpV0lQI6tiy5fD8ygms6zFHtnVdd0tknRJmOIdB0GhSzYxstFdX1L0IA0SkNFUU0EtNNENDRKity4LcVKTCiNKeARtmGdA3nOY0WGBQ0t0KzLSP%2BTTwV8/zArYh4HX9PS4RPU5HyUlqpSsuM8xVVkzMfKT2MSeDvQtVxgCQEyiRG74xt69FQMXbEAMuHqHTQMqbDYLZdvlPA2AxfrXWxPBvm2qUjv2zBDstY7ToUBA8CoAhsUYdwtggVxJBFcVcXSrdioucKtx%2BMcwafYGx0kUGTnOaQ4a%2BMlEfeMl61OHqnKG4kto%2BJbuy/ScGi2ZtBqrRavm1c4JoUBDptmmqHQXFhfsKmcqBYAgZzuk6VGIE0dzHBqvQ3QKSZTUWvH0/0yzRUc5bhc76UV6XTkpZX9ObMtpbuL4VZuNayxy9Sx1S9yBdOtXwpNQMWGwBh3AgG3MAxV73oIEiUxI9iHXam49eRSMoZuVRQYgZtlu/IPTdDrckWwABNRco7W2PvxLNXb0TsdI%2BhiBc6z795YVfWt03Iv/VL3tsRndytn55AwYVVwHoorKv2bby9dUx4csHx5gPF7MAqK3tjBYFguG7RcMpTNF0AAdxIIr/RLAK8EjaNp5YDQyw4BZaE4LxeE8DgtFIVBOAALTMLYFCWFZO6%2BHweFIT6r%2BPhZ1q8LgM54iih8JIHwlxLiSA0F4S4799CcEkLwFgEgNApUvtfW%2BHBeAMhSt/LQCw4CwCQEdOg0RyCUBIfQGIxgNBcAAXwOgA5FSUAiJoXgERggNAAJ6cE/hw5gxAuH8QiNoZkvDeC7Q/AQfiDBaA8J/qQLAEQZqODELQBk3BeBYHnEYcQCj8AumsHgScmANHX0BASXCaxP4miqGw/QeAIjEG4c4LA9iFQnXEaQExxAIgpEwKcTAOjgC0GCKAH%2BCwqAGGAAoHYeBMAr34okRgXj%2BCCBEGIdgUgZCCEUCodQCjdBPgMEYEAphzChIiAySACxUDHXypwAAtPxLYAAlKomAFyYFNAuWUjSRjAFdFsVQZJLgYkgY00JLtVBbEaSwZAiRXBI2MAwHxXhZn8BYIQRpVBnFsEackWxxBeCoB8cLLA1SIALEsMydI9hZJjE8Fwfwslpj9BiM8vIaQBCPL0F8mobzSgDGeTcoxAguijBcC0PQoKagQp6MEPoQKPkWBGE0KF2QQVosBbMQCixlirAkCfM%2BF97GYK2OUrYGgZx0Jpb9XAhASCRg/nMXgeDf6kB9EwLAMQrmkHWpIMkM4vAaC%2BJcGBPgfBeDiGA0Uop4EcEQaQZBXBUEzlQRqzVmrnnoJOZwbBIBcFsO5Aqr4SCUFoLJfqr%2BxqFg%2BNSHYSQQA) [LLVM version:](https://llvm.godbo.lt/#g:!((g:!((g:!((h:codeEditor,i:(filename:'1',fontScale:14,fontUsePx:'0',j:1,lang:llvm,selection:(endColumn:1,endLineNumber:14,positionColumn:1,positionLineNumber:14,selectionStartColumn:1,selectionStartLineNumber:14,startColumn:1,startLineNumber:14),source:'define+dso_local+%3C16+x+i32%3E+@foo(ptr+nonnull+align+4+%250,+%3C16+x+i8%3E+%251,+%3C16+x+i32%3E+%252)+local_unnamed_addr+%7B%0AEntry:%0A++%253+%3D+ptrtoint+ptr+%250+to+i64%0A++%254+%3D+insertelement+%3C1+x+i64%3E+poison,+i64+%253,+i64+0%0A++%255+%3D+shufflevector+%3C1+x+i64%3E+%254,+%3C1+x+i64%3E+poison,+%3C16+x+i32%3E+zeroinitializer%0A++%256+%3D+zext+%3C16+x+i32%3E+%252+to+%3C16+x+i64%3E%0A++%257+%3D+add+nuw+%3C16+x+i64%3E+%255,+%256%0A++%258+%3D+inttoptr+%3C16+x+i64%3E+%257+to+%3C16+x+ptr%3E%0A++%259+%3D+tail+call+fastcc+%3C16+x+i32%3E+@llvm.masked.gather.v16i32.v16p0(%3C16+x+ptr%3E+%258,+i32+4,+%3C16+x+i1%3E+splat+(i1+true),+%3C16+x+i32%3E+poison)%0A++ret+%3C16+x+i32%3E+%259%0A%7D%0A%0Adeclare+fastcc+%3C16+x+i32%3E+@llvm.masked.gather.v16i32.v16p0(%3C16+x+ptr%3E,+i32+immarg,+%3C16+x+i1%3E,+%3C16+x+i32%3E)+%231%0A'),l:'5',n:'0',o:'LLVM+IR+source+%231',t:'0')),k:50,l:'4',n:'0',o:'',s:0,t:'0'),(g:!((h:compiler,i:(compiler:llctrunk,filters:(b:'0',binary:'1',binaryObject:'1',commentOnly:'0',debugCalls:'1',demangle:'0',directives:'0',execute:'1',intel:'0',libraryCode:'0',trim:'1',verboseDemangling:'0'),flagsViewOpen:'1',fontScale:14,fontUsePx:'0',j:1,lang:llvm,libs:!(),options:'-O3+-mcpu%3Dznver5',overrides:!(),selection:(endColumn:1,endLineNumber:1,positionColumn:1,positionLineNumber:1,selectionStartColumn:1,selectionStartLineNumber:1,startColumn:1,startLineNumber:1),source:1),l:'5',n:'0',o:'+llc+(trunk)+(Editor+%231)',t:'0')),k:50,l:'4',m:100,n:'0',o:'',s:0,t:'0')),l:'2',n:'0',o:'',t:'0')),version:4) ```llvm define dso_local <16 x i32> @foo(ptr nonnull align 4 %0, <16 x i8> %1, <16 x i32> %2) local_unnamed_addr { Entry: %3 = ptrtoint ptr %0 to i64 %4 = insertelement <1 x i64> poison, i64 %3, i64 0 %5 = shufflevector <1 x i64> %4, <1 x i64> poison, <16 x i32> zeroinitializer %6 = zext <16 x i32> %2 to <16 x i64> %7 = add nuw <16 x i64> %5, %6 %8 = inttoptr <16 x i64> %7 to <16 x ptr> %9 = tail call fastcc <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> %8, i32 4, <16 x i1> splat (i1 true), <16 x i32> poison) ret <16 x i32> %9 }

declare fastcc <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr>, i32 immarg, <16 x i1>, <16 x i32>) #1

Emit:

```asm
foo:
        vpmovzxdq       zmm0, ymm1
        vextracti64x4   ymm1, zmm1, 1
        kxnorw  k1, k0, k0
        kxnorw  k2, k0, k0
        vpxor   xmm2, xmm2, xmm2
        vpxor   xmm3, xmm3, xmm3
        vpmovzxdq       zmm1, ymm1
        vpgatherqd      ymm2 {k1}, dword ptr [rdi + zmm0]
        vpgatherqd      ymm3 {k2}, dword ptr [rdi + zmm1]
        vinserti64x4    zmm0, zmm2, ymm3, 1
        ret

Should be:

bar:
        vxor    xmm0, xmm0, xmm0
        kxnorw  k1, k0, k0
        vpgatherdd      zmm0 {k1}, dword ptr [rdi + zmm1]
        ret

[topperc]

Wouldn't the IR need to use sext, not zext? We still don't match it, but as far as I remember vpgatherdd, sign extends the index.

[topperc]

It works with sext if the insertelement uses <16 x i64> instead of <1 x i64>. Looks like we don't recognize it as a splat early enough.

[RKSimon]

Like @topperc mentioned - this should be a sign extension.

  t9: v1i64 = insert_vector_elt poison:v1i64, t2, Constant:i64<0>
        t10: v16i64 = concat_vectors t9, t9, t9, t9, t9, t9, t9, t9, t9, t9, t9, t9, t9, t9, t9, t9

To detect the splat earlier there's several different things we could do:

• Update SelectionDAG's foldCONCAT_VECTORS to try to fold concat(insert_vector_elt <1 x X>) patterns to build_vector - but this might be too aggressive?
• Alter X86's combineGatherScatter to use DAG.isSplat() instead of just using BuildVectorSDNode->getSplatValue() - but this might result in additional vector element extraction instructions.