PR middle-end/109766: Prevent cprop_hardreg bloating code with -Os.

PR 109766 is an interesting case of large code being generated on x86_64,
caused by an interaction/conflict between register allocation and hardreg
cprop, that's tricky to fix/resolve within the middle-end.

The task/challenge is to push a DImode value in an SSE register on to
the stack, when optimizing for size.  GCC's register allocator makes
the optimal choice to move the SSE register to a GPR, and then use push.
So after reload we have:

(insn 46 3 4 2 (set (reg:DF 1 dx [101])
        (reg:DF 21 xmm1 [ D1 ])) "pr109766.c":15:74 151 {*movdf_internal}
     (nil))
(insn 28 27 29 2 (set (mem:DF (pre_dec:DI (reg/f:DI 7 sp)) [0  S8 A64])
        (reg:DF 1 dx [101])) "pr109766.c":16:5 142 {*pushdf}
     (expr_list:REG_ARGS_SIZE (const_int 56 [0x38])
        (nil)))

which corresponds to the short 6 byte sequence:
66 48 0f 7e ca          movq   %xmm1,%rdx  [5 bytes]
52                      push   %rdx        [1 byte]

The problem is that several passes later, after pro_and_epilogue has
determined that the function doesn't need a stack frame, that the
hard register cprop pass sees the above two instructions, including
the initial register to register move, and decides to "simplify" it
as:

(insn 68 67 69 2 (set (mem:DI (pre_dec:DI (reg/f:DI 7 sp)) [0  S8 A64])
        (reg:DI 21 xmm1 [101])) "pr109766.c":16:5 62 {*pushdi2_rex64}
     (expr_list:REG_ARGS_SIZE (const_int 56 [0x38])
        (nil)))

but as x86_64 doesn't directly support push from SSE registers, the
above is split during split3 into:

(insn 92 91 93 2 (set (reg/f:DI 7 sp)
        (plus:DI (reg/f:DI 7 sp)
            (const_int -8 [0xfffffffffffffff8]))) "pr109766.c":16:5 247
{*leadi}
     (expr_list:REG_ARGS_SIZE (const_int 56 [0x38])
        (nil)))
(insn 93 92 94 2 (set (mem:DI (reg/f:DI 7 sp) [0  S8 A64])
        (reg:DI 21 xmm1 [101])) "pr109766.c":16:5 88 {*movdi_internal}
     (nil))

which corresponds to the bigger 10 byte sequence:

48 8d 64 24 f8          lea    -0x8(%rsp),%rsp  [5 bytes]
66 0f d6 0c 24          movq   %xmm1,(%rsp)     [5 bytes]

Clearly the cprop_hardreg substitution is questionable with -Os, but how
to prevent it is a challenge.  One (labor intensive) approach might be
to have regcprop.cc query the target's rtx_costs before performing
this type of substitution, which only works if the backend is
sufficiently parameterized.  Unfortunately, i386 like many targets
defines the rtx_cost of (set (dst) (src)) to be rtx_cost(dst) +
rtx_cost(src), which misses the subtlety of pushing an SSE register
to the stack.

An alternate solution, which can be implemented entirely in the
backend, is to prevent *pushdi2_rex64 being recognized (by
cprop_hardreg) with an SSE hard register operand after reload
when optimizing for size.

This patch has been tested on x86_64-pc-linux-gnu with make bootstrap
and make -k check, both with and without --target_board=unix{-m32},
with no new failures.  Ok for mainline?

2023-05-11  Roger Sayle  <roger@nextmovesoftware.com>

gcc/ChangeLog
        PR middle-end/109766
        * config/i386/i386.md (*pushdi_rex64): Disallow SSE registers
        after reload when optimizing for size.
        (*pushsi2_rex64): Likewise.
        (*pushsi2): Likewise.

gcc/testsuite/ChangeLog
        PR middle-end/109766
        * gcc.target/i386/pr109766.c: New test case.

Thanks in advance,
Roger

gcc/config/i386/i386.md

@@ -2036,7 +2036,10 @@
 (define_insn "*pushdi2_rex64"
   [(set (match_operand:DI 0 "push_operand" "=<,<,!<")
 	(match_operand:DI 1 "general_no_elim_operand" "re*m,*v,n"))]
-  "TARGET_64BIT"
+  "TARGET_64BIT
+   && (!reload_completed
+       || !SSE_REG_P (operands[1])
+       || !optimize_insn_for_size_p ())"
   "@
    push{q}\t%1
    #
@@ -2079,7 +2082,10 @@
 (define_insn "*pushsi2_rex64"
   [(set (match_operand:SI 0 "push_operand" "=X,X")
 	(match_operand:SI 1 "nonmemory_no_elim_operand" "re,*v"))]
-  "TARGET_64BIT"
+  "TARGET_64BIT
+   && (!reload_completed
+       || !SSE_REG_P (operands[1])
+       || !optimize_insn_for_size_p ())"
   "@
    push{q}\t%q1
    #"
@@ -2089,7 +2095,10 @@
 (define_insn "*pushsi2"
   [(set (match_operand:SI 0 "push_operand" "=<,<")
 	(match_operand:SI 1 "general_no_elim_operand" "ri*m,*v"))]
-  "!TARGET_64BIT"
+  "!TARGET_64BIT
+   && (!reload_completed
+       || !SSE_REG_P (operands[1])
+       || !optimize_insn_for_size_p ())"
   "@
    push{l}\t%1
    #"

gcc/testsuite/gcc.target/i386/pr109766.c

@@ -0,0 +1,21 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-Os" } */
+#define $expr(...) (__extension__({__VA_ARGS__;}))
+#define $regF0 $expr(register double x __asm("xmm0"); x)
+#define $regF1 $expr(register double x __asm("xmm1"); x)
+#define $regF2 $expr(register double x __asm("xmm2"); x)
+#define $regF3 $expr(register double x __asm("xmm3"); x)
+#define $regF4 $expr(register double x __asm("xmm4"); x)
+#define $regF5 $expr(register double x __asm("xmm5"); x)
+#define $regF6 $expr(register double x __asm("xmm6"); x)
+#define $regF7 $expr(register double x __asm("xmm7"); x)
+
+void func(char const*Fmt, ...);
+void callfunc(char const*Fmt, double D0, double D1, double D2, double D3,
+                              double D4, double D5, double D6, double D7){
+    func(Fmt,$regF0,$regF1,$regF2,$regF3,$regF4,$regF5,$regF6,$regF7,
+            D0,D1,D2,D3,D4,D5,D6,D7);
+}
+
+/* { dg-final { scan-assembler-times "pushq" 9 } } */
+/* { dg-final { scan-assembler-not "leaq" } } */