ARM64.rules

// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

(Add(Ptr|64|32|16|8) ...) => (ADD ...)
(Add(32|64)F ...) => (FADD(S|D) ...)

(Sub(Ptr|64|32|16|8) ...) => (SUB ...)
(Sub(32|64)F ...) => (FSUB(S|D) ...)

(Mul64 ...) => (MUL ...)
(Mul(32|16|8) ...) => (MULW ...)
(Mul(32|64)F  ...) => (FMUL(S|D) ...)

(Hmul64  ...) => (MULH ...)
(Hmul64u ...) => (UMULH ...)
(Hmul32  x y) => (SRAconst (MULL <typ.Int64> x y) [32])
(Hmul32u x y) => (SRAconst (UMULL <typ.UInt64> x y) [32])
(Select0 (Mul64uhilo x y)) => (UMULH x y)
(Select1 (Mul64uhilo x y)) => (MUL x y)

(Div64 [false] x y) => (DIV  x y)
(Div32 [false] x y) => (DIVW x y)
(Div16 [false] x y) => (DIVW (SignExt16to32 x) (SignExt16to32 y))
(Div16u x y) => (UDIVW (ZeroExt16to32 x) (ZeroExt16to32 y))
(Div8   x y) => (DIVW  (SignExt8to32  x) (SignExt8to32  y))
(Div8u  x y) => (UDIVW (ZeroExt8to32  x) (ZeroExt8to32  y))
(Div64u ...) => (UDIV  ...)
(Div32u ...) => (UDIVW ...)
(Div32F ...) => (FDIVS ...)
(Div64F ...) => (FDIVD ...)

(Mod64 x y) => (MOD x y)
(Mod32 x y) => (MODW x y)
(Mod64u ...) => (UMOD ...)
(Mod32u ...) => (UMODW ...)
(Mod(16|8)  x y) => (MODW  (SignExt(16|8)to32 x) (SignExt(16|8)to32 y))
(Mod(16|8)u x y) => (UMODW (ZeroExt(16|8)to32 x) (ZeroExt(16|8)to32 y))

// (x + y) / 2 with x>=y    =>    (x - y) / 2 + y
(Avg64u <t> x y) => (ADD (SRLconst <t> (SUB <t> x y) [1]) y)

(And(64|32|16|8) ...) => (AND ...)
(Or(64|32|16|8)  ...) => (OR ...)
(Xor(64|32|16|8) ...) => (XOR ...)

// unary ops
(Neg(64|32|16|8) ...) => (NEG ...)
(Neg(32|64)F     ...) => (FNEG(S|D) ...)
(Com(64|32|16|8) ...) => (MVN ...)

// math package intrinsics
(Abs         ...) => (FABSD   ...)
(Sqrt        ...) => (FSQRTD  ...)
(Ceil        ...) => (FRINTPD ...)
(Floor       ...) => (FRINTMD ...)
(Round       ...) => (FRINTAD ...)
(RoundToEven ...) => (FRINTND ...)
(Trunc       ...) => (FRINTZD ...)
(FMA       x y z) => (FMADDD z x y)

(Sqrt32 ...) => (FSQRTS ...)

(Min(64|32)F ...) => (FMIN(D|S) ...)
(Max(64|32)F ...) => (FMAX(D|S) ...)

// lowering rotates
// we do rotate detection in generic rules, if the following rules need to be changed, check generic rules first.
(RotateLeft8  <t> x (MOVDconst [c])) => (Or8 (Lsh8x64 <t> x (MOVDconst [c&7])) (Rsh8Ux64 <t> x (MOVDconst [-c&7])))
(RotateLeft8  <t> x y) => (OR <t> (SLL <t> x (ANDconst <typ.Int64> [7] y)) (SRL <t> (ZeroExt8to64 x) (ANDconst <typ.Int64> [7] (NEG <typ.Int64> y))))
(RotateLeft16 <t> x (MOVDconst [c])) => (Or16 (Lsh16x64 <t> x (MOVDconst [c&15])) (Rsh16Ux64 <t> x (MOVDconst [-c&15])))
(RotateLeft16 <t> x y) => (RORW <t> (ORshiftLL <typ.UInt32> (ZeroExt16to32 x) (ZeroExt16to32 x) [16]) (NEG <typ.Int64> y))
(RotateLeft32 x y) => (RORW x (NEG <y.Type> y))
(RotateLeft64 x y) => (ROR x (NEG <y.Type> y))

(Ctz(64|32|16|8)NonZero ...) => (Ctz(64|32|32|32) ...)

(Ctz64 <t> x) => (CLZ  (RBIT  <t> x))
(Ctz32 <t> x) => (CLZW (RBITW <t> x))
(Ctz16 <t> x) => (CLZW <t> (RBITW <typ.UInt32> (ORconst <typ.UInt32> [0x10000] x)))
(Ctz8  <t> x) => (CLZW <t> (RBITW <typ.UInt32> (ORconst <typ.UInt32> [0x100] x)))

(PopCount64 <t> x) => (FMOVDfpgp <t> (VUADDLV <typ.Float64> (VCNT <typ.Float64> (FMOVDgpfp <typ.Float64> x))))
(PopCount32 <t> x) => (FMOVDfpgp <t> (VUADDLV <typ.Float64> (VCNT <typ.Float64> (FMOVDgpfp <typ.Float64> (ZeroExt32to64 x)))))
(PopCount16 <t> x) => (FMOVDfpgp <t> (VUADDLV <typ.Float64> (VCNT <typ.Float64> (FMOVDgpfp <typ.Float64> (ZeroExt16to64 x)))))

// Load args directly into the register class where it will be used.
(FMOVDgpfp <t> (Arg [off] {sym})) => @b.Func.Entry (Arg <t> [off] {sym})
(FMOVDfpgp <t> (Arg [off] {sym})) => @b.Func.Entry (Arg <t> [off] {sym})

// Similarly for stores, if we see a store after FPR <=> GPR move, then redirect store to use the other register set.
(MOVDstore  [off] {sym} ptr (FMOVDfpgp val) mem) => (FMOVDstore [off] {sym} ptr val mem)
(FMOVDstore [off] {sym} ptr (FMOVDgpfp val) mem) => (MOVDstore [off] {sym} ptr val mem)
(MOVWstore  [off] {sym} ptr (FMOVSfpgp val) mem) => (FMOVSstore [off] {sym} ptr val mem)
(FMOVSstore [off] {sym} ptr (FMOVSgpfp val) mem) => (MOVWstore [off] {sym} ptr val mem)

// float <=> int register moves, with no conversion.
// These come up when compiling math.{Float64bits, Float64frombits, Float32bits, Float32frombits}.
(MOVDload  [off] {sym} ptr (FMOVDstore [off] {sym} ptr val _)) => (FMOVDfpgp val)
(FMOVDload [off] {sym} ptr (MOVDstore  [off] {sym} ptr val _)) => (FMOVDgpfp val)
(MOVWUload [off] {sym} ptr (FMOVSstore [off] {sym} ptr val _)) => (FMOVSfpgp val)
(FMOVSload [off] {sym} ptr (MOVWstore  [off] {sym} ptr val _)) => (FMOVSgpfp val)

(BitLen64 x) => (SUB (MOVDconst [64]) (CLZ <typ.Int> x))
(BitLen32 x) => (SUB (MOVDconst [32]) (CLZW <typ.Int> x))
(BitLen(16|8) x) => (BitLen64 (ZeroExt(16|8)to64 x))

(Bswap64 ...) => (REV ...)
(Bswap32 ...) => (REVW ...)
(Bswap16 ...) => (REV16W ...)

(BitRev64 ...) => (RBIT ...)
(BitRev32 ...) => (RBITW ...)
(BitRev16   x) => (SRLconst [48] (RBIT <typ.UInt64> x))
(BitRev8    x) => (SRLconst [56] (RBIT <typ.UInt64> x))

// In fact, UMOD will be translated into UREM instruction, and UREM is originally translated into
// UDIV and MSUB instructions. But if there is already an identical UDIV instruction just before or
// after UREM (case like quo, rem := z/y, z%y), then the second UDIV instruction becomes redundant.
// The purpose of this rule is to have this extra UDIV instruction removed in CSE pass.
(UMOD  <typ.UInt64> x y) => (MSUB <typ.UInt64> x y (UDIV <typ.UInt64> x y))
(UMODW <typ.UInt32> x y) => (MSUBW <typ.UInt32> x y (UDIVW <typ.UInt32> x y))

// 64-bit addition with carry.
(Select0 (Add64carry x y c)) => (Select0 <typ.UInt64> (ADCSflags x y (Select1 <types.TypeFlags> (ADDSconstflags [-1] c))))
(Select1 (Add64carry x y c)) => (ADCzerocarry <typ.UInt64> (Select1 <types.TypeFlags> (ADCSflags x y (Select1 <types.TypeFlags> (ADDSconstflags [-1] c)))))

// 64-bit subtraction with borrowing.
(Select0 (Sub64borrow x y bo)) => (Select0 <typ.UInt64> (SBCSflags x y (Select1 <types.TypeFlags> (NEGSflags bo))))
(Select1 (Sub64borrow x y bo)) => (NEG <typ.UInt64> (NGCzerocarry <typ.UInt64> (Select1 <types.TypeFlags> (SBCSflags x y (Select1 <types.TypeFlags> (NEGSflags bo))))))

// boolean ops -- booleans are represented with 0=false, 1=true
(AndB ...) => (AND ...)
(OrB  ...) => (OR ...)
(EqB  x y) => (XOR (MOVDconst [1]) (XOR <typ.Bool> x y))
(NeqB ...) => (XOR ...)
(Not    x) => (XOR (MOVDconst [1]) x)

// shifts
// hardware instruction uses only the low 6 bits of the shift
// we compare to 64 to ensure Go semantics for large shifts
// Rules about rotates with non-const shift are based on the following rules,
// if the following rules change, please also modify the rules based on them.

// check shiftIsBounded first, if shift value is proved to be valid then we
// can do the shift directly.
// left shift
(Lsh32x(64|32|16|8) ...) => (Lsh64x(64|32|16|8) ...)
(Lsh16x(64|32|16|8) ...) => (Lsh64x(64|32|16|8) ...)
(Lsh8x(64|32|16|8)  ...) => (Lsh64x(64|32|16|8) ...)

(Lsh64x(64|32|16|8) <t> x y) && ssa.ShiftIsBounded(v) => (SLL <t> x y)

// signed right shift
(Rsh32x(64|32|16|8) <t> [bounded] x y) => (Rsh64x(64|32|16|8) <t> [bounded] (SignExt32to64 x) y)
(Rsh16x(64|32|16|8) <t> [bounded] x y) => (Rsh64x(64|32|16|8) <t> [bounded] (SignExt16to64 x) y)
(Rsh8x(64|32|16|8)  <t> [bounded] x y) => (Rsh64x(64|32|16|8) <t> [bounded] (SignExt8to64 x)  y)

(Rsh64x(64|32|16|8) <t> x y) && ssa.ShiftIsBounded(v) => (SRA <t> x y)

// unsigned right shift
(Rsh32Ux(64|32|16|8) <t> [bounded] x y) => (Rsh64Ux(64|32|16|8) <t> [bounded] (ZeroExt32to64 x) y)
(Rsh16Ux(64|32|16|8) <t> [bounded] x y) => (Rsh64Ux(64|32|16|8) <t> [bounded] (ZeroExt16to64 x) y)
(Rsh8Ux(64|32|16|8)  <t> [bounded] x y) => (Rsh64Ux(64|32|16|8) <t> [bounded] (ZeroExt8to64 x)  y)

(Rsh64Ux(64|32|16|8) <t> x y) && ssa.ShiftIsBounded(v) => (SRL <t> x y)

// shift value may be out of range, use CMP + CSEL instead
(Lsh64x64 <t> x y) && !ssa.ShiftIsBounded(v) => (CSEL [ssaop.OpARM64LessThanU] (SLL <t> x y) (Const64 <t> [0]) (CMPconst  [64] y))
(Lsh64x32 <t> x y) && !ssa.ShiftIsBounded(v) => (CSEL [ssaop.OpARM64LessThanU] (SLL <t> x y) (Const64 <t> [0]) (CMPWconst [64] y))
(Lsh64x(16|8) <t> [bounded] x y) && !ssa.ShiftIsBounded(v) => (Lsh64x32 <t> [bounded] x (ZeroExt(16|8)to32 y))

(Rsh64Ux64 <t> x y) && !ssa.ShiftIsBounded(v) => (CSEL [ssaop.OpARM64LessThanU] (SRL <t> x y) (Const64 <t> [0]) (CMPconst  [64] y))
(Rsh64Ux32 <t> x y) && !ssa.ShiftIsBounded(v) => (CSEL [ssaop.OpARM64LessThanU] (SRL <t> x y) (Const64 <t> [0]) (CMPWconst [64] y))
(Rsh64Ux(16|8) <t> [bounded] x y) && !ssa.ShiftIsBounded(v) => (Rsh64Ux32 <t> [bounded] x (ZeroExt(16|8)to32 y))

(Rsh64x64 <t> x y) && !ssa.ShiftIsBounded(v) => (SRA x (CSEL [ssaop.OpARM64LessThanU] <y.Type> y (Const64 <y.Type> [63]) (CMPconst  [64] y)))
(Rsh64x32 <t> x y) && !ssa.ShiftIsBounded(v) => (SRA x (CSEL [ssaop.OpARM64LessThanU] <y.Type> y (Const64 <y.Type> [63]) (CMPWconst [64] y)))
(Rsh64x(16|8) <t> [bounded] x y) && !ssa.ShiftIsBounded(v) => (Rsh64x32 <t> [bounded] x (ZeroExt(16|8)to32 y))

// constants
(Const(64|32|16|8) [val]) => (MOVDconst [int64(val)])
(Const(32|64)F    [val]) => (FMOV(S|D)const [float64(val)])
(ConstNil) => (MOVDconst [0])
(ConstBool [t]) => (MOVDconst [ssa.B2i(t)])

(Slicemask <t> x) => (SRAconst (NEG <t> x) [63])

// truncations
// Because we ignore high parts of registers, truncates are just copies.
(Trunc16to8  ...) => (Copy ...)
(Trunc32to8  ...) => (Copy ...)
(Trunc32to16 ...) => (Copy ...)
(Trunc64to8  ...) => (Copy ...)
(Trunc64to16 ...) => (Copy ...)
(Trunc64to32 ...) => (Copy ...)

// Zero-/Sign-extensions
(ZeroExt8to16  ...) => (MOVBUreg ...)
(ZeroExt8to32  ...) => (MOVBUreg ...)
(ZeroExt16to32 ...) => (MOVHUreg ...)
(ZeroExt8to64  ...) => (MOVBUreg ...)
(ZeroExt16to64 ...) => (MOVHUreg ...)
(ZeroExt32to64 ...) => (MOVWUreg ...)

(SignExt8to16  ...) => (MOVBreg ...)
(SignExt8to32  ...) => (MOVBreg ...)
(SignExt16to32 ...) => (MOVHreg ...)
(SignExt8to64  ...) => (MOVBreg ...)
(SignExt16to64 ...) => (MOVHreg ...)
(SignExt32to64 ...) => (MOVWreg ...)

// float <=> int conversion
(Cvt32to32F  ...) => (SCVTFWS ...)
(Cvt32to64F  ...) => (SCVTFWD ...)
(Cvt64to32F  ...) => (SCVTFS ...)
(Cvt64to64F  ...) => (SCVTFD ...)
(Cvt32Uto32F ...) => (UCVTFWS ...)
(Cvt32Uto64F ...) => (UCVTFWD ...)
(Cvt64Uto32F ...) => (UCVTFS ...)
(Cvt64Uto64F ...) => (UCVTFD ...)
(Cvt32Fto32  ...) => (FCVTZSSW ...)
(Cvt64Fto32  ...) => (FCVTZSDW ...)
(Cvt32Fto64  ...) => (FCVTZSS ...)
(Cvt64Fto64  ...) => (FCVTZSD ...)
(Cvt32Fto32U ...) => (FCVTZUSW ...)
(Cvt64Fto32U ...) => (FCVTZUDW ...)
(Cvt32Fto64U ...) => (FCVTZUS ...)
(Cvt64Fto64U ...) => (FCVTZUD ...)
(Cvt32Fto64F ...) => (FCVTSD ...)
(Cvt64Fto32F ...) => (FCVTDS ...)

(CvtBoolToUint8 ...) => (Copy ...)

(Round32F ...) => (LoweredRound32F ...)
(Round64F ...) => (LoweredRound64F ...)

// comparisons
(Eq8  x y)  => (Equal (CMPW (ZeroExt8to32  x) (ZeroExt8to32  y)))
(Eq16  x y) => (Equal (CMPW (ZeroExt16to32 x) (ZeroExt16to32 y)))
(Eq32  x y) => (Equal (CMPW  x y))
(Eq64  x y) => (Equal (CMP   x y))
(EqPtr x y) => (Equal (CMP   x y))
(Eq32F x y) => (Equal (FCMPS x y))
(Eq64F x y) => (Equal (FCMPD x y))

(Neq8   x y) => (NotEqual (CMPW (ZeroExt8to32  x) (ZeroExt8to32  y)))
(Neq16  x y) => (NotEqual (CMPW (ZeroExt16to32 x) (ZeroExt16to32 y)))
(Neq32  x y) => (NotEqual (CMPW  x y))
(Neq64  x y) => (NotEqual (CMP   x y))
(NeqPtr x y) => (NotEqual (CMP   x y))
(Neq(32|64)F x y) => (NotEqual (FCMP(S|D) x y))

(Less(8|16) x y) => (LessThan (CMPW (SignExt(8|16)to32 x) (SignExt(8|16)to32 y)))
(Less32 x y) => (LessThan (CMPW x y))
(Less64 x y) => (LessThan (CMP  x y))

// Set condition flags for floating-point comparisons "x < y"
// and "x <= y". Because if either or both of the operands are
// NaNs, all three of (x < y), (x == y) and (x > y) are false,
// and ARM Manual says FCMP instruction sets PSTATE.<N,Z,C,V>
// of this case to (0, 0, 1, 1).
(Less32F x y) => (LessThanF (FCMPS x y))
(Less64F x y) => (LessThanF (FCMPD x y))

// For an unsigned integer x, the following rules are useful when combining branch
// 0 <  x  =>  x != 0
// x <= 0  =>  x == 0
// x <  1  =>  x == 0
// 1 <= x  =>  x != 0
(Less(8U|16U|32U|64U) zero:(MOVDconst [0]) x) => (Neq(8|16|32|64) zero x)
(Leq(8U|16U|32U|64U)  x zero:(MOVDconst [0])) => (Eq(8|16|32|64)  x zero)
(Less(8U|16U|32U|64U) x (MOVDconst [1])) => (Eq(8|16|32|64)  x (MOVDconst [0]))
(Leq(8U|16U|32U|64U)  (MOVDconst [1]) x) => (Neq(8|16|32|64) (MOVDconst [0]) x)

(Less8U  x y) => (LessThanU (CMPW (ZeroExt8to32  x) (ZeroExt8to32  y)))
(Less16U x y) => (LessThanU (CMPW (ZeroExt16to32 x) (ZeroExt16to32 y)))
(Less32U x y) => (LessThanU (CMPW x y))
(Less64U x y) => (LessThanU (CMP x y))

(Leq8  x y) => (LessEqual (CMPW (SignExt8to32  x) (SignExt8to32  y)))
(Leq16 x y) => (LessEqual (CMPW (SignExt16to32 x) (SignExt16to32 y)))
(Leq32 x y) => (LessEqual (CMPW x y))
(Leq64 x y) => (LessEqual (CMP x y))

// Refer to the comments for op Less64F above.
(Leq32F x y) => (LessEqualF (FCMPS x y))
(Leq64F x y) => (LessEqualF (FCMPD x y))

(Leq8U  x y) => (LessEqualU (CMPW (ZeroExt8to32  x) (ZeroExt8to32  y)))
(Leq16U x y) => (LessEqualU (CMPW (ZeroExt16to32 x) (ZeroExt16to32 y)))
(Leq32U x y) => (LessEqualU (CMPW x y))
(Leq64U x y) => (LessEqualU (CMP x y))

// Optimize comparison between a floating-point value and 0.0 with "FCMP $(0.0), Fn"
(FCMPS x (FMOVSconst [0])) => (FCMPS0 x)
(FCMPS (FMOVSconst [0]) x) => (InvertFlags (FCMPS0 x))
(FCMPD x (FMOVDconst [0])) => (FCMPD0 x)
(FCMPD (FMOVDconst [0]) x) => (InvertFlags (FCMPD0 x))

// Float min/max comparison-select idioms lower to a compare plus FCSEL on the
// FP registers. The ops carry their semantics directly, so lowering always
// triggers.
(Min64FSel x y) => (FCSELD [ssaop.OpARM64LessThanF] x y (FCMPD x y))
(Min32FSel x y) => (FCSELS [ssaop.OpARM64LessThanF] x y (FCMPS x y))
(Max64FSel x y) => (FCSELD [ssaop.OpARM64GreaterThanF] x y (FCMPD x y))
(Max32FSel x y) => (FCSELS [ssaop.OpARM64GreaterThanF] x y (FCMPS x y))

// CSEL needs a flag-generating argument. Synthesize a TSTW if necessary.
(CondSelect x y boolval) && ssa.FlagArg(boolval) != nil => (CSEL [boolval.Op] x y ssa.FlagArg(boolval))
(CondSelect x y boolval) && ssa.FlagArg(boolval) == nil => (CSEL [ssaop.OpARM64NotEqual] x y (TSTWconst [1] boolval))

(OffPtr [off] ptr:(SP)) && ssa.Is32Bit(off) => (MOVDaddr [int32(off)] ptr)
(OffPtr [off] ptr) => (ADDconst [off] ptr)

(Addr {sym} base) => (MOVDaddr {sym} base)
(LocalAddr <t> {sym} base mem) && t.Elem().HasPointers() => (MOVDaddr {sym} (SPanchored base mem))
(LocalAddr <t> {sym} base _)  && !t.Elem().HasPointers() => (MOVDaddr {sym} base)

// loads
(Load <t> ptr mem) && t.IsBoolean() => (MOVBUload ptr mem)
(Load <t> ptr mem) && (ssa.Is8BitInt(t)  &&  t.IsSigned()) => (MOVBload ptr mem)
(Load <t> ptr mem) && (ssa.Is8BitInt(t)  && !t.IsSigned()) => (MOVBUload ptr mem)
(Load <t> ptr mem) && (ssa.Is16BitInt(t) &&  t.IsSigned()) => (MOVHload ptr mem)
(Load <t> ptr mem) && (ssa.Is16BitInt(t) && !t.IsSigned()) => (MOVHUload ptr mem)
(Load <t> ptr mem) && (ssa.Is32BitInt(t) &&  t.IsSigned()) => (MOVWload ptr mem)
(Load <t> ptr mem) && (ssa.Is32BitInt(t) && !t.IsSigned()) => (MOVWUload ptr mem)
(Load <t> ptr mem) && (ssa.Is64BitInt(t) ||ssa.IsPtr(t)) => (MOVDload ptr mem)
(Load <t> ptr mem) && ssa.Is32BitFloat(t) => (FMOVSload ptr mem)
(Load <t> ptr mem) && ssa.Is64BitFloat(t) => (FMOVDload ptr mem)
(Load <t> ptr mem) && t.Size() == 16 => (FMOVQload ptr mem)

// stores
(Store {t} ptr val mem) && t.Size() == 1 => (MOVBstore ptr val mem)
(Store {t} ptr val mem) && t.Size() == 2 => (MOVHstore ptr val mem)
(Store {t} ptr val mem) && t.Size() == 4 && !t.IsFloat() => (MOVWstore ptr val mem)
(Store {t} ptr val mem) && t.Size() == 8 && !t.IsFloat() && !t.IsSIMD() => (MOVDstore ptr val mem)
(Store {t} ptr val mem) && t.Size() == 4 &&  t.IsFloat() => (FMOVSstore ptr val mem)
(Store {t} ptr val mem) && t.Size() == 8 &&  t.IsFloat() => (FMOVDstore ptr val mem)
(Store {t} ptr val mem) && t.Size() == 16 => (FMOVQstore ptr val mem)

// zeroing
(Zero [0] _   mem) => mem
(Zero [1] ptr mem) => (MOVBstore ptr (MOVDconst [0]) mem)
(Zero [2] ptr mem) => (MOVHstore ptr (MOVDconst [0]) mem)
(Zero [4] ptr mem) => (MOVWstore ptr (MOVDconst [0]) mem)
(Zero [3] ptr mem) =>
	(MOVBstore [2] ptr (MOVDconst [0])
		(MOVHstore ptr (MOVDconst [0]) mem))
(Zero [5] ptr mem) =>
	(MOVBstore [4] ptr (MOVDconst [0])
		(MOVWstore ptr (MOVDconst [0]) mem))
(Zero [6] ptr mem) =>
	(MOVHstore [4] ptr (MOVDconst [0])
		(MOVWstore ptr (MOVDconst [0]) mem))
(Zero [7] ptr mem) =>
	(MOVWstore [3] ptr (MOVDconst [0])
		(MOVWstore ptr (MOVDconst [0]) mem))
(Zero [8] ptr mem) => (MOVDstore ptr (MOVDconst [0]) mem)
(Zero [9] ptr mem) =>
	(MOVBstore [8] ptr (MOVDconst [0])
		(MOVDstore ptr (MOVDconst [0]) mem))
(Zero [10] ptr mem) =>
	(MOVHstore [8] ptr (MOVDconst [0])
		(MOVDstore ptr (MOVDconst [0]) mem))
(Zero [11] ptr mem) =>
	(MOVDstore [3] ptr (MOVDconst [0])
		(MOVDstore ptr (MOVDconst [0]) mem))
(Zero [12] ptr mem) =>
	(MOVWstore [8] ptr (MOVDconst [0])
		(MOVDstore ptr (MOVDconst [0]) mem))
(Zero [13] ptr mem) =>
	(MOVDstore [5] ptr (MOVDconst [0])
		(MOVDstore ptr (MOVDconst [0]) mem))
(Zero [14] ptr mem) =>
	(MOVDstore [6] ptr (MOVDconst [0])
		(MOVDstore ptr (MOVDconst [0]) mem))
(Zero [15] ptr mem) =>
	(MOVDstore [7] ptr (MOVDconst [0])
		(MOVDstore ptr (MOVDconst [0]) mem))
(Zero [16] ptr mem) =>
	(STP [0] ptr (MOVDconst [0]) (MOVDconst [0]) mem)

(Zero [s] ptr mem) && s > 16 && s < 192 => (LoweredZero [s] ptr mem)
(Zero [s] ptr mem) && s >= 192 => (LoweredZeroLoop [s] ptr mem)

// moves
(Move [0] _   _   mem) => mem
(Move [1] dst src mem) => (MOVBstore dst (MOVBUload src mem) mem)
(Move [2] dst src mem) => (MOVHstore dst (MOVHUload src mem) mem)
(Move [3] dst src mem) =>
	(MOVBstore [2] dst (MOVBUload [2] src mem)
		(MOVHstore dst (MOVHUload src mem) mem))
(Move [4] dst src mem) => (MOVWstore dst (MOVWUload src mem) mem)
(Move [5] dst src mem) =>
	(MOVBstore [4] dst (MOVBUload [4] src mem)
		(MOVWstore dst (MOVWUload src mem) mem))
(Move [6] dst src mem) =>
	(MOVHstore [4] dst (MOVHUload [4] src mem)
		(MOVWstore dst (MOVWUload src mem) mem))
(Move [7] dst src mem) =>
	(MOVWstore [3] dst (MOVWUload [3] src mem)
		(MOVWstore dst (MOVWUload src mem) mem))
(Move [8] dst src mem) => (MOVDstore dst (MOVDload src mem) mem)
(Move [9] dst src mem) =>
	(MOVBstore [8] dst (MOVBUload [8] src mem)
		(MOVDstore dst (MOVDload src mem) mem))
(Move [10] dst src mem) =>
	(MOVHstore [8] dst (MOVHUload [8] src mem)
		(MOVDstore dst (MOVDload src mem) mem))
(Move [11] dst src mem) =>
	(MOVDstore [3] dst (MOVDload [3] src mem)
		(MOVDstore dst (MOVDload src mem) mem))
(Move [12] dst src mem) =>
	(MOVWstore [8] dst (MOVWUload [8] src mem)
		(MOVDstore dst (MOVDload src mem) mem))
(Move [13] dst src mem) =>
	(MOVDstore [5] dst (MOVDload [5] src mem)
		(MOVDstore dst (MOVDload src mem) mem))
(Move [14] dst src mem) =>
	(MOVDstore [6] dst (MOVDload [6] src mem)
		(MOVDstore dst (MOVDload src mem) mem))
(Move [15] dst src mem) =>
	(MOVDstore [7] dst (MOVDload [7] src mem)
		(MOVDstore dst (MOVDload src mem) mem))
(Move [16] dst src mem) => (FMOVQstore dst (FMOVQload src mem) mem)

(Move [s] dst src mem) && s > 16 && s <= 24 =>
	(MOVDstore [int32(s-8)] dst (MOVDload [int32(s-8)] src mem)
		(FMOVQstore dst (FMOVQload src mem) mem))
(Move [s] dst src mem) && s > 24 && s < 32 =>
	(FMOVQstore [int32(s-16)] dst (FMOVQload [int32(s-16)] src mem)
		(FMOVQstore dst (FMOVQload src mem) mem))
(Move [32] dst src mem) =>
	(FSTPQ dst (Select0 <typ.Vec128> (FLDPQ src mem)) (Select1 <typ.Vec128> (FLDPQ src mem)) mem)
(Move [s] dst src mem) && s > 32 && s <= 40 =>
	(MOVDstore [int32(s-8)] dst (MOVDload [int32(s-8)] src mem)
		(FSTPQ dst (Select0 <typ.Vec128> (FLDPQ src mem)) (Select1 <typ.Vec128> (FLDPQ src mem)) mem))
(Move [s] dst src mem) && s > 40 && s <= 48 =>
	(FMOVQstore [int32(s-16)] dst (FMOVQload [int32(s-16)] src mem)
		(FSTPQ dst (Select0 <typ.Vec128> (FLDPQ src mem)) (Select1 <typ.Vec128> (FLDPQ src mem)) mem))
(Move [s] dst src mem) && s > 48 && s <= 64 =>
	(FSTPQ [int32(s-32)] dst (Select0 <typ.Vec128> (FLDPQ [int32(s-32)] src mem)) (Select1 <typ.Vec128> (FLDPQ [int32(s-32)] src mem))
		(FSTPQ dst (Select0 <typ.Vec128> (FLDPQ src mem)) (Select1 <typ.Vec128> (FLDPQ src mem)) mem))

(Move [s] dst src mem) && s > 64 && s < 192 && ssa.LogLargeCopyValue(v, s) => (LoweredMove [s] dst src mem)
(Move [s] dst src mem) && s >= 192 && ssa.LogLargeCopyValue(v, s) => (LoweredMoveLoop [s] dst src mem)

// calls
(StaticCall    ...) => (CALLstatic    ...)
(ClosureCall   ...) => (CALLclosure   ...)
(InterCall     ...) => (CALLinter     ...)
(TailCall      ...) => (CALLtail      ...)
(TailCallInter ...) => (CALLtailinter ...)

// checks
(NilCheck ...) => (LoweredNilCheck ...)
(IsNonNil ptr) => (NotEqual (CMPconst [0] ptr))
(IsInBounds      idx len) => (LessThanU  (CMP idx len))
(IsSliceInBounds idx len) => (LessEqualU (CMP idx len))

// pseudo-ops
(GetClosurePtr ...) => (LoweredGetClosurePtr ...)
(GetCallerSP   ...) => (LoweredGetCallerSP   ...)
(GetCallerPC   ...) => (LoweredGetCallerPC   ...)
(MemEq ...) => (LoweredMemEq ...)

// Absorb pseudo-ops into blocks.
(If (Equal         cc) yes no) => (EQ cc yes no)
(If (NotEqual      cc) yes no) => (NE cc yes no)
(If (LessThan      cc) yes no) => (LT cc yes no)
(If (LessThanU     cc) yes no) => (ULT cc yes no)
(If (LessEqual     cc) yes no) => (LE cc yes no)
(If (LessEqualU    cc) yes no) => (ULE cc yes no)
(If (GreaterThan   cc) yes no) => (GT cc yes no)
(If (GreaterThanU  cc) yes no) => (UGT cc yes no)
(If (GreaterEqual  cc) yes no) => (GE cc yes no)
(If (GreaterEqualU cc) yes no) => (UGE cc yes no)
(If (LessThanF     cc) yes no) => (FLT cc yes no)
(If (LessEqualF    cc) yes no) => (FLE cc yes no)
(If (GreaterThanF  cc) yes no) => (FGT cc yes no)
(If (GreaterEqualF cc) yes no) => (FGE cc yes no)

(If cond yes no) => (TBNZ [0] cond yes no)

(JumpTable idx) => (JUMPTABLE {ssa.MakeJumpTableSym(b)} idx (MOVDaddr <typ.Uintptr> {ssa.MakeJumpTableSym(b)} (SB)))

// atomic intrinsics
// Note: these ops do not accept offset.
(AtomicLoad8   ...) => (LDARB ...)
(AtomicLoad32  ...) => (LDARW ...)
(AtomicLoad64  ...) => (LDAR  ...)
(AtomicLoadPtr ...) => (LDAR  ...)

(AtomicStore8       ...) => (STLRB ...)
(AtomicStore32      ...) => (STLRW ...)
(AtomicStore64      ...) => (STLR  ...)
(AtomicStorePtrNoWB ...) => (STLR  ...)

(AtomicExchange(8|32|64)       ...) => (LoweredAtomicExchange(8|32|64) ...)
(AtomicAdd(32|64)            ...) => (LoweredAtomicAdd(32|64)      ...)
(AtomicCompareAndSwap(32|64) ...) => (LoweredAtomicCas(32|64)      ...)

(AtomicAdd(32|64)Variant            ...) => (LoweredAtomicAdd(32|64)Variant      ...)
(AtomicExchange(8|32|64)Variant       ...) => (LoweredAtomicExchange(8|32|64)Variant ...)
(AtomicCompareAndSwap(32|64)Variant ...) => (LoweredAtomicCas(32|64)Variant      ...)

// Return old contents.
(AtomicAnd(64|32|8)value            ...) => (LoweredAtomicAnd(64|32|8)            ...)
(AtomicOr(64|32|8)value             ...) => (LoweredAtomicOr(64|32|8)             ...)
(AtomicAnd(64|32|8)valueVariant     ...) => (LoweredAtomicAnd(64|32|8)Variant     ...)
(AtomicOr(64|32|8)valueVariant      ...) => (LoweredAtomicOr(64|32|8)Variant      ...)

// Write barrier.
(WB ...) => (LoweredWB ...)

// Publication barrier (0xe is ST option)
(PubBarrier mem) => (DMB [0xe] mem)

(PanicBounds ...) => (LoweredPanicBoundsRR ...)
(LoweredPanicBoundsRR [kind] x (MOVDconst [c]) mem) => (LoweredPanicBoundsRC [kind] x {ssa.PanicBoundsC{C:c}} mem)
(LoweredPanicBoundsRR [kind] (MOVDconst [c]) y mem) => (LoweredPanicBoundsCR [kind] {ssa.PanicBoundsC{C:c}} y mem)
(LoweredPanicBoundsRC [kind] {p} (MOVDconst [c]) mem) => (LoweredPanicBoundsCC [kind] {ssa.PanicBoundsCC{Cx:c, Cy:p.C}} mem)
(LoweredPanicBoundsCR [kind] {p} (MOVDconst [c]) mem) => (LoweredPanicBoundsCC [kind] {ssa.PanicBoundsCC{Cx:p.C, Cy:c}} mem)

// Optimizations

// Replace widen -> wide_unop -> narrow with narrow_unop when one exists.
(FCVTDS (F(ABS|SQRT|RINTP|RINTM|RINTA|RINTN|RINTZ)D (FCVTSD x))) => 
    (F(ABS|SQRT|RINTP|RINTM|RINTA|RINTN|RINTZ)S x)


// Absorb boolean tests into block
(NZ (Equal         cc) yes no) => (EQ  cc yes no)
(NZ (NotEqual      cc) yes no) => (NE  cc yes no)
(NZ (LessThan      cc) yes no) => (LT  cc yes no)
(NZ (LessThanU     cc) yes no) => (ULT cc yes no)
(NZ (LessEqual     cc) yes no) => (LE  cc yes no)
(NZ (LessEqualU    cc) yes no) => (ULE cc yes no)
(NZ (GreaterThan   cc) yes no) => (GT  cc yes no)
(NZ (GreaterThanU  cc) yes no) => (UGT cc yes no)
(NZ (GreaterEqual  cc) yes no) => (GE  cc yes no)
(NZ (GreaterEqualU cc) yes no) => (UGE cc yes no)
(NZ (LessThanF     cc) yes no) => (FLT cc yes no)
(NZ (LessEqualF    cc) yes no) => (FLE cc yes no)
(NZ (GreaterThanF  cc) yes no) => (FGT cc yes no)
(NZ (GreaterEqualF cc) yes no) => (FGE cc yes no)

(TBNZ [0] (Equal         cc) yes no) => (EQ  cc yes no)
(TBNZ [0] (NotEqual      cc) yes no) => (NE  cc yes no)
(TBNZ [0] (LessThan      cc) yes no) => (LT  cc yes no)
(TBNZ [0] (LessThanU     cc) yes no) => (ULT cc yes no)
(TBNZ [0] (LessEqual     cc) yes no) => (LE  cc yes no)
(TBNZ [0] (LessEqualU    cc) yes no) => (ULE cc yes no)
(TBNZ [0] (GreaterThan   cc) yes no) => (GT  cc yes no)
(TBNZ [0] (GreaterThanU  cc) yes no) => (UGT cc yes no)
(TBNZ [0] (GreaterEqual  cc) yes no) => (GE  cc yes no)
(TBNZ [0] (GreaterEqualU cc) yes no) => (UGE cc yes no)
(TBNZ [0] (LessThanF     cc) yes no) => (FLT cc yes no)
(TBNZ [0] (LessEqualF    cc) yes no) => (FLE cc yes no)
(TBNZ [0] (GreaterThanF  cc) yes no) => (FGT cc yes no)
(TBNZ [0] (GreaterEqualF cc) yes no) => (FGE cc yes no)

(TB(Z|NZ) [0] (XORconst [1] x) yes no) => (TB(NZ|Z) [0] x yes no)

((EQ|NE|LT|LE|GT|GE) (CMPconst  [0] z:(AND        x y)) yes no) && z.Uses == 1 => ((EQ|NE|LT|LE|GT|GE) (TST                x y) yes no)
((EQ|NE|LT|LE|GT|GE) (CMPconst  [0] x:(ANDconst [c] y)) yes no) && x.Uses == 1 => ((EQ|NE|LT|LE|GT|GE) (TSTconst         [c] y) yes no)
((EQ|NE|LT|LE|GT|GE) (CMPWconst [0] z:(AND        x y)) yes no) && z.Uses == 1 => ((EQ|NE|LT|LE|GT|GE) (TSTW               x y) yes no)
((EQ|NE|LT|LE|GT|GE) (CMPWconst [0] x:(ANDconst [c] y)) yes no) && x.Uses == 1 => ((EQ|NE|LT|LE|GT|GE) (TSTWconst [int32(c)] y) yes no)

// For conditional instructions such as CSET, CSEL.
((Equal|NotEqual|LessThan|LessEqual|GreaterThan|GreaterEqual) (CMPconst [0]  z:(AND        x y))) && z.Uses == 1 =>
	((Equal|NotEqual|LessThan|LessEqual|GreaterThan|GreaterEqual) (TST x y))
((Equal|NotEqual|LessThan|LessEqual|GreaterThan|GreaterEqual) (CMPWconst [0] x:(ANDconst [c] y))) && x.Uses == 1 =>
	((Equal|NotEqual|LessThan|LessEqual|GreaterThan|GreaterEqual) (TSTWconst [int32(c)] y))
((Equal|NotEqual|LessThan|LessEqual|GreaterThan|GreaterEqual) (CMPWconst [0] z:(AND        x y))) && z.Uses == 1 =>
	((Equal|NotEqual|LessThan|LessEqual|GreaterThan|GreaterEqual) (TSTW x y))
((Equal|NotEqual|LessThan|LessEqual|GreaterThan|GreaterEqual) (CMPconst [0]  x:(ANDconst [c] y))) && x.Uses == 1 =>
	((Equal|NotEqual|LessThan|LessEqual|GreaterThan|GreaterEqual) (TSTconst [c] y))

((EQ|NE|LT|LE|GT|GE) (CMPconst  [0] x:(ADDconst [c] y)) yes no) && x.Uses == 1 => ((EQ|NE|LTnoov|LEnoov|GTnoov|GEnoov) (CMNconst         [c] y) yes no)
((EQ|NE|LT|LE|GT|GE) (CMPWconst [0] x:(ADDconst [c] y)) yes no) && x.Uses == 1 => ((EQ|NE|LTnoov|LEnoov|GTnoov|GEnoov) (CMNWconst [int32(c)] y) yes no)
((EQ|NE|LT|LE|GT|GE) (CMPconst  [0] z:(ADD        x y)) yes no) && z.Uses == 1 => ((EQ|NE|LTnoov|LEnoov|GTnoov|GEnoov) (CMN                x y) yes no)
((EQ|NE|LT|LE|GT|GE) (CMPWconst [0] z:(ADD        x y)) yes no) && z.Uses == 1 => ((EQ|NE|LTnoov|LEnoov|GTnoov|GEnoov) (CMNW               x y) yes no)

// CMP(x,-y) -> CMN(x,y) is only valid for unordered comparison, if y can be -1<<63
((EQ|NE) (CMP x z:(NEG y)) yes no)   && z.Uses == 1 => ((EQ|NE) (CMN x y) yes no)
((Equal|NotEqual) (CMP x z:(NEG y))) && z.Uses == 1 => ((Equal|NotEqual) (CMN x y))

// CMPW(x,-y) -> CMNW(x,y) is only valid for unordered comparison, if y can be -1<<31
((EQ|NE) (CMPW x z:(NEG y)) yes no)   && z.Uses == 1 => ((EQ|NE) (CMNW x y) yes no)
((Equal|NotEqual) (CMPW x z:(NEG y))) && z.Uses == 1 => ((Equal|NotEqual) (CMNW x y))

// For conditional instructions such as CSET, CSEL.
// TODO: add support for LE, GT, overflow needs to be considered.
((Equal|NotEqual|LessThan|GreaterEqual) (CMPconst  [0] x:(ADDconst [c] y))) && x.Uses == 1 => ((Equal|NotEqual|LessThanNoov|GreaterEqualNoov) (CMNconst [c] y))
((Equal|NotEqual|LessThan|GreaterEqual) (CMPWconst [0] x:(ADDconst [c] y))) && x.Uses == 1 => ((Equal|NotEqual|LessThanNoov|GreaterEqualNoov) (CMNWconst [int32(c)] y))
((Equal|NotEqual|LessThan|GreaterEqual) (CMPconst  [0] z:(ADD        x y))) && z.Uses == 1 => ((Equal|NotEqual|LessThanNoov|GreaterEqualNoov) (CMN  x y))
((Equal|NotEqual|LessThan|GreaterEqual) (CMPWconst [0] z:(ADD        x y))) && z.Uses == 1 => ((Equal|NotEqual|LessThanNoov|GreaterEqualNoov) (CMNW x y))
((Equal|NotEqual|LessThan|GreaterEqual) (CMPconst  [0] z:(MADD     a x y))) && z.Uses == 1 => ((Equal|NotEqual|LessThanNoov|GreaterEqualNoov) (CMN  a (MUL  <x.Type> x y)))
((Equal|NotEqual|LessThan|GreaterEqual) (CMPWconst [0] z:(MADDW    a x y))) && z.Uses == 1 => ((Equal|NotEqual|LessThanNoov|GreaterEqualNoov) (CMNW a (MULW <x.Type> x y)))

// Skip LessThan/GreaterEqual in the following rewrites.
// MSUB[W] a x y is a-x*y, which folds to a CMP.
// That can end up being reversed into InvertFlags.
// But absorbing InvertFlags into a no-overflow comparison flips < and >,
// which is wrong when the difference is the minimum int.
((Equal|NotEqual) (CMPconst  [0] z:(MSUB     a x y))) && z.Uses == 1 => ((Equal|NotEqual) (CMP  a (MUL  <x.Type> x y)))
((Equal|NotEqual) (CMPWconst [0] z:(MSUBW    a x y))) && z.Uses == 1 => ((Equal|NotEqual) (CMPW a (MULW <x.Type> x y)))

((CMPconst|CMNconst)   [c] y) && c < 0 && c != -1<<63 => ((CMNconst|CMPconst)   [-c] y)
((CMPWconst|CMNWconst) [c] y) && c < 0 && c != -1<<31 => ((CMNWconst|CMPWconst) [-c] y)

((EQ|NE) (CMPconst  [0] x) yes no) => ((Z|NZ)   x yes no)
((EQ|NE) (CMPWconst [0] x) yes no) => ((ZW|NZW) x yes no)

((ULE|UGT) (CMPconst  [0] x)) => ((EQ|NE) (CMPconst  [0] x))
((ULE|UGT) (CMPWconst [0] x)) => ((EQ|NE) (CMPWconst [0] x))

((Z|NZ)   sub:(SUB        x y)) && sub.Uses == 1 => ((EQ|NE) (CMP                x y))
((ZW|NZW) sub:(SUB        x y)) && sub.Uses == 1 => ((EQ|NE) (CMPW               x y))
((Z|NZ)   sub:(SUBconst [c] y)) && sub.Uses == 1 => ((EQ|NE) (CMPconst         [c] y))
((ZW|NZW) sub:(SUBconst [c] y)) && sub.Uses == 1 => ((EQ|NE) (CMPWconst [int32(c)] y))

((EQ|NE|LT|LE|GT|GE) (CMPconst  [0] z:(MADD a x y))  yes no) && z.Uses==1 => ((EQ|NE|LTnoov|LEnoov|GTnoov|GEnoov) (CMN  a (MUL  <x.Type> x y)) yes no)
((EQ|NE|LT|LE|GT|GE) (CMPWconst [0] z:(MADDW a x y)) yes no) && z.Uses==1 => ((EQ|NE|LTnoov|LEnoov|GTnoov|GEnoov) (CMNW a (MULW <x.Type> x y)) yes no)
// No ordering ops here; see comments above about MSUB[W].
((EQ|NE) (CMPconst  [0] z:(MSUB a x y))  yes no) && z.Uses==1 => ((EQ|NE) (CMP  a (MUL  <x.Type> x y)) yes no)
((EQ|NE) (CMPWconst [0] z:(MSUBW a x y)) yes no) && z.Uses==1 => ((EQ|NE) (CMPW a (MULW <x.Type> x y)) yes no)

// Absorb bit-tests into block
(Z   (ANDconst  [c] x) yes no) && ssa.OneBit(c) => (TBZ  [int64(ssa.Ntz64(c))] x yes no)
(NZ  (ANDconst  [c] x) yes no) && ssa.OneBit(c) => (TBNZ [int64(ssa.Ntz64(c))] x yes no)
(ZW  (ANDconst  [c] x) yes no) && ssa.OneBit(int64(uint32(c))) => (TBZ  [int64(ssa.Ntz64(int64(uint32(c))))] x yes no)
(NZW (ANDconst  [c] x) yes no) && ssa.OneBit(int64(uint32(c))) => (TBNZ [int64(ssa.Ntz64(int64(uint32(c))))] x yes no)
(EQ  (TSTconst  [c] x) yes no) && ssa.OneBit(c) => (TBZ  [int64(ssa.Ntz64(c))] x yes no)
(NE  (TSTconst  [c] x) yes no) && ssa.OneBit(c) => (TBNZ [int64(ssa.Ntz64(c))] x yes no)
(EQ  (TSTWconst [c] x) yes no) && ssa.OneBit(int64(uint32(c))) => (TBZ  [int64(ssa.Ntz64(int64(uint32(c))))] x yes no)
(NE  (TSTWconst [c] x) yes no) && ssa.OneBit(int64(uint32(c))) => (TBNZ [int64(ssa.Ntz64(int64(uint32(c))))] x yes no)

((Z|NZ) s:(SRLconst [63] x) yes no) && s.Uses == 1 => (TB(Z|NZ) [63] x yes no)
((Z|NZ) s:(SRAconst [63] x) yes no) && s.Uses == 1 => (TB(Z|NZ) [63] x yes no)

// Merge more operations into TBZ & TBNZ
(TB(Z|NZ) [t] sv:(SRLconst [s] x) yes no) && t+s <  64 && sv.Uses == 1 => (TB(Z|NZ) [t+s] x yes no )
(TBZ      [t]    (SRLconst [s] x) yes no) && t+s >= 64                 => (First            yes no )
(TBNZ     [t]    (SRLconst [s] x) yes no) && t+s >= 64                 => (First            no  yes)

(TB(Z|NZ) [t] sv:(SLLconst [s] x) yes no) && t-s >= 0  && sv.Uses == 1 => (TB(Z|NZ) [t-s] x yes no )
(TBZ      [t]    (SLLconst [s] x) yes no) && t-s <  0                  => (First            yes no )
(TBNZ     [t]    (SLLconst [s] x) yes no) && t-s <  0                  => (First            no  yes)

(TB(Z|NZ) [t] rv:(RORconst [r] x) yes no) && rv.Uses == 1 => (TB(Z|NZ) [int64(uint64(t+r)%64)] x yes no)

(TB(Z|NZ) [t] sv:(SRAconst [s] x) yes no) && t+s <  64 && sv.Uses == 1 => (TB(Z|NZ) [t+s] x yes no)
(TB(Z|NZ) [t] sv:(SRAconst [s] x) yes no) && t+s >= 64 && sv.Uses == 1 => (TB(Z|NZ) [63 ] x yes no)

// Test sign-bit for signed comparisons against zero
(GE (CMPWconst [0] x) yes no) => (TBZ  [31] x yes no)
(GE (CMPconst [0] x)  yes no) => (TBZ  [63] x yes no)
(LT (CMPWconst [0] x) yes no) => (TBNZ [31] x yes no)
(LT (CMPconst [0] x)  yes no) => (TBNZ [63] x yes no)

// A byte value (a zero-extended-byte op, so provably in [0,255]) compared
// against 128 (=2^7) is a bit-7 test, for both compare widths and both
// signednesses: with the value in [0,255] the signed and unsigned orderings
// against 128 coincide.
((UGE|ULT) (CMPWconst [128] x) yes no) && ssa.ZeroUpper56Bits(x) => ((TBNZ|TBZ) [7] x yes no)
((UGE|ULT) (CMPconst [128] x)  yes no) && ssa.ZeroUpper56Bits(x) => ((TBNZ|TBZ) [7] x yes no)
((GE|LT)   (CMPWconst [128] x) yes no) && ssa.ZeroUpper56Bits(x) => ((TBNZ|TBZ) [7] x yes no)
((GE|LT)   (CMPconst [128] x)  yes no) && ssa.ZeroUpper56Bits(x) => ((TBNZ|TBZ) [7] x yes no)

// fold offset into address
(ADDconst [off1] (MOVDaddr [off2] {sym} ptr)) && ssa.Is32Bit(off1+int64(off2)) =>
	 (MOVDaddr [int32(off1)+off2] {sym} ptr)

// fold address into load/store.
// Do not fold global variable access in -dynlink mode, where it will
// be rewritten to use the GOT via REGTMP, which currently cannot handle
// large offset.
(MOVBload [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVBload [off1+int32(off2)] {sym} ptr mem)
(MOVBUload [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVBUload [off1+int32(off2)] {sym} ptr mem)
(MOVHload [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVHload [off1+int32(off2)] {sym} ptr mem)
(MOVHUload [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVHUload [off1+int32(off2)] {sym} ptr mem)
(MOVWload [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVWload [off1+int32(off2)] {sym} ptr mem)
(MOVWUload [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVWUload [off1+int32(off2)] {sym} ptr mem)
(MOVDload [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVDload [off1+int32(off2)] {sym} ptr mem)
(LDP [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(LDP [off1+int32(off2)] {sym} ptr mem)
(FMOVSload [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVSload [off1+int32(off2)] {sym} ptr mem)
(FMOVDload [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVDload [off1+int32(off2)] {sym} ptr mem)
(FMOVQload [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVQload [off1+int32(off2)] {sym} ptr mem)
(FLDPQ [off1] {sym} (ADDconst [off2] ptr) mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FLDPQ [off1+int32(off2)] {sym} ptr mem)

// register indexed load
(MOVDload  [off] {sym} (ADD ptr idx) mem) && off == 0 && sym == nil => (MOVDloadidx ptr idx mem)
(MOVWUload [off] {sym} (ADD ptr idx) mem) && off == 0 && sym == nil => (MOVWUloadidx ptr idx mem)
(MOVWload  [off] {sym} (ADD ptr idx) mem) && off == 0 && sym == nil => (MOVWloadidx ptr idx mem)
(MOVHUload [off] {sym} (ADD ptr idx) mem) && off == 0 && sym == nil => (MOVHUloadidx ptr idx mem)
(MOVHload  [off] {sym} (ADD ptr idx) mem) && off == 0 && sym == nil => (MOVHloadidx ptr idx mem)
(MOVBUload [off] {sym} (ADD ptr idx) mem) && off == 0 && sym == nil => (MOVBUloadidx ptr idx mem)
(MOVBload  [off] {sym} (ADD ptr idx) mem) && off == 0 && sym == nil => (MOVBloadidx ptr idx mem)
(FMOVSload [off] {sym} (ADD ptr idx) mem) && off == 0 && sym == nil => (FMOVSloadidx ptr idx mem)
(FMOVDload [off] {sym} (ADD ptr idx) mem) && off == 0 && sym == nil => (FMOVDloadidx ptr idx mem)

(MOVDloadidx  ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c) => (MOVDload  [int32(c)] ptr mem)
(MOVDloadidx  (MOVDconst [c]) ptr mem) && ssa.Is32Bit(c) => (MOVDload  [int32(c)] ptr mem)
(MOVWUloadidx ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c) => (MOVWUload [int32(c)] ptr mem)
(MOVWUloadidx (MOVDconst [c]) ptr mem) && ssa.Is32Bit(c) => (MOVWUload [int32(c)] ptr mem)
(MOVWloadidx  ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c) => (MOVWload  [int32(c)] ptr mem)
(MOVWloadidx  (MOVDconst [c]) ptr mem) && ssa.Is32Bit(c) => (MOVWload  [int32(c)] ptr mem)
(MOVHUloadidx ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c) => (MOVHUload [int32(c)] ptr mem)
(MOVHUloadidx (MOVDconst [c]) ptr mem) && ssa.Is32Bit(c) => (MOVHUload [int32(c)] ptr mem)
(MOVHloadidx  ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c) => (MOVHload  [int32(c)] ptr mem)
(MOVHloadidx  (MOVDconst [c]) ptr mem) && ssa.Is32Bit(c) => (MOVHload  [int32(c)] ptr mem)
(MOVBUloadidx ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c) => (MOVBUload [int32(c)] ptr mem)
(MOVBUloadidx (MOVDconst [c]) ptr mem) && ssa.Is32Bit(c) => (MOVBUload [int32(c)] ptr mem)
(MOVBloadidx  ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c) => (MOVBload  [int32(c)] ptr mem)
(MOVBloadidx  (MOVDconst [c]) ptr mem) && ssa.Is32Bit(c) => (MOVBload  [int32(c)] ptr mem)
(FMOVSloadidx ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c) => (FMOVSload [int32(c)] ptr mem)
(FMOVSloadidx (MOVDconst [c]) ptr mem) && ssa.Is32Bit(c) => (FMOVSload [int32(c)] ptr mem)
(FMOVDloadidx ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c) => (FMOVDload [int32(c)] ptr mem)
(FMOVDloadidx (MOVDconst [c]) ptr mem) && ssa.Is32Bit(c) => (FMOVDload [int32(c)] ptr mem)

// shifted register indexed load
(MOVDload  [off] {sym} (ADDshiftLL [3] ptr idx) mem) && off == 0 && sym == nil => (MOVDloadidx8 ptr idx mem)
(MOVWUload [off] {sym} (ADDshiftLL [2] ptr idx) mem) && off == 0 && sym == nil => (MOVWUloadidx4 ptr idx mem)
(MOVWload  [off] {sym} (ADDshiftLL [2] ptr idx) mem) && off == 0 && sym == nil => (MOVWloadidx4 ptr idx mem)
(MOVHUload [off] {sym} (ADDshiftLL [1] ptr idx) mem) && off == 0 && sym == nil => (MOVHUloadidx2 ptr idx mem)
(MOVHload  [off] {sym} (ADDshiftLL [1] ptr idx) mem) && off == 0 && sym == nil => (MOVHloadidx2 ptr idx mem)
(MOVDloadidx  ptr (SLLconst [3] idx) mem) => (MOVDloadidx8 ptr idx mem)
(MOVWloadidx  ptr (SLLconst [2] idx) mem) => (MOVWloadidx4 ptr idx mem)
(MOVWUloadidx ptr (SLLconst [2] idx) mem) => (MOVWUloadidx4 ptr idx mem)
(MOVHloadidx  ptr (SLLconst [1] idx) mem) => (MOVHloadidx2 ptr idx mem)
(MOVHUloadidx ptr (SLLconst [1] idx) mem) => (MOVHUloadidx2 ptr idx mem)
(MOVHloadidx  ptr (ADD idx idx) mem) => (MOVHloadidx2 ptr idx mem)
(MOVHUloadidx ptr (ADD idx idx) mem) => (MOVHUloadidx2 ptr idx mem)
(MOVDloadidx  (SLLconst [3] idx) ptr mem) => (MOVDloadidx8 ptr idx mem)
(MOVWloadidx  (SLLconst [2] idx) ptr mem) => (MOVWloadidx4 ptr idx mem)
(MOVWUloadidx (SLLconst [2] idx) ptr mem) => (MOVWUloadidx4 ptr idx mem)
(MOVHloadidx  (ADD idx idx) ptr mem) => (MOVHloadidx2 ptr idx mem)
(MOVHUloadidx (ADD idx idx) ptr mem) => (MOVHUloadidx2 ptr idx mem)
(MOVDloadidx8  ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c<<3) => (MOVDload  [int32(c)<<3] ptr mem)
(MOVWUloadidx4 ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c<<2) => (MOVWUload [int32(c)<<2] ptr mem)
(MOVWloadidx4  ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c<<2) => (MOVWload  [int32(c)<<2] ptr mem)
(MOVHUloadidx2 ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c<<1) => (MOVHUload [int32(c)<<1] ptr mem)
(MOVHloadidx2  ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c<<1) => (MOVHload  [int32(c)<<1] ptr mem)

(FMOVDload [off] {sym} (ADDshiftLL [3] ptr idx) mem) && off == 0 && sym == nil => (FMOVDloadidx8 ptr idx mem)
(FMOVSload [off] {sym} (ADDshiftLL [2] ptr idx) mem) && off == 0 && sym == nil => (FMOVSloadidx4 ptr idx mem)
(FMOVDloadidx ptr (SLLconst [3] idx) mem) => (FMOVDloadidx8 ptr idx mem)
(FMOVSloadidx ptr (SLLconst [2] idx) mem) => (FMOVSloadidx4 ptr idx mem)
(FMOVDloadidx (SLLconst [3] idx) ptr mem) => (FMOVDloadidx8 ptr idx mem)
(FMOVSloadidx (SLLconst [2] idx) ptr mem) => (FMOVSloadidx4 ptr idx mem)
(FMOVDloadidx8 ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c<<3) => (FMOVDload ptr [int32(c)<<3] mem)
(FMOVSloadidx4 ptr (MOVDconst [c]) mem) && ssa.Is32Bit(c<<2) => (FMOVSload ptr [int32(c)<<2] mem)

(MOVBstore [off1] {sym} (ADDconst [off2] ptr) val mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVBstore [off1+int32(off2)] {sym} ptr val mem)
(MOVHstore [off1] {sym} (ADDconst [off2] ptr) val mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVHstore [off1+int32(off2)] {sym} ptr val mem)
(MOVWstore [off1] {sym} (ADDconst [off2] ptr) val mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVWstore [off1+int32(off2)] {sym} ptr val mem)
(MOVDstore [off1] {sym} (ADDconst [off2] ptr) val mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVDstore [off1+int32(off2)] {sym} ptr val mem)
(STP [off1] {sym} (ADDconst [off2] ptr) val1 val2 mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(STP [off1+int32(off2)] {sym} ptr val1 val2 mem)
(FMOVSstore [off1] {sym} (ADDconst [off2] ptr) val mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVSstore [off1+int32(off2)] {sym} ptr val mem)
(FMOVDstore [off1] {sym} (ADDconst [off2] ptr) val mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVDstore [off1+int32(off2)] {sym} ptr val mem)
(FMOVQstore [off1] {sym} (ADDconst [off2] ptr) val mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVQstore [off1+int32(off2)] {sym} ptr val mem)
(FSTPQ [off1] {sym} (ADDconst [off2] ptr) val1 val2 mem) && ssa.Is32Bit(int64(off1)+off2)
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FSTPQ [off1+int32(off2)] {sym} ptr val1 val2 mem)

// register indexed store
(MOVDstore  [off] {sym} (ADD ptr idx) val mem) && off == 0 && sym == nil => (MOVDstoreidx ptr idx val mem)
(MOVWstore  [off] {sym} (ADD ptr idx) val mem) && off == 0 && sym == nil => (MOVWstoreidx ptr idx val mem)
(MOVHstore  [off] {sym} (ADD ptr idx) val mem) && off == 0 && sym == nil => (MOVHstoreidx ptr idx val mem)
(MOVBstore  [off] {sym} (ADD ptr idx) val mem) && off == 0 && sym == nil => (MOVBstoreidx ptr idx val mem)
(FMOVDstore [off] {sym} (ADD ptr idx) val mem) && off == 0 && sym == nil => (FMOVDstoreidx ptr idx val mem)
(FMOVSstore [off] {sym} (ADD ptr idx) val mem) && off == 0 && sym == nil => (FMOVSstoreidx ptr idx val mem)
(MOVDstoreidx  ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c) => (MOVDstore  [int32(c)] ptr val mem)
(MOVDstoreidx  (MOVDconst [c]) idx val mem) && ssa.Is32Bit(c) => (MOVDstore  [int32(c)] idx val mem)
(MOVWstoreidx  ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c) => (MOVWstore  [int32(c)] ptr val mem)
(MOVWstoreidx  (MOVDconst [c]) idx val mem) && ssa.Is32Bit(c) => (MOVWstore  [int32(c)] idx val mem)
(MOVHstoreidx  ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c) => (MOVHstore  [int32(c)] ptr val mem)
(MOVHstoreidx  (MOVDconst [c]) idx val mem) && ssa.Is32Bit(c) => (MOVHstore  [int32(c)] idx val mem)
(MOVBstoreidx  ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c) => (MOVBstore  [int32(c)] ptr val mem)
(MOVBstoreidx  (MOVDconst [c]) idx val mem) && ssa.Is32Bit(c) => (MOVBstore  [int32(c)] idx val mem)
(FMOVDstoreidx ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c) => (FMOVDstore [int32(c)] ptr val mem)
(FMOVDstoreidx (MOVDconst [c]) idx val mem) && ssa.Is32Bit(c) => (FMOVDstore [int32(c)] idx val mem)
(FMOVSstoreidx ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c) => (FMOVSstore [int32(c)] ptr val mem)
(FMOVSstoreidx (MOVDconst [c]) idx val mem) && ssa.Is32Bit(c) => (FMOVSstore [int32(c)] idx val mem)

// shifted register indexed store
(MOVDstore [off] {sym} (ADDshiftLL [3] ptr idx) val mem) && off == 0 && sym == nil => (MOVDstoreidx8 ptr idx val mem)
(MOVWstore [off] {sym} (ADDshiftLL [2] ptr idx) val mem) && off == 0 && sym == nil => (MOVWstoreidx4 ptr idx val mem)
(MOVHstore [off] {sym} (ADDshiftLL [1] ptr idx) val mem) && off == 0 && sym == nil => (MOVHstoreidx2 ptr idx val mem)
(MOVDstoreidx  ptr (SLLconst [3] idx) val mem) => (MOVDstoreidx8 ptr idx val mem)
(MOVWstoreidx  ptr (SLLconst [2] idx) val mem) => (MOVWstoreidx4 ptr idx val mem)
(MOVHstoreidx  ptr (SLLconst [1] idx) val mem) => (MOVHstoreidx2 ptr idx val mem)
(MOVHstoreidx  ptr (ADD      idx idx) val mem) => (MOVHstoreidx2 ptr idx val mem)
(MOVDstoreidx  (SLLconst [3] idx) ptr val mem) => (MOVDstoreidx8 ptr idx val mem)
(MOVWstoreidx  (SLLconst [2] idx) ptr val mem) => (MOVWstoreidx4 ptr idx val mem)
(MOVHstoreidx  (SLLconst [1] idx) ptr val mem) => (MOVHstoreidx2 ptr idx val mem)
(MOVHstoreidx  (ADD      idx idx) ptr val mem) => (MOVHstoreidx2 ptr idx val mem)
(MOVDstoreidx8 ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c<<3) => (MOVDstore [int32(c)<<3] ptr val mem)
(MOVWstoreidx4 ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c<<2) => (MOVWstore [int32(c)<<2] ptr val mem)
(MOVHstoreidx2 ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c<<1) => (MOVHstore [int32(c)<<1] ptr val mem)

(FMOVDstore [off] {sym} (ADDshiftLL [3] ptr idx) val mem) && off == 0 && sym == nil => (FMOVDstoreidx8 ptr idx val mem)
(FMOVSstore [off] {sym} (ADDshiftLL [2] ptr idx) val mem) && off == 0 && sym == nil => (FMOVSstoreidx4 ptr idx val mem)
(FMOVDstoreidx ptr (SLLconst [3] idx) val mem) => (FMOVDstoreidx8 ptr idx val mem)
(FMOVSstoreidx ptr (SLLconst [2] idx) val mem) => (FMOVSstoreidx4 ptr idx val mem)
(FMOVDstoreidx (SLLconst [3] idx) ptr val mem) => (FMOVDstoreidx8 ptr idx val mem)
(FMOVSstoreidx (SLLconst [2] idx) ptr val mem) => (FMOVSstoreidx4 ptr idx val mem)
(FMOVDstoreidx8 ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c<<3) => (FMOVDstore [int32(c)<<3] ptr val mem)
(FMOVSstoreidx4 ptr (MOVDconst [c]) val mem) && ssa.Is32Bit(c<<2) => (FMOVSstore [int32(c)<<2] ptr val mem)

(MOVBload [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVBload [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(MOVBUload [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVBUload [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(MOVHload [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVHload [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(MOVHUload [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVHUload [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(MOVWload [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVWload [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(MOVWUload [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVWUload [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(MOVDload [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVDload [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(LDP [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(LDP [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(FMOVSload [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVSload [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(FMOVDload [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVDload [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(FMOVQload [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVQload [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)
(FLDPQ [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FLDPQ [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr mem)

(MOVBstore [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) val mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVBstore [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr val mem)
(MOVHstore [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) val mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVHstore [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr val mem)
(MOVWstore [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) val mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVWstore [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr val mem)
(MOVDstore [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) val mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(MOVDstore [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr val mem)
(STP [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) val1 val2 mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(STP [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr val1 val2 mem)
(FMOVSstore [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) val mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVSstore [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr val mem)
(FMOVDstore [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) val mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVDstore [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr val mem)
(FMOVQstore [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) val mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FMOVQstore [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr val mem)
(FSTPQ [off1] {sym1} (MOVDaddr [off2] {sym2} ptr) val1 val2 mem)
	&& ssa.CanMergeSym(sym1,sym2) && ssa.Is32Bit(int64(off1)+int64(off2))
	&& (ptr.Op != ssaop.OpSB || !config.Ctxt.Flag_dynlink) =>
	(FSTPQ [off1+off2] {ssa.MergeSym(sym1,sym2)} ptr val1 val2 mem)

// replace load from same location as preceding store with zero/sign extension (or copy in case of full width)
// these seem to have bad interaction with other rules, resulting in slower code
//(MOVBload  [off] {sym} ptr (MOVBstore  [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && ssa.IsSamePtr(ptr, ptr2) -> (MOVBreg x)
//(MOVBUload [off] {sym} ptr (MOVBstore  [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && ssa.IsSamePtr(ptr, ptr2) -> (MOVBUreg x)
//(MOVHload  [off] {sym} ptr (MOVHstore  [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && ssa.IsSamePtr(ptr, ptr2) -> (MOVHreg x)
//(MOVHUload [off] {sym} ptr (MOVHstore  [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && ssa.IsSamePtr(ptr, ptr2) -> (MOVHUreg x)
//(MOVWload  [off] {sym} ptr (MOVWstore  [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && ssa.IsSamePtr(ptr, ptr2) -> (MOVWreg x)
//(MOVWUload [off] {sym} ptr (MOVWstore  [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && ssa.IsSamePtr(ptr, ptr2) -> (MOVWUreg x)
//(MOVDload  [off] {sym} ptr (MOVDstore  [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && ssa.IsSamePtr(ptr, ptr2) -> x
//(FMOVSload [off] {sym} ptr (FMOVSstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && ssa.IsSamePtr(ptr, ptr2) -> x
//(FMOVDload [off] {sym} ptr (FMOVDstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && ssa.IsSamePtr(ptr, ptr2) -> x
//(LDP       [off] {sym} ptr (STP      [off2] {sym2} ptr2 x y _)) && sym == sym2 && off == off2 && ssa.IsSamePtr(ptr, ptr2) -> x y

// don't extend before store
(MOVBstore [off] {sym} ptr (MOVBreg  x) mem) => (MOVBstore [off] {sym} ptr x mem)
(MOVBstore [off] {sym} ptr (MOVBUreg x) mem) => (MOVBstore [off] {sym} ptr x mem)
(MOVBstore [off] {sym} ptr (MOVHreg  x) mem) => (MOVBstore [off] {sym} ptr x mem)
(MOVBstore [off] {sym} ptr (MOVHUreg x) mem) => (MOVBstore [off] {sym} ptr x mem)
(MOVBstore [off] {sym} ptr (MOVWreg  x) mem) => (MOVBstore [off] {sym} ptr x mem)
(MOVBstore [off] {sym} ptr (MOVWUreg x) mem) => (MOVBstore [off] {sym} ptr x mem)
(MOVHstore [off] {sym} ptr (MOVHreg  x) mem) => (MOVHstore [off] {sym} ptr x mem)
(MOVHstore [off] {sym} ptr (MOVHUreg x) mem) => (MOVHstore [off] {sym} ptr x mem)
(MOVHstore [off] {sym} ptr (MOVWreg  x) mem) => (MOVHstore [off] {sym} ptr x mem)
(MOVHstore [off] {sym} ptr (MOVWUreg x) mem) => (MOVHstore [off] {sym} ptr x mem)
(MOVWstore [off] {sym} ptr (MOVWreg  x) mem) => (MOVWstore [off] {sym} ptr x mem)
(MOVWstore [off] {sym} ptr (MOVWUreg x) mem) => (MOVWstore [off] {sym} ptr x mem)
(MOVBstoreidx  ptr idx (MOVBreg  x) mem) => (MOVBstoreidx  ptr idx x mem)
(MOVBstoreidx  ptr idx (MOVBUreg x) mem) => (MOVBstoreidx  ptr idx x mem)
(MOVBstoreidx  ptr idx (MOVHreg  x) mem) => (MOVBstoreidx  ptr idx x mem)
(MOVBstoreidx  ptr idx (MOVHUreg x) mem) => (MOVBstoreidx  ptr idx x mem)
(MOVBstoreidx  ptr idx (MOVWreg  x) mem) => (MOVBstoreidx  ptr idx x mem)
(MOVBstoreidx  ptr idx (MOVWUreg x) mem) => (MOVBstoreidx  ptr idx x mem)
(MOVHstoreidx  ptr idx (MOVHreg  x) mem) => (MOVHstoreidx  ptr idx x mem)
(MOVHstoreidx  ptr idx (MOVHUreg x) mem) => (MOVHstoreidx  ptr idx x mem)
(MOVHstoreidx  ptr idx (MOVWreg  x) mem) => (MOVHstoreidx  ptr idx x mem)
(MOVHstoreidx  ptr idx (MOVWUreg x) mem) => (MOVHstoreidx  ptr idx x mem)
(MOVWstoreidx  ptr idx (MOVWreg  x) mem) => (MOVWstoreidx  ptr idx x mem)
(MOVWstoreidx  ptr idx (MOVWUreg x) mem) => (MOVWstoreidx  ptr idx x mem)
(MOVHstoreidx2 ptr idx (MOVHreg  x) mem) => (MOVHstoreidx2 ptr idx x mem)
(MOVHstoreidx2 ptr idx (MOVHUreg x) mem) => (MOVHstoreidx2 ptr idx x mem)
(MOVHstoreidx2 ptr idx (MOVWreg  x) mem) => (MOVHstoreidx2 ptr idx x mem)
(MOVHstoreidx2 ptr idx (MOVWUreg x) mem) => (MOVHstoreidx2 ptr idx x mem)
(MOVWstoreidx4 ptr idx (MOVWreg  x) mem) => (MOVWstoreidx4 ptr idx x mem)
(MOVWstoreidx4 ptr idx (MOVWUreg x) mem) => (MOVWstoreidx4 ptr idx x mem)

// if a register move has only 1 use, just use the same register without emitting instruction
// MOVDnop doesn't emit instruction, only for ensuring the type.
(MOVDreg x) && x.Uses == 1 => (MOVDnop x)

// TODO: we should be able to get rid of MOVDnop all together.
// But for now, this is enough to get rid of lots of them.
(MOVDnop (MOVDconst [c])) => (MOVDconst [c])

// fold constant into arithmetic ops
(ADD  x (MOVDconst <t> [c])) && !t.IsPtr() => (ADDconst [c] x)
(SUB  x (MOVDconst [c])) => (SUBconst [c] x)
(AND  x (MOVDconst [c])) => (ANDconst [c] x)
(OR   x (MOVDconst [c])) => (ORconst  [c] x)
(XOR  x (MOVDconst [c])) => (XORconst [c] x)
(TST  x (MOVDconst [c])) => (TSTconst [c] x)
(TSTW x (MOVDconst [c])) => (TSTWconst [int32(c)] x)
(CMN  x (MOVDconst [c])) => (CMNconst [c] x)
(CMNW x (MOVDconst [c])) => (CMNWconst [int32(c)] x)
(BIC  x (MOVDconst [c])) => (ANDconst [^c] x)
(EON  x (MOVDconst [c])) => (XORconst [^c] x)
(ORN  x (MOVDconst [c])) => (ORconst  [^c] x)

(SLL x (MOVDconst [c])) => (SLLconst x [c&63])
(SRL x (MOVDconst [c])) => (SRLconst x [c&63])
(SRA x (MOVDconst [c])) => (SRAconst x [c&63])
(SLL x (ANDconst [63] y)) => (SLL x y)
(SRL x (ANDconst [63] y)) => (SRL x y)
(SRA x (ANDconst [63] y)) => (SRA x y)

(CMP  x (MOVDconst [c])) => (CMPconst [c] x)
(CMP  (MOVDconst [c]) x) => (InvertFlags (CMPconst [c] x))
(CMPW x (MOVDconst [c])) => (CMPWconst [int32(c)] x)
(CMPW (MOVDconst [c]) x) => (InvertFlags (CMPWconst [int32(c)] x))

(ROR  x (MOVDconst [c])) => (RORconst x [c&63])
(RORW x (MOVDconst [c])) => (RORWconst x [c&31])

(ADDSflags x (MOVDconst [c]))  => (ADDSconstflags [c] x)

(ADDconst [c] y) && c < 0 => (SUBconst [-c] y)

// Canonicalize the order of arguments to comparisons - helps with CSE.
((CMP|CMPW) x y) && ssa.CanonLessThan(x,y) => (InvertFlags ((CMP|CMPW) y x))

// mul-neg => mneg
(NEG  (MUL  x y)) => (MNEG  x y)
(NEG  (MULW x y)) && v.Type.Size() <= 4 => (MNEGW x y)
(MUL  (NEG  x) y) => (MNEG  x y)
(MULW (NEG  x) y) => (MNEGW x y)

// madd/msub
(ADD a l:(MUL  x y)) && l.Uses==1 && ssa.Clobber(l) => (MADD a x y)
(SUB a l:(MUL  x y)) && l.Uses==1 && ssa.Clobber(l) => (MSUB a x y)
(ADD a l:(MNEG x y)) && l.Uses==1 && ssa.Clobber(l) => (MSUB a x y)
(SUB a l:(MNEG x y)) && l.Uses==1 && ssa.Clobber(l) => (MADD a x y)

(ADD a l:(MULW  x y)) && v.Type.Size() <= 4 && l.Uses==1 && ssa.Clobber(l) => (MADDW a x y)
(SUB a l:(MULW  x y)) && v.Type.Size() <= 4 && l.Uses==1 && ssa.Clobber(l) => (MSUBW a x y)
(ADD a l:(MNEGW x y)) && v.Type.Size() <= 4 && l.Uses==1 && ssa.Clobber(l) => (MSUBW a x y)
(SUB a l:(MNEGW x y)) && v.Type.Size() <= 4 && l.Uses==1 && ssa.Clobber(l) => (MADDW a x y)

// madd/msub can't take constant arguments, so do a bit of reordering if a non-constant is available.
// Note: don't reorder arithmetic concerning pointers, as we must ensure that
// no intermediate computations are invalid pointers.
(ADD <t> a p:(ADDconst [c] m:((MUL|MULW|MNEG|MNEGW) _ _))) && p.Uses==1 && m.Uses==1 && !t.IsPtrShaped() => (ADDconst [c] (ADD <v.Type> a m))
(ADD <t> a p:(SUBconst [c] m:((MUL|MULW|MNEG|MNEGW) _ _))) && p.Uses==1 && m.Uses==1 && !t.IsPtrShaped() => (SUBconst [c] (ADD <v.Type> a m))
(SUB <t> a p:(ADDconst [c] m:((MUL|MULW|MNEG|MNEGW) _ _))) && p.Uses==1 && m.Uses==1 && !t.IsPtrShaped() => (SUBconst [c] (SUB <v.Type> a m))
(SUB <t> a p:(SUBconst [c] m:((MUL|MULW|MNEG|MNEGW) _ _))) && p.Uses==1 && m.Uses==1 && !t.IsPtrShaped() => (ADDconst [c] (SUB <v.Type> a m))

// optimize ADCSflags, SBCSflags and friends
(ADCSflags x y (Select1 <types.TypeFlags> (ADDSconstflags [-1] (ADCzerocarry <typ.UInt64> c)))) => (ADCSflags x y c)
(ADCSflags x y (Select1 <types.TypeFlags> (ADDSconstflags [-1] (MOVDconst [0])))) => (ADDSflags x y)
(SBCSflags x y (Select1 <types.TypeFlags> (NEGSflags (NEG <typ.UInt64> (NGCzerocarry <typ.UInt64> bo))))) => (SBCSflags x y bo)
(SBCSflags x y (Select1 <types.TypeFlags> (NEGSflags (MOVDconst [0])))) => (SUBSflags x y)

// mul by constant
(MUL _ (MOVDconst [0])) => (MOVDconst [0])
(MUL x (MOVDconst [1])) => x

(MULW _ (MOVDconst [c])) && int32(c)==0 => (MOVDconst [0])
(MULW x (MOVDconst [c])) && int32(c)==1 => (MOVWUreg x)

(MUL  x (MOVDconst [c])) && ssa.CanMulStrengthReduce(config, c) => {ssa.MulStrengthReduce(v, x, c)}
(MULW x (MOVDconst [c])) && v.Type.Size() <= 4 && ssa.CanMulStrengthReduce32(config, int32(c)) => {ssa.MulStrengthReduce32(v, x, int32(c))}

// mneg by constant
(MNEG x (MOVDconst [-1])) => x
(MNEG _ (MOVDconst [0])) => (MOVDconst [0])
(MNEG x (MOVDconst [1])) => (NEG x)
(MNEG x (MOVDconst [c])) && ssa.IsPowerOfTwo(c) => (NEG (SLLconst <x.Type> [ssa.Log64(c)] x))
(MNEG x (MOVDconst [c])) && ssa.IsPowerOfTwo(c-1) && c >= 3 => (NEG (ADDshiftLL <x.Type> x x [ssa.Log64(c-1)]))
(MNEG x (MOVDconst [c])) && ssa.IsPowerOfTwo(c+1) && c >= 7 => (NEG (ADDshiftLL <x.Type> (NEG <x.Type> x) x [ssa.Log64(c+1)]))
(MNEG x (MOVDconst [c])) && c%3 == 0 && ssa.IsPowerOfTwo(c/3) => (SLLconst <x.Type> [ssa.Log64(c/3)] (SUBshiftLL <x.Type> x x [2]))
(MNEG x (MOVDconst [c])) && c%5 == 0 && ssa.IsPowerOfTwo(c/5) => (NEG (SLLconst <x.Type> [ssa.Log64(c/5)] (ADDshiftLL <x.Type> x x [2])))
(MNEG x (MOVDconst [c])) && c%7 == 0 && ssa.IsPowerOfTwo(c/7) => (SLLconst <x.Type> [ssa.Log64(c/7)] (SUBshiftLL <x.Type> x x [3]))
(MNEG x (MOVDconst [c])) && c%9 == 0 && ssa.IsPowerOfTwo(c/9) => (NEG (SLLconst <x.Type> [ssa.Log64(c/9)] (ADDshiftLL <x.Type> x x [3])))


(MNEGW x (MOVDconst [c])) && int32(c)==-1 => (MOVWUreg x)
(MNEGW _ (MOVDconst [c])) && int32(c)==0 => (MOVDconst [0])
(MNEGW x (MOVDconst [c])) && int32(c)==1 => (MOVWUreg (NEG <x.Type> x))
(MNEGW x (MOVDconst [c])) && ssa.IsPowerOfTwo(c) => (NEG (SLLconst <x.Type> [ssa.Log64(c)] x))
(MNEGW x (MOVDconst [c])) && ssa.IsPowerOfTwo(c-1) && int32(c) >= 3 => (MOVWUreg (NEG <x.Type> (ADDshiftLL <x.Type> x x [ssa.Log64(c-1)])))
(MNEGW x (MOVDconst [c])) && ssa.IsPowerOfTwo(c+1) && int32(c) >= 7 => (MOVWUreg (NEG <x.Type> (ADDshiftLL <x.Type> (NEG <x.Type> x) x [ssa.Log64(c+1)])))
(MNEGW x (MOVDconst [c])) && c%3 == 0 && ssa.IsPowerOfTwo(c/3) && ssa.Is32Bit(c) => (MOVWUreg (SLLconst <x.Type> [ssa.Log64(c/3)] (SUBshiftLL <x.Type> x x [2])))
(MNEGW x (MOVDconst [c])) && c%5 == 0 && ssa.IsPowerOfTwo(c/5) && ssa.Is32Bit(c) => (MOVWUreg (NEG <x.Type> (SLLconst <x.Type> [ssa.Log64(c/5)] (ADDshiftLL <x.Type> x x [2]))))
(MNEGW x (MOVDconst [c])) && c%7 == 0 && ssa.IsPowerOfTwo(c/7) && ssa.Is32Bit(c) => (MOVWUreg (SLLconst <x.Type> [ssa.Log64(c/7)] (SUBshiftLL <x.Type> x x [3])))
(MNEGW x (MOVDconst [c])) && c%9 == 0 && ssa.IsPowerOfTwo(c/9) && ssa.Is32Bit(c) => (MOVWUreg (NEG <x.Type> (SLLconst <x.Type> [ssa.Log64(c/9)] (ADDshiftLL <x.Type> x x [3]))))


(MADD a x (MOVDconst [-1])) => (SUB a x)
(MADD a _ (MOVDconst [0])) => a
(MADD a x (MOVDconst [1])) => (ADD a x)
(MADD a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c) => (ADDshiftLL a x [ssa.Log64(c)])
(MADD a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c-1) && c>=3 => (ADD a (ADDshiftLL <x.Type> x x [ssa.Log64(c-1)]))
(MADD a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c+1) && c>=7 => (SUB a (SUBshiftLL <x.Type> x x [ssa.Log64(c+1)]))
(MADD a x (MOVDconst [c])) && c%3 == 0 && ssa.IsPowerOfTwo(c/3) => (SUBshiftLL a (SUBshiftLL <x.Type> x x [2]) [ssa.Log64(c/3)])
(MADD a x (MOVDconst [c])) && c%5 == 0 && ssa.IsPowerOfTwo(c/5) => (ADDshiftLL a (ADDshiftLL <x.Type> x x [2]) [ssa.Log64(c/5)])
(MADD a x (MOVDconst [c])) && c%7 == 0 && ssa.IsPowerOfTwo(c/7) => (SUBshiftLL a (SUBshiftLL <x.Type> x x [3]) [ssa.Log64(c/7)])
(MADD a x (MOVDconst [c])) && c%9 == 0 && ssa.IsPowerOfTwo(c/9) => (ADDshiftLL a (ADDshiftLL <x.Type> x x [3]) [ssa.Log64(c/9)])

(MADD a (MOVDconst [-1]) x) => (SUB a x)
(MADD a (MOVDconst [0]) _) => a
(MADD a (MOVDconst [1]) x) => (ADD a x)
(MADD a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c) => (ADDshiftLL a x [ssa.Log64(c)])
(MADD a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c-1) && c>=3 => (ADD a (ADDshiftLL <x.Type> x x [ssa.Log64(c-1)]))
(MADD a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c+1) && c>=7 => (SUB a (SUBshiftLL <x.Type> x x [ssa.Log64(c+1)]))
(MADD a (MOVDconst [c]) x) && c%3 == 0 && ssa.IsPowerOfTwo(c/3) => (SUBshiftLL a (SUBshiftLL <x.Type> x x [2]) [ssa.Log64(c/3)])
(MADD a (MOVDconst [c]) x) && c%5 == 0 && ssa.IsPowerOfTwo(c/5) => (ADDshiftLL a (ADDshiftLL <x.Type> x x [2]) [ssa.Log64(c/5)])
(MADD a (MOVDconst [c]) x) && c%7 == 0 && ssa.IsPowerOfTwo(c/7) => (SUBshiftLL a (SUBshiftLL <x.Type> x x [3]) [ssa.Log64(c/7)])
(MADD a (MOVDconst [c]) x) && c%9 == 0 && ssa.IsPowerOfTwo(c/9) => (ADDshiftLL a (ADDshiftLL <x.Type> x x [3]) [ssa.Log64(c/9)])

(MADDW a x (MOVDconst [c])) && int32(c)==-1 => (MOVWUreg (SUB <a.Type> a x))
(MADDW a _ (MOVDconst [c])) && int32(c)==0 => (MOVWUreg a)
(MADDW a x (MOVDconst [c])) && int32(c)==1 => (MOVWUreg (ADD <a.Type> a x))
(MADDW a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c) => (MOVWUreg (ADDshiftLL <a.Type> a x [ssa.Log64(c)]))
(MADDW a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c-1) && int32(c)>=3 => (MOVWUreg (ADD <a.Type> a (ADDshiftLL <x.Type> x x [ssa.Log64(c-1)])))
(MADDW a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c+1) && int32(c)>=7 => (MOVWUreg (SUB <a.Type> a (SUBshiftLL <x.Type> x x [ssa.Log64(c+1)])))
(MADDW a x (MOVDconst [c])) && c%3 == 0 && ssa.IsPowerOfTwo(c/3) && ssa.Is32Bit(c) => (MOVWUreg (SUBshiftLL <a.Type> a (SUBshiftLL <x.Type> x x [2]) [ssa.Log64(c/3)]))
(MADDW a x (MOVDconst [c])) && c%5 == 0 && ssa.IsPowerOfTwo(c/5) && ssa.Is32Bit(c) => (MOVWUreg (ADDshiftLL <a.Type> a (ADDshiftLL <x.Type> x x [2]) [ssa.Log64(c/5)]))
(MADDW a x (MOVDconst [c])) && c%7 == 0 && ssa.IsPowerOfTwo(c/7) && ssa.Is32Bit(c) => (MOVWUreg (SUBshiftLL <a.Type> a (SUBshiftLL <x.Type> x x [3]) [ssa.Log64(c/7)]))
(MADDW a x (MOVDconst [c])) && c%9 == 0 && ssa.IsPowerOfTwo(c/9) && ssa.Is32Bit(c) => (MOVWUreg (ADDshiftLL <a.Type> a (ADDshiftLL <x.Type> x x [3]) [ssa.Log64(c/9)]))

(MADDW a (MOVDconst [c]) x) && int32(c)==-1 => (MOVWUreg (SUB <a.Type> a x))
(MADDW a (MOVDconst [c]) _) && int32(c)==0 => (MOVWUreg a)
(MADDW a (MOVDconst [c]) x) && int32(c)==1 => (MOVWUreg (ADD <a.Type> a x))
(MADDW a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c) => (MOVWUreg (ADDshiftLL <a.Type> a x [ssa.Log64(c)]))
(MADDW a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c-1) && int32(c)>=3 => (MOVWUreg (ADD <a.Type> a (ADDshiftLL <x.Type> x x [ssa.Log64(c-1)])))
(MADDW a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c+1) && int32(c)>=7 => (MOVWUreg (SUB <a.Type> a (SUBshiftLL <x.Type> x x [ssa.Log64(c+1)])))
(MADDW a (MOVDconst [c]) x) && c%3 == 0 && ssa.IsPowerOfTwo(c/3) && ssa.Is32Bit(c) => (MOVWUreg (SUBshiftLL <a.Type> a (SUBshiftLL <x.Type> x x [2]) [ssa.Log64(c/3)]))
(MADDW a (MOVDconst [c]) x) && c%5 == 0 && ssa.IsPowerOfTwo(c/5) && ssa.Is32Bit(c) => (MOVWUreg (ADDshiftLL <a.Type> a (ADDshiftLL <x.Type> x x [2]) [ssa.Log64(c/5)]))
(MADDW a (MOVDconst [c]) x) && c%7 == 0 && ssa.IsPowerOfTwo(c/7) && ssa.Is32Bit(c) => (MOVWUreg (SUBshiftLL <a.Type> a (SUBshiftLL <x.Type> x x [3]) [ssa.Log64(c/7)]))
(MADDW a (MOVDconst [c]) x) && c%9 == 0 && ssa.IsPowerOfTwo(c/9) && ssa.Is32Bit(c) => (MOVWUreg (ADDshiftLL <a.Type> a (ADDshiftLL <x.Type> x x [3]) [ssa.Log64(c/9)]))

(MSUB a x (MOVDconst [-1])) => (ADD a x)
(MSUB a _ (MOVDconst [0])) => a
(MSUB a x (MOVDconst [1])) => (SUB a x)
(MSUB a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c) => (SUBshiftLL a x [ssa.Log64(c)])
(MSUB a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c-1) && c>=3 => (SUB a (ADDshiftLL <x.Type> x x [ssa.Log64(c-1)]))
(MSUB a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c+1) && c>=7 => (ADD a (SUBshiftLL <x.Type> x x [ssa.Log64(c+1)]))
(MSUB a x (MOVDconst [c])) && c%3 == 0 && ssa.IsPowerOfTwo(c/3) => (ADDshiftLL a (SUBshiftLL <x.Type> x x [2]) [ssa.Log64(c/3)])
(MSUB a x (MOVDconst [c])) && c%5 == 0 && ssa.IsPowerOfTwo(c/5) => (SUBshiftLL a (ADDshiftLL <x.Type> x x [2]) [ssa.Log64(c/5)])
(MSUB a x (MOVDconst [c])) && c%7 == 0 && ssa.IsPowerOfTwo(c/7) => (ADDshiftLL a (SUBshiftLL <x.Type> x x [3]) [ssa.Log64(c/7)])
(MSUB a x (MOVDconst [c])) && c%9 == 0 && ssa.IsPowerOfTwo(c/9) => (SUBshiftLL a (ADDshiftLL <x.Type> x x [3]) [ssa.Log64(c/9)])

(MSUB a (MOVDconst [-1]) x) => (ADD a x)
(MSUB a (MOVDconst [0]) _) => a
(MSUB a (MOVDconst [1]) x) => (SUB a x)
(MSUB a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c) => (SUBshiftLL a x [ssa.Log64(c)])
(MSUB a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c-1) && c>=3 => (SUB a (ADDshiftLL <x.Type> x x [ssa.Log64(c-1)]))
(MSUB a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c+1) && c>=7 => (ADD a (SUBshiftLL <x.Type> x x [ssa.Log64(c+1)]))
(MSUB a (MOVDconst [c]) x) && c%3 == 0 && ssa.IsPowerOfTwo(c/3) => (ADDshiftLL a (SUBshiftLL <x.Type> x x [2]) [ssa.Log64(c/3)])
(MSUB a (MOVDconst [c]) x) && c%5 == 0 && ssa.IsPowerOfTwo(c/5) => (SUBshiftLL a (ADDshiftLL <x.Type> x x [2]) [ssa.Log64(c/5)])
(MSUB a (MOVDconst [c]) x) && c%7 == 0 && ssa.IsPowerOfTwo(c/7) => (ADDshiftLL a (SUBshiftLL <x.Type> x x [3]) [ssa.Log64(c/7)])
(MSUB a (MOVDconst [c]) x) && c%9 == 0 && ssa.IsPowerOfTwo(c/9) => (SUBshiftLL a (ADDshiftLL <x.Type> x x [3]) [ssa.Log64(c/9)])

(MSUBW a x (MOVDconst [c])) && int32(c)==-1 => (MOVWUreg (ADD <a.Type> a x))
(MSUBW a _ (MOVDconst [c])) && int32(c)==0 => (MOVWUreg a)
(MSUBW a x (MOVDconst [c])) && int32(c)==1 => (MOVWUreg (SUB <a.Type> a x))
(MSUBW a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c) => (MOVWUreg (SUBshiftLL <a.Type> a x [ssa.Log64(c)]))
(MSUBW a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c-1) && int32(c)>=3 => (MOVWUreg (SUB <a.Type> a (ADDshiftLL <x.Type> x x [ssa.Log64(c-1)])))
(MSUBW a x (MOVDconst [c])) && ssa.IsPowerOfTwo(c+1) && int32(c)>=7 => (MOVWUreg (ADD <a.Type> a (SUBshiftLL <x.Type> x x [ssa.Log64(c+1)])))
(MSUBW a x (MOVDconst [c])) && c%3 == 0 && ssa.IsPowerOfTwo(c/3) && ssa.Is32Bit(c) => (MOVWUreg (ADDshiftLL <a.Type> a (SUBshiftLL <x.Type> x x [2]) [ssa.Log64(c/3)]))
(MSUBW a x (MOVDconst [c])) && c%5 == 0 && ssa.IsPowerOfTwo(c/5) && ssa.Is32Bit(c) => (MOVWUreg (SUBshiftLL <a.Type> a (ADDshiftLL <x.Type> x x [2]) [ssa.Log64(c/5)]))
(MSUBW a x (MOVDconst [c])) && c%7 == 0 && ssa.IsPowerOfTwo(c/7) && ssa.Is32Bit(c) => (MOVWUreg (ADDshiftLL <a.Type> a (SUBshiftLL <x.Type> x x [3]) [ssa.Log64(c/7)]))
(MSUBW a x (MOVDconst [c])) && c%9 == 0 && ssa.IsPowerOfTwo(c/9) && ssa.Is32Bit(c) => (MOVWUreg (SUBshiftLL <a.Type> a (ADDshiftLL <x.Type> x x [3]) [ssa.Log64(c/9)]))

(MSUBW a (MOVDconst [c]) x) && int32(c)==-1 => (MOVWUreg (ADD <a.Type> a x))
(MSUBW a (MOVDconst [c]) _) && int32(c)==0 => (MOVWUreg a)
(MSUBW a (MOVDconst [c]) x) && int32(c)==1 => (MOVWUreg (SUB <a.Type> a x))
(MSUBW a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c) => (MOVWUreg (SUBshiftLL <a.Type> a x [ssa.Log64(c)]))
(MSUBW a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c-1) && int32(c)>=3 => (MOVWUreg (SUB <a.Type> a (ADDshiftLL <x.Type> x x [ssa.Log64(c-1)])))
(MSUBW a (MOVDconst [c]) x) && ssa.IsPowerOfTwo(c+1) && int32(c)>=7 => (MOVWUreg (ADD <a.Type> a (SUBshiftLL <x.Type> x x [ssa.Log64(c+1)])))
(MSUBW a (MOVDconst [c]) x) && c%3 == 0 && ssa.IsPowerOfTwo(c/3) && ssa.Is32Bit(c) => (MOVWUreg (ADDshiftLL <a.Type> a (SUBshiftLL <x.Type> x x [2]) [ssa.Log64(c/3)]))
(MSUBW a (MOVDconst [c]) x) && c%5 == 0 && ssa.IsPowerOfTwo(c/5) && ssa.Is32Bit(c) => (MOVWUreg (SUBshiftLL <a.Type> a (ADDshiftLL <x.Type> x x [2]) [ssa.Log64(c/5)]))
(MSUBW a (MOVDconst [c]) x) && c%7 == 0 && ssa.IsPowerOfTwo(c/7) && ssa.Is32Bit(c) => (MOVWUreg (ADDshiftLL <a.Type> a (SUBshiftLL <x.Type> x x [3]) [ssa.Log64(c/7)]))
(MSUBW a (MOVDconst [c]) x) && c%9 == 0 && ssa.IsPowerOfTwo(c/9) && ssa.Is32Bit(c) => (MOVWUreg (SUBshiftLL <a.Type> a (ADDshiftLL <x.Type> x x [3]) [ssa.Log64(c/9)]))

// div by constant
(UDIV  x (MOVDconst [1])) => x
(UDIV  x (MOVDconst [c])) && ssa.IsPowerOfTwo(c) => (SRLconst [ssa.Log64(c)] x)
(UDIVW x (MOVDconst [c])) && uint32(c)==1 => (MOVWUreg x)
(UDIVW x (MOVDconst [c])) && ssa.IsPowerOfTwo(c) && ssa.Is32Bit(c) => (SRLconst [ssa.Log64(c)] (MOVWUreg <v.Type> x))
(UMOD  _ (MOVDconst [1])) => (MOVDconst [0])
(UMOD  x (MOVDconst [c])) && ssa.IsPowerOfTwo(c) => (ANDconst [c-1] x)
(UMODW _ (MOVDconst [c])) && uint32(c)==1 => (MOVDconst [0])
(UMODW x (MOVDconst [c])) && ssa.IsPowerOfTwo(c) && ssa.Is32Bit(c) => (ANDconst [c-1] x)

// generic simplifications
(ADD x (NEG y)) => (SUB x y)
(SUB x (NEG y)) => (ADD x y)
(SUB x x) => (MOVDconst [0])
(AND x x) => x
(OR  x x) => x
(XOR x x) => (MOVDconst [0])
(BIC x x) => (MOVDconst [0])
(EON x x) => (MOVDconst [-1])
(ORN x x) => (MOVDconst [-1])
(AND x (MVN y)) => (BIC x y)
(XOR x (MVN y)) => (EON x y)
(OR  x (MVN y)) => (ORN x y)
(MVN (XOR x y)) => (EON x y)
(NEG (SUB x y)) => (SUB y x)
(NEG (NEG x)) => x

(CSEL [cc] (MOVDconst [-1]) (MOVDconst [0]) flag) => (CSETM [cc] flag)
(CSEL [cc] (MOVDconst [0]) (MOVDconst [-1]) flag) => (CSETM [arm64Negate(cc)] flag)
(CSEL [cc] x (MOVDconst [0]) flag) => (CSEL0 [cc] x flag)
(CSEL [cc] (MOVDconst [0]) y flag) => (CSEL0 [arm64Negate(cc)] y flag)
(CSEL [cc] x (ADDconst [1] a) flag) => (CSINC [cc] x a flag)
(CSEL [cc] (ADDconst [1] a) x flag) => (CSINC [arm64Negate(cc)] x a flag)
(CSEL [cc] x (MVN a) flag) => (CSINV [cc] x a flag)
(CSEL [cc] (MVN a) x flag) => (CSINV [arm64Negate(cc)] x a flag)
(CSEL [cc] x (NEG a) flag) => (CSNEG [cc] x a flag)
(CSEL [cc] (NEG a) x flag) => (CSNEG [arm64Negate(cc)] x a flag)

(SUB x (SUB y z)) => (SUB (ADD <v.Type> x z) y)
(SUB (SUB x y) z) => (SUB x (ADD <y.Type> y z))

// remove redundant *const ops
(ADDconst [0]  x) => x
(SUBconst [0]  x) => x
(ANDconst [0]  _) => (MOVDconst [0])
(ANDconst [-1] x) => x
(ORconst  [0]  x) => x
(ORconst  [-1] _) => (MOVDconst [-1])
(XORconst [0]  x) => x
(XORconst [-1] x) => (MVN x)

// generic constant folding
(ADDconst [c] (MOVDconst [d]))  => (MOVDconst [c+d])
(ADDconst [c] (ADDconst [d] x)) => (ADDconst [c+d] x)
(ADDconst [c] (SUBconst [d] x)) => (ADDconst [c-d] x)
(SUBconst [c] (MOVDconst [d]))  => (MOVDconst [d-c])
(SUBconst [c] (SUBconst [d] x)) => (ADDconst [-c-d] x)
(SUBconst [c] (ADDconst [d] x)) => (ADDconst [-c+d] x)
(SLLconst [c] (MOVDconst [d]))  => (MOVDconst [d<<uint64(c)])
(SRLconst [c] (MOVDconst [d]))  => (MOVDconst [int64(uint64(d)>>uint64(c))])
(SRAconst [c] (MOVDconst [d]))  => (MOVDconst [d>>uint64(c)])
(MUL   (MOVDconst [c]) (MOVDconst [d])) => (MOVDconst [c*d])
(MNEG  (MOVDconst [c]) (MOVDconst [d])) => (MOVDconst [-c*d])
(MULW  (MOVDconst [c]) (MOVDconst [d])) => (MOVDconst [int64(uint32(c*d))])
(MNEGW (MOVDconst [c]) (MOVDconst [d])) => (MOVDconst [int64(uint32(-c*d))])
(MADD  (MOVDconst [c]) x y) => (ADDconst [c] (MUL  <x.Type> x y))
(MSUB  (MOVDconst [c]) x y) => (ADDconst [c] (MNEG <x.Type> x y))
(MADD  a (MOVDconst [c]) (MOVDconst [d])) => (ADDconst [c*d] a)
(MSUB  a (MOVDconst [c]) (MOVDconst [d])) => (SUBconst [c*d] a)
(MADDW (MOVDconst [c]) x y) => (MOVWUreg (ADDconst <x.Type> [c] (MULW  <x.Type> x y)))
(MSUBW (MOVDconst [c]) x y) => (MOVWUreg (ADDconst <x.Type> [c] (MNEGW <x.Type> x y)))
(MADDW a (MOVDconst [c]) (MOVDconst [d])) => (MOVWUreg (ADDconst <a.Type> [c*d] a))
(MSUBW a (MOVDconst [c]) (MOVDconst [d])) => (MOVWUreg (SUBconst <a.Type> [c*d] a))
(DIV   (MOVDconst [c]) (MOVDconst [d])) && d != 0 => (MOVDconst [c/d])
(UDIV  (MOVDconst [c]) (MOVDconst [d])) && d != 0 => (MOVDconst [int64(uint64(c)/uint64(d))])
(DIVW  (MOVDconst [c]) (MOVDconst [d])) && d != 0 => (MOVDconst [int64(uint32(int32(c)/int32(d)))])
(UDIVW (MOVDconst [c]) (MOVDconst [d])) && d != 0 => (MOVDconst [int64(uint32(c)/uint32(d))])
(MOD   (MOVDconst [c]) (MOVDconst [d])) && d != 0 => (MOVDconst [c%d])
(UMOD  (MOVDconst [c]) (MOVDconst [d])) && d != 0 => (MOVDconst [int64(uint64(c)%uint64(d))])
(MODW  (MOVDconst [c]) (MOVDconst [d])) && d != 0 => (MOVDconst [int64(uint32(int32(c)%int32(d)))])
(UMODW (MOVDconst [c]) (MOVDconst [d])) && d != 0 => (MOVDconst [int64(uint32(c)%uint32(d))])
(ANDconst [c] (MOVDconst [d]))  => (MOVDconst [c&d])
(ANDconst [c] (ANDconst [d] x)) => (ANDconst [c&d] x)
(ANDconst [c] (MOVWUreg x)) => (ANDconst [c&(1<<32-1)] x)
(ANDconst [c] (MOVHUreg x)) => (ANDconst [c&(1<<16-1)] x)
(ANDconst [c] (MOVBUreg x)) => (ANDconst [c&(1<<8-1)] x)
(MOVWUreg (ANDconst [c] x)) => (ANDconst [c&(1<<32-1)] x)
(MOVHUreg (ANDconst [c] x)) => (ANDconst [c&(1<<16-1)] x)
(MOVBUreg (ANDconst [c] x)) => (ANDconst [c&(1<<8-1)] x)
(ORconst  [c] (MOVDconst [d]))  => (MOVDconst [c|d])
(ORconst  [c] (ORconst [d] x))  => (ORconst [c|d] x)
(XORconst [c] (MOVDconst [d]))  => (MOVDconst [c^d])
(XORconst [c] (XORconst [d] x)) => (XORconst [c^d] x)
(MVN (MOVDconst [c])) => (MOVDconst [^c])
(NEG (MOVDconst [c])) => (MOVDconst [-c])
(MOVBreg  (MOVDconst [c])) => (MOVDconst [int64(int8(c))])
(MOVBUreg (MOVDconst [c])) => (MOVDconst [int64(uint8(c))])
(MOVHreg  (MOVDconst [c])) => (MOVDconst [int64(int16(c))])
(MOVHUreg (MOVDconst [c])) => (MOVDconst [int64(uint16(c))])
(MOVWreg  (MOVDconst [c])) => (MOVDconst [int64(int32(c))])
(MOVWUreg (MOVDconst [c])) => (MOVDconst [int64(uint32(c))])
(MOVDreg  (MOVDconst [c])) => (MOVDconst [c])

// constant comparisons
(CMPconst  (MOVDconst [x]) [y]) => (FlagConstant [ssa.SubFlags64(x,y)])
(CMPWconst (MOVDconst [x]) [y]) => (FlagConstant [ssa.SubFlags32(int32(x),y)])
(TSTconst  (MOVDconst [x]) [y]) => (FlagConstant [ssa.LogicFlags64(x&y)])
(TSTWconst (MOVDconst [x]) [y]) => (FlagConstant [ssa.LogicFlags32(int32(x)&y)])
(CMNconst  (MOVDconst [x]) [y]) => (FlagConstant [ssa.AddFlags64(x,y)])
(CMNWconst (MOVDconst [x]) [y]) => (FlagConstant [ssa.AddFlags32(int32(x),y)])

// other known comparisons
(CMPconst  (MOVBUreg _) [c]) && 0xff < c       => (FlagConstant [ssa.SubFlags64(0,1)])
(CMPconst  (MOVHUreg _) [c]) && 0xffff < c     => (FlagConstant [ssa.SubFlags64(0,1)])
(CMPconst  (MOVWUreg _) [c]) && 0xffffffff < c => (FlagConstant [ssa.SubFlags64(0,1)])
(CMPconst  (ANDconst _ [m]) [n]) && 0 <= m && m < n => (FlagConstant [ssa.SubFlags64(0,1)])
(CMPconst  (SRLconst _ [c]) [n]) && 0 <= n && 0 < c && c <= 63 && (1<<uint64(64-c)) <= uint64(n) => (FlagConstant [ssa.SubFlags64(0,1)])
(CMPWconst (MOVBUreg _) [c]) && 0xff   < c => (FlagConstant [ssa.SubFlags64(0,1)])
(CMPWconst (MOVHUreg _) [c]) && 0xffff < c => (FlagConstant [ssa.SubFlags64(0,1)])

// absorb flag constants into branches
(EQ (FlagConstant [fc]) yes no) &&  fc.Eq() => (First yes no)
(EQ (FlagConstant [fc]) yes no) && !fc.Eq() => (First no yes)

(NE (FlagConstant [fc]) yes no) &&  fc.Ne() => (First yes no)
(NE (FlagConstant [fc]) yes no) && !fc.Ne() => (First no yes)

(LT (FlagConstant [fc]) yes no) &&  fc.Lt() => (First yes no)
(LT (FlagConstant [fc]) yes no) && !fc.Lt() => (First no yes)

(LE (FlagConstant [fc]) yes no) &&  fc.Le() => (First yes no)
(LE (FlagConstant [fc]) yes no) && !fc.Le() => (First no yes)

(GT (FlagConstant [fc]) yes no) &&  fc.Gt() => (First yes no)
(GT (FlagConstant [fc]) yes no) && !fc.Gt() => (First no yes)

(GE (FlagConstant [fc]) yes no) &&  fc.Ge() => (First yes no)
(GE (FlagConstant [fc]) yes no) && !fc.Ge() => (First no yes)

(ULT (FlagConstant [fc]) yes no) &&  fc.Ult() => (First yes no)
(ULT (FlagConstant [fc]) yes no) && !fc.Ult() => (First no yes)

(ULE (FlagConstant [fc]) yes no) &&  fc.Ule() => (First yes no)
(ULE (FlagConstant [fc]) yes no) && !fc.Ule() => (First no yes)

(UGT (FlagConstant [fc]) yes no) &&  fc.Ugt() => (First yes no)
(UGT (FlagConstant [fc]) yes no) && !fc.Ugt() => (First no yes)

(UGE (FlagConstant [fc]) yes no) &&  fc.Uge() => (First yes no)
(UGE (FlagConstant [fc]) yes no) && !fc.Uge() => (First no yes)

(LTnoov (FlagConstant [fc]) yes no) &&  fc.LtNoov() => (First yes no)
(LTnoov (FlagConstant [fc]) yes no) && !fc.LtNoov() => (First no yes)

(LEnoov (FlagConstant [fc]) yes no) &&  fc.LeNoov() => (First yes no)
(LEnoov (FlagConstant [fc]) yes no) && !fc.LeNoov() => (First no yes)

(GTnoov (FlagConstant [fc]) yes no) &&  fc.GtNoov() => (First yes no)
(GTnoov (FlagConstant [fc]) yes no) && !fc.GtNoov() => (First no yes)

(GEnoov (FlagConstant [fc]) yes no) &&  fc.GeNoov() => (First yes no)
(GEnoov (FlagConstant [fc]) yes no) && !fc.GeNoov() => (First no yes)

(Z   (MOVDconst [0]) yes no)                  => (First yes no)
(Z   (MOVDconst [c]) yes no) && c != 0        => (First no yes)
(NZ  (MOVDconst [0]) yes no)                  => (First no yes)
(NZ  (MOVDconst [c]) yes no) && c != 0        => (First yes no)
(ZW  (MOVDconst [c]) yes no) && int32(c) == 0 => (First yes no)
(ZW  (MOVDconst [c]) yes no) && int32(c) != 0 => (First no yes)
(NZW (MOVDconst [c]) yes no) && int32(c) == 0 => (First no yes)
(NZW (MOVDconst [c]) yes no) && int32(c) != 0 => (First yes no)

// absorb InvertFlags into branches
(LT  (InvertFlags cmp) yes no) => (GT cmp yes no)
(GT  (InvertFlags cmp) yes no) => (LT cmp yes no)
(LE  (InvertFlags cmp) yes no) => (GE cmp yes no)
(GE  (InvertFlags cmp) yes no) => (LE cmp yes no)
(ULT (InvertFlags cmp) yes no) => (UGT cmp yes no)
(UGT (InvertFlags cmp) yes no) => (ULT cmp yes no)
(ULE (InvertFlags cmp) yes no) => (UGE cmp yes no)
(UGE (InvertFlags cmp) yes no) => (ULE cmp yes no)
(EQ  (InvertFlags cmp) yes no) => (EQ cmp yes no)
(NE  (InvertFlags cmp) yes no) => (NE cmp yes no)
(FLT (InvertFlags cmp) yes no) => (FGT cmp yes no)
(FGT (InvertFlags cmp) yes no) => (FLT cmp yes no)
(FLE (InvertFlags cmp) yes no) => (FGE cmp yes no)
(FGE (InvertFlags cmp) yes no) => (FLE cmp yes no)

// absorb InvertFlags into conditional instructions
(CSEL  [cc] x y (InvertFlags cmp)) => (CSEL  [arm64Invert(cc)] x y cmp)
(CSEL0 [cc] x   (InvertFlags cmp)) => (CSEL0 [arm64Invert(cc)] x   cmp)
(FCSELD [cc] x y (InvertFlags cmp)) => (FCSELD [arm64Invert(cc)] x y cmp)
(FCSELS [cc] x y (InvertFlags cmp)) => (FCSELS [arm64Invert(cc)] x y cmp)
(CSETM [cc]     (InvertFlags cmp)) => (CSETM [arm64Invert(cc)]     cmp)
(CSINC [cc] x y (InvertFlags cmp)) => (CSINC [arm64Invert(cc)] x y cmp)
(CSINV [cc] x y (InvertFlags cmp)) => (CSINV [arm64Invert(cc)] x y cmp)
(CSNEG [cc] x y (InvertFlags cmp)) => (CSNEG [arm64Invert(cc)] x y cmp)

// absorb flag constants into boolean values
(Equal             (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.Eq())])
(NotEqual          (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.Ne())])
(LessThan          (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.Lt())])
(LessThanU         (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.Ult())])
(LessEqual         (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.Le())])
(LessEqualU        (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.Ule())])
(GreaterThan       (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.Gt())])
(GreaterThanU      (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.Ugt())])
(GreaterEqual      (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.Ge())])
(GreaterEqualU     (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.Uge())])
(LessThanNoov      (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.LtNoov())])
(GreaterEqualNoov  (FlagConstant [fc])) => (MOVDconst [ssa.B2i(fc.GeNoov())])

// absorb InvertFlags into boolean values
(Equal            (InvertFlags x)) => (Equal x)
(NotEqual         (InvertFlags x)) => (NotEqual x)
(LessThan         (InvertFlags x)) => (GreaterThan x)
(LessThanU        (InvertFlags x)) => (GreaterThanU x)
(GreaterThan      (InvertFlags x)) => (LessThan x)
(GreaterThanU     (InvertFlags x)) => (LessThanU x)
(LessEqual        (InvertFlags x)) => (GreaterEqual x)
(LessEqualU       (InvertFlags x)) => (GreaterEqualU x)
(GreaterEqual     (InvertFlags x)) => (LessEqual x)
(GreaterEqualU    (InvertFlags x)) => (LessEqualU x)
(LessThanF        (InvertFlags x)) => (GreaterThanF x)
(LessEqualF       (InvertFlags x)) => (GreaterEqualF x)
(GreaterThanF     (InvertFlags x)) => (LessThanF x)
(GreaterEqualF    (InvertFlags x)) => (LessEqualF x)

(GreaterEqual (CMPconst x [0])) => (XORconst [1] (SRLconst <v.Type> [63] x))
(LessThan (CMPconst x [0])) => (SRLconst [63] x)
(GreaterEqual (CMPWconst x [0])) => (XORconst [1] (UBFX <v.Type> [ssa.ArmBFAuxInt(31,1)] x))
(LessThan (CMPWconst x [0])) => (UBFX [ssa.ArmBFAuxInt(31,1)] x)

// Don't bother extending if we're not using the higher bits.
(MOV(B|BU)reg x) && v.Type.Size() <= 1 => x
(MOV(H|HU)reg x) && v.Type.Size() <= 2 => x
(MOV(W|WU)reg x) && v.Type.Size() <= 4 => x

// omit sign extension
(MOVWreg <t> (ANDconst x [c])) && uint64(c) & uint64(0xffffffff80000000) == 0 => (ANDconst <t> x [c])
(MOVHreg <t> (ANDconst x [c])) && uint64(c) & uint64(0xffffffffffff8000) == 0 => (ANDconst <t> x [c])
(MOVBreg <t> (ANDconst x [c])) && uint64(c) & uint64(0xffffffffffffff80) == 0 => (ANDconst <t> x [c])

// absorb flag constants into conditional instructions
(CSEL  [cc] x _ flag) && ccARM64Eval(cc, flag) > 0 => x
(CSEL  [cc] _ y flag) && ccARM64Eval(cc, flag) < 0 => y
(CSEL0 [cc] x   flag) && ccARM64Eval(cc, flag) > 0 => x
(CSEL0 [cc] _   flag) && ccARM64Eval(cc, flag) < 0 => (MOVDconst [0])
(CSNEG [cc] x _ flag) && ccARM64Eval(cc, flag) > 0 => x
(CSNEG [cc] _ y flag) && ccARM64Eval(cc, flag) < 0 => (NEG y)
(CSINV [cc] x _ flag) && ccARM64Eval(cc, flag) > 0 => x
(CSINV [cc] _ y flag) && ccARM64Eval(cc, flag) < 0 => (Not y)
(CSINC [cc] x _ flag) && ccARM64Eval(cc, flag) > 0 => x
(CSINC [cc] _ y flag) && ccARM64Eval(cc, flag) < 0 => (ADDconst [1] y)
(CSETM [cc]     flag) && ccARM64Eval(cc, flag) > 0 => (MOVDconst [-1])
(CSETM [cc]     flag) && ccARM64Eval(cc, flag) < 0 => (MOVDconst [0])

// absorb flags back into boolean CSEL
(CSEL [cc] x y (CMPWconst [0] boolval)) && cc == ssaop.OpARM64NotEqual && ssa.FlagArg(boolval) != nil =>
      (CSEL [boolval.Op] x y ssa.FlagArg(boolval))
(CSEL [cc] x y (CMPWconst [0] boolval)) && cc == ssaop.OpARM64Equal && ssa.FlagArg(boolval) != nil =>
      (CSEL [arm64Negate(boolval.Op)] x y ssa.FlagArg(boolval))
(CSEL0 [cc] x (CMPWconst [0] boolval)) && cc == ssaop.OpARM64NotEqual && ssa.FlagArg(boolval) != nil =>
      (CSEL0 [boolval.Op] x ssa.FlagArg(boolval))
(CSEL0 [cc] x (CMPWconst [0] boolval)) && cc == ssaop.OpARM64Equal && ssa.FlagArg(boolval) != nil =>
      (CSEL0 [arm64Negate(boolval.Op)] x ssa.FlagArg(boolval))

// absorb shifts into ops
(NEG x:(SLLconst [c] y)) && ssa.ClobberIfDead(x) => (NEGshiftLL [c] y)
(NEG x:(SRLconst [c] y)) && ssa.ClobberIfDead(x) => (NEGshiftRL [c] y)
(NEG x:(SRAconst [c] y)) && ssa.ClobberIfDead(x) => (NEGshiftRA [c] y)
(MVN x:(SLLconst [c] y)) && ssa.ClobberIfDead(x) => (MVNshiftLL [c] y)
(MVN x:(SRLconst [c] y)) && ssa.ClobberIfDead(x) => (MVNshiftRL [c] y)
(MVN x:(SRAconst [c] y)) && ssa.ClobberIfDead(x) => (MVNshiftRA [c] y)
(MVN x:(RORconst [c] y)) && ssa.ClobberIfDead(x) => (MVNshiftRO [c] y)
(ADD x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (ADDshiftLL x0 y [c])
(ADD x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (ADDshiftRL x0 y [c])
(ADD x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (ADDshiftRA x0 y [c])
(SUB x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (SUBshiftLL x0 y [c])
(SUB x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (SUBshiftRL x0 y [c])
(SUB x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (SUBshiftRA x0 y [c])
(AND x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (ANDshiftLL x0 y [c])
(AND x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (ANDshiftRL x0 y [c])
(AND x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (ANDshiftRA x0 y [c])
(AND x0 x1:(RORconst [c] y)) && ssa.ClobberIfDead(x1) => (ANDshiftRO x0 y [c])
(OR  x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (ORshiftLL  x0 y [c]) // useful for combined load
(OR  x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (ORshiftRL  x0 y [c])
(OR  x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (ORshiftRA  x0 y [c])
(OR  x0 x1:(RORconst [c] y)) && ssa.ClobberIfDead(x1) => (ORshiftRO  x0 y [c])
(XOR x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (XORshiftLL x0 y [c])
(XOR x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (XORshiftRL x0 y [c])
(XOR x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (XORshiftRA x0 y [c])
(XOR x0 x1:(RORconst [c] y)) && ssa.ClobberIfDead(x1) => (XORshiftRO x0 y [c])
(BIC x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (BICshiftLL x0 y [c])
(BIC x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (BICshiftRL x0 y [c])
(BIC x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (BICshiftRA x0 y [c])
(BIC x0 x1:(RORconst [c] y)) && ssa.ClobberIfDead(x1) => (BICshiftRO x0 y [c])
(ORN x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (ORNshiftLL x0 y [c])
(ORN x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (ORNshiftRL x0 y [c])
(ORN x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (ORNshiftRA x0 y [c])
(ORN x0 x1:(RORconst [c] y)) && ssa.ClobberIfDead(x1) => (ORNshiftRO x0 y [c])
(EON x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (EONshiftLL x0 y [c])
(EON x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (EONshiftRL x0 y [c])
(EON x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (EONshiftRA x0 y [c])
(EON x0 x1:(RORconst [c] y)) && ssa.ClobberIfDead(x1) => (EONshiftRO x0 y [c])
(CMP x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (CMPshiftLL x0 y [c])
(CMP x0:(SLLconst [c] y) x1) && ssa.ClobberIfDead(x0) => (InvertFlags (CMPshiftLL x1 y [c]))
(CMP x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (CMPshiftRL x0 y [c])
(CMP x0:(SRLconst [c] y) x1) && ssa.ClobberIfDead(x0) => (InvertFlags (CMPshiftRL x1 y [c]))
(CMP x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (CMPshiftRA x0 y [c])
(CMP x0:(SRAconst [c] y) x1) && ssa.ClobberIfDead(x0) => (InvertFlags (CMPshiftRA x1 y [c]))
(CMN x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (CMNshiftLL x0 y [c])
(CMN x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (CMNshiftRL x0 y [c])
(CMN x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (CMNshiftRA x0 y [c])
(TST x0 x1:(SLLconst [c] y)) && ssa.ClobberIfDead(x1) => (TSTshiftLL x0 y [c])
(TST x0 x1:(SRLconst [c] y)) && ssa.ClobberIfDead(x1) => (TSTshiftRL x0 y [c])
(TST x0 x1:(SRAconst [c] y)) && ssa.ClobberIfDead(x1) => (TSTshiftRA x0 y [c])
(TST x0 x1:(RORconst [c] y)) && ssa.ClobberIfDead(x1) => (TSTshiftRO x0 y [c])

// prefer *const ops to *shift ops
(ADDshiftLL (MOVDconst [c]) x [d]) => (ADDconst [c] (SLLconst <x.Type> x [d]))
(ADDshiftRL (MOVDconst [c]) x [d]) => (ADDconst [c] (SRLconst <x.Type> x [d]))
(ADDshiftRA (MOVDconst [c]) x [d]) => (ADDconst [c] (SRAconst <x.Type> x [d]))
(ANDshiftLL (MOVDconst [c]) x [d]) => (ANDconst [c] (SLLconst <x.Type> x [d]))
(ANDshiftRL (MOVDconst [c]) x [d]) => (ANDconst [c] (SRLconst <x.Type> x [d]))
(ANDshiftRA (MOVDconst [c]) x [d]) => (ANDconst [c] (SRAconst <x.Type> x [d]))
(ANDshiftRO (MOVDconst [c]) x [d]) => (ANDconst [c] (RORconst <x.Type> x [d]))
(ORshiftLL  (MOVDconst [c]) x [d]) => (ORconst  [c] (SLLconst <x.Type> x [d]))
(ORshiftRL  (MOVDconst [c]) x [d]) => (ORconst  [c] (SRLconst <x.Type> x [d]))
(ORshiftRA  (MOVDconst [c]) x [d]) => (ORconst  [c] (SRAconst <x.Type> x [d]))
(ORshiftRO  (MOVDconst [c]) x [d]) => (ORconst  [c] (RORconst <x.Type> x [d]))
(XORshiftLL (MOVDconst [c]) x [d]) => (XORconst [c] (SLLconst <x.Type> x [d]))
(XORshiftRL (MOVDconst [c]) x [d]) => (XORconst [c] (SRLconst <x.Type> x [d]))
(XORshiftRA (MOVDconst [c]) x [d]) => (XORconst [c] (SRAconst <x.Type> x [d]))
(XORshiftRO (MOVDconst [c]) x [d]) => (XORconst [c] (RORconst <x.Type> x [d]))
(CMPshiftLL (MOVDconst [c]) x [d]) => (InvertFlags (CMPconst [c] (SLLconst <x.Type> x [d])))
(CMPshiftRL (MOVDconst [c]) x [d]) => (InvertFlags (CMPconst [c] (SRLconst <x.Type> x [d])))
(CMPshiftRA (MOVDconst [c]) x [d]) => (InvertFlags (CMPconst [c] (SRAconst <x.Type> x [d])))
(CMNshiftLL (MOVDconst [c]) x [d]) => (CMNconst [c] (SLLconst <x.Type> x [d]))
(CMNshiftRL (MOVDconst [c]) x [d]) => (CMNconst [c] (SRLconst <x.Type> x [d]))
(CMNshiftRA (MOVDconst [c]) x [d]) => (CMNconst [c] (SRAconst <x.Type> x [d]))
(TSTshiftLL (MOVDconst [c]) x [d]) => (TSTconst [c] (SLLconst <x.Type> x [d]))
(TSTshiftRL (MOVDconst [c]) x [d]) => (TSTconst [c] (SRLconst <x.Type> x [d]))
(TSTshiftRA (MOVDconst [c]) x [d]) => (TSTconst [c] (SRAconst <x.Type> x [d]))
(TSTshiftRO (MOVDconst [c]) x [d]) => (TSTconst [c] (RORconst <x.Type> x [d]))

// constant folding in *shift ops
(MVNshiftLL (MOVDconst [c]) [d]) => (MOVDconst [^int64(uint64(c)<<uint64(d))])
(MVNshiftRL (MOVDconst [c]) [d]) => (MOVDconst [^int64(uint64(c)>>uint64(d))])
(MVNshiftRA (MOVDconst [c]) [d]) => (MOVDconst [^(c>>uint64(d))])
(MVNshiftRO (MOVDconst [c]) [d]) => (MOVDconst [^rotateRight64(c, d)])
(NEGshiftLL (MOVDconst [c]) [d]) => (MOVDconst [-int64(uint64(c)<<uint64(d))])
(NEGshiftRL (MOVDconst [c]) [d]) => (MOVDconst [-int64(uint64(c)>>uint64(d))])
(NEGshiftRA (MOVDconst [c]) [d]) => (MOVDconst [-(c>>uint64(d))])
(ADDshiftLL x (MOVDconst [c]) [d]) => (ADDconst x [int64(uint64(c)<<uint64(d))])
(ADDshiftRL x (MOVDconst [c]) [d]) => (ADDconst x [int64(uint64(c)>>uint64(d))])
(ADDshiftRA x (MOVDconst [c]) [d]) => (ADDconst x [c>>uint64(d)])
(SUBshiftLL x (MOVDconst [c]) [d]) => (SUBconst x [int64(uint64(c)<<uint64(d))])
(SUBshiftRL x (MOVDconst [c]) [d]) => (SUBconst x [int64(uint64(c)>>uint64(d))])
(SUBshiftRA x (MOVDconst [c]) [d]) => (SUBconst x [c>>uint64(d)])
(ANDshiftLL x (MOVDconst [c]) [d]) => (ANDconst x [int64(uint64(c)<<uint64(d))])
(ANDshiftRL x (MOVDconst [c]) [d]) => (ANDconst x [int64(uint64(c)>>uint64(d))])
(ANDshiftRA x (MOVDconst [c]) [d]) => (ANDconst x [c>>uint64(d)])
(ANDshiftRO x (MOVDconst [c]) [d]) => (ANDconst x [rotateRight64(c, d)])
(ORshiftLL  x (MOVDconst [c]) [d]) => (ORconst  x [int64(uint64(c)<<uint64(d))])
(ORshiftRL  x (MOVDconst [c]) [d]) => (ORconst  x [int64(uint64(c)>>uint64(d))])
(ORshiftRA  x (MOVDconst [c]) [d]) => (ORconst  x [c>>uint64(d)])
(ORshiftRO  x (MOVDconst [c]) [d]) => (ORconst  x [rotateRight64(c, d)])
(XORshiftLL x (MOVDconst [c]) [d]) => (XORconst x [int64(uint64(c)<<uint64(d))])
(XORshiftRL x (MOVDconst [c]) [d]) => (XORconst x [int64(uint64(c)>>uint64(d))])
(XORshiftRA x (MOVDconst [c]) [d]) => (XORconst x [c>>uint64(d)])
(XORshiftRO x (MOVDconst [c]) [d]) => (XORconst x [rotateRight64(c, d)])
(BICshiftLL x (MOVDconst [c]) [d]) => (ANDconst x [^int64(uint64(c)<<uint64(d))])
(BICshiftRL x (MOVDconst [c]) [d]) => (ANDconst x [^int64(uint64(c)>>uint64(d))])
(BICshiftRA x (MOVDconst [c]) [d]) => (ANDconst x [^(c>>uint64(d))])
(BICshiftRO x (MOVDconst [c]) [d]) => (ANDconst x [^rotateRight64(c, d)])
(ORNshiftLL x (MOVDconst [c]) [d]) => (ORconst  x [^int64(uint64(c)<<uint64(d))])
(ORNshiftRL x (MOVDconst [c]) [d]) => (ORconst  x [^int64(uint64(c)>>uint64(d))])
(ORNshiftRA x (MOVDconst [c]) [d]) => (ORconst  x [^(c>>uint64(d))])
(ORNshiftRO x (MOVDconst [c]) [d]) => (ORconst  x [^rotateRight64(c, d)])
(EONshiftLL x (MOVDconst [c]) [d]) => (XORconst x [^int64(uint64(c)<<uint64(d))])
(EONshiftRL x (MOVDconst [c]) [d]) => (XORconst x [^int64(uint64(c)>>uint64(d))])
(EONshiftRA x (MOVDconst [c]) [d]) => (XORconst x [^(c>>uint64(d))])
(EONshiftRO x (MOVDconst [c]) [d]) => (XORconst x [^rotateRight64(c, d)])
(CMPshiftLL x (MOVDconst [c]) [d]) => (CMPconst x [int64(uint64(c)<<uint64(d))])
(CMPshiftRL x (MOVDconst [c]) [d]) => (CMPconst x [int64(uint64(c)>>uint64(d))])
(CMPshiftRA x (MOVDconst [c]) [d]) => (CMPconst x [c>>uint64(d)])
(CMNshiftLL x (MOVDconst [c]) [d]) => (CMNconst x [int64(uint64(c)<<uint64(d))])
(CMNshiftRL x (MOVDconst [c]) [d]) => (CMNconst x [int64(uint64(c)>>uint64(d))])
(CMNshiftRA x (MOVDconst [c]) [d]) => (CMNconst x [c>>uint64(d)])
(TSTshiftLL x (MOVDconst [c]) [d]) => (TSTconst x [int64(uint64(c)<<uint64(d))])
(TSTshiftRL x (MOVDconst [c]) [d]) => (TSTconst x [int64(uint64(c)>>uint64(d))])
(TSTshiftRA x (MOVDconst [c]) [d]) => (TSTconst x [c>>uint64(d)])
(TSTshiftRO x (MOVDconst [c]) [d]) => (TSTconst x [rotateRight64(c, d)])

// simplification with *shift ops
(SUBshiftLL (SLLconst x [c]) x [c]) => (MOVDconst [0])
(SUBshiftRL (SRLconst x [c]) x [c]) => (MOVDconst [0])
(SUBshiftRA (SRAconst x [c]) x [c]) => (MOVDconst [0])
(ANDshiftLL y:(SLLconst x [c]) x [c]) => y
(ANDshiftRL y:(SRLconst x [c]) x [c]) => y
(ANDshiftRA y:(SRAconst x [c]) x [c]) => y
(ANDshiftRO y:(RORconst x [c]) x [c]) => y
(ORshiftLL  y:(SLLconst x [c]) x [c]) => y
(ORshiftRL  y:(SRLconst x [c]) x [c]) => y
(ORshiftRA  y:(SRAconst x [c]) x [c]) => y
(ORshiftRO  y:(RORconst x [c]) x [c]) => y
(XORshiftLL (SLLconst x [c]) x [c]) => (MOVDconst [0])
(XORshiftRL (SRLconst x [c]) x [c]) => (MOVDconst [0])
(XORshiftRA (SRAconst x [c]) x [c]) => (MOVDconst [0])
(XORshiftRO (RORconst x [c]) x [c]) => (MOVDconst [0])
(BICshiftLL (SLLconst x [c]) x [c]) => (MOVDconst [0])
(BICshiftRL (SRLconst x [c]) x [c]) => (MOVDconst [0])
(BICshiftRA (SRAconst x [c]) x [c]) => (MOVDconst [0])
(BICshiftRO (RORconst x [c]) x [c]) => (MOVDconst [0])
(EONshiftLL (SLLconst x [c]) x [c]) => (MOVDconst [-1])
(EONshiftRL (SRLconst x [c]) x [c]) => (MOVDconst [-1])
(EONshiftRA (SRAconst x [c]) x [c]) => (MOVDconst [-1])
(EONshiftRO (RORconst x [c]) x [c]) => (MOVDconst [-1])
(ORNshiftLL (SLLconst x [c]) x [c]) => (MOVDconst [-1])
(ORNshiftRL (SRLconst x [c]) x [c]) => (MOVDconst [-1])
(ORNshiftRA (SRAconst x [c]) x [c]) => (MOVDconst [-1])
(ORNshiftRO (RORconst x [c]) x [c]) => (MOVDconst [-1])

// rev16w | rev16
// ((x>>8) | (x<<8)) => (REV16W x), the type of x is uint16, "|" can also be "^" or "+".
((ADDshiftLL|ORshiftLL|XORshiftLL) <typ.UInt16> [8] (UBFX <typ.UInt16> [ssa.ArmBFAuxInt(8, 8)] x) x) => (REV16W x)

// ((x & 0xff00ff00)>>8) | ((x & 0x00ff00ff)<<8), "|" can also be "^" or "+".
((ADDshiftLL|ORshiftLL|XORshiftLL) [8] (UBFX [ssa.ArmBFAuxInt(8, 24)] (ANDconst [c1] x)) (ANDconst [c2] x))
	&& uint32(c1) == 0xff00ff00 && uint32(c2) == 0x00ff00ff
	=> (REV16W x)

// ((x & 0xff00ff00ff00ff00)>>8) | ((x & 0x00ff00ff00ff00ff)<<8), "|" can also be "^" or "+".
((ADDshiftLL|ORshiftLL|XORshiftLL) [8] (SRLconst [8] (ANDconst [c1] x)) (ANDconst [c2] x))
	&& (uint64(c1) == 0xff00ff00ff00ff00 && uint64(c2) == 0x00ff00ff00ff00ff)
	=> (REV16 x)

// ((x & 0xff00ff00)>>8) | ((x & 0x00ff00ff)<<8), "|" can also be "^" or "+".
((ADDshiftLL|ORshiftLL|XORshiftLL) [8] (SRLconst [8] (ANDconst [c1] x)) (ANDconst [c2] x))
	&& (uint64(c1) == 0xff00ff00 && uint64(c2) == 0x00ff00ff)
	=> (REV16 (ANDconst <x.Type> [0xffffffff] x))

(REV16 (MOVWUreg x)) => (REV16W x)

// Extract from reg pair
(ADDshiftLL [c] (SRLconst x [64-c]) x2) => (EXTRconst [64-c] x2 x)
( ORshiftLL [c] (SRLconst x [64-c]) x2) => (EXTRconst [64-c] x2 x)
(XORshiftLL [c] (SRLconst x [64-c]) x2) => (EXTRconst [64-c] x2 x)

(ADDshiftLL <t> [c] (UBFX [bfc] x) x2) && c < 32 && t.Size() == 4 && bfc == ssa.ArmBFAuxInt(32-c, c)
	=> (EXTRWconst [32-c] x2 x)
( ORshiftLL <t> [c] (UBFX [bfc] x) x2) && c < 32 && t.Size() == 4 && bfc == ssa.ArmBFAuxInt(32-c, c)
	=> (EXTRWconst [32-c] x2 x)
(XORshiftLL <t> [c] (UBFX [bfc] x) x2) && c < 32 && t.Size() == 4 && bfc == ssa.ArmBFAuxInt(32-c, c)
	=> (EXTRWconst [32-c] x2 x)

// Rewrite special pairs of shifts to AND.
// On ARM64 the bitmask can fit into an instruction.
(SRLconst [c] (SLLconst [c] x)) && 0 < c && c < 64 => (ANDconst [1<<uint(64-c)-1] x) // mask out high bits
(SLLconst [c] (SRLconst [c] x)) && 0 < c && c < 64 => (ANDconst [^(1<<uint(c)-1)] x) // mask out low bits

// Special case setting bit as 1. An example is math.Copysign(c,-1)
(ORconst [c1] (ANDconst [c2] x)) && c2|c1 == ^0  => (ORconst [c1] x)

// If the shift amount is larger than the datasize(32, 16, 8), we can optimize to constant 0.
(MOVWUreg (SLLconst [lc] x)) && lc >= 32 => (MOVDconst [0])
(MOVHUreg (SLLconst [lc] x)) && lc >= 16 => (MOVDconst [0])
(MOVBUreg (SLLconst [lc] x)) && lc >= 8 => (MOVDconst [0])

// After zero extension, the upper (64-datasize(32|16|8)) bits are zero, we can optimiza to constant 0.
(SRLconst [rc] (MOVWUreg x)) && rc >= 32 => (MOVDconst [0])
(SRLconst [rc] (MOVHUreg x)) && rc >= 16 => (MOVDconst [0])
(SRLconst [rc] (MOVBUreg x)) && rc >= 8 => (MOVDconst [0])

// Special cases for slice operations
(ADD x0 x1:(ANDshiftRA x2:(SLLconst [sl] y) z [63])) && x1.Uses == 1 && x2.Uses == 1 => (ADDshiftLL x0 (ANDshiftRA <y.Type> y z [63]) [sl])
(ADD x0 x1:(ANDshiftLL x2:(SRAconst [63] z) y [sl])) && x1.Uses == 1 && x2.Uses == 1 => (ADDshiftLL x0 (ANDshiftRA <y.Type> y z [63]) [sl])

// bitfield ops

// sbfiz
// (x << lc) >> rc
(SRAconst [rc] (SLLconst [lc] x)) && lc > rc => (SBFIZ [ssa.ArmBFAuxInt(lc-rc, 64-lc)] x)
// int64(x << lc)
(MOVWreg (SLLconst [lc] x)) && lc < 32 => (SBFIZ [ssa.ArmBFAuxInt(lc, 32-lc)] x)
(MOVHreg (SLLconst [lc] x)) && lc < 16 => (SBFIZ [ssa.ArmBFAuxInt(lc, 16-lc)] x)
(MOVBreg (SLLconst [lc] x)) && lc < 8  => (SBFIZ [ssa.ArmBFAuxInt(lc,  8-lc)] x)
// int64(x) << lc
(SLLconst [lc] (MOVWreg x))  => (SBFIZ [ssa.ArmBFAuxInt(lc, min(32, 64-lc))] x)
(SLLconst [lc] (MOVHreg x))  => (SBFIZ [ssa.ArmBFAuxInt(lc, min(16, 64-lc))] x)
(SLLconst [lc] (MOVBreg x))  => (SBFIZ [ssa.ArmBFAuxInt(lc, min(8,  64-lc))] x)

// sbfx
// (x << lc) >> rc
(SRAconst [rc] (SLLconst [lc] x)) && lc <= rc => (SBFX [ssa.ArmBFAuxInt(rc-lc, 64-rc)] x)
// int64(x) >> rc
(SRAconst [rc] (MOVWreg x)) && rc < 32 => (SBFX [ssa.ArmBFAuxInt(rc, 32-rc)] x)
(SRAconst [rc] (MOVHreg x)) && rc < 16 => (SBFX [ssa.ArmBFAuxInt(rc, 16-rc)] x)
(SRAconst [rc] (MOVBreg x)) && rc < 8  => (SBFX [ssa.ArmBFAuxInt(rc,  8-rc)] x)
// merge sbfx and sign-extension into sbfx
(MOVWreg (SBFX [bfc] x)) && bfc.Width() <= 32 => (SBFX [bfc] x)
(MOVHreg (SBFX [bfc] x)) && bfc.Width() <= 16 => (SBFX [bfc] x)
(MOVBreg (SBFX [bfc] x)) && bfc.Width() <=  8 => (SBFX [bfc] x)

// sbfiz/sbfx combinations: merge shifts into bitfield ops
(SRAconst [sc] (SBFIZ [bfc] x)) && sc < bfc.Lsb()
	=> (SBFIZ [ssa.ArmBFAuxInt(bfc.Lsb()-sc, bfc.Width())] x)
(SRAconst [sc] (SBFIZ [bfc] x)) && sc >= bfc.Lsb()
	&& sc < bfc.Lsb()+bfc.Width()
	=> (SBFX [ssa.ArmBFAuxInt(sc-bfc.Lsb(), bfc.Lsb()+bfc.Width()-sc)] x)
(SBFX [bfc] s:(SLLconst [sc] x))
	&& s.Uses == 1
	&& sc <= bfc.Lsb()
	=> (SBFX [ssa.ArmBFAuxInt(bfc.Lsb() - sc, bfc.Width())] x)
(SBFX [bfc] s:(SLLconst [sc] x))
	&& s.Uses == 1
	&& sc > bfc.Lsb()
	&& sc < bfc.Lsb()+bfc.Width()
	=> (SBFIZ [ssa.ArmBFAuxInt(sc - bfc.Lsb(), bfc.Width() - (sc-bfc.Lsb()))] x)

// ubfiz
// (x << lc) >> rc
(SRLconst [rc] (SLLconst [lc] x)) && lc > rc => (UBFIZ [ssa.ArmBFAuxInt(lc-rc, 64-lc)] x)
// uint64(x) << lc
(SLLconst [lc] (MOVWUreg x))  => (UBFIZ [ssa.ArmBFAuxInt(lc, min(32, 64-lc))] x)
(SLLconst [lc] (MOVHUreg x))  => (UBFIZ [ssa.ArmBFAuxInt(lc, min(16, 64-lc))] x)
(SLLconst [lc] (MOVBUreg x))  => (UBFIZ [ssa.ArmBFAuxInt(lc, min(8,  64-lc))] x)
// uint64(x << lc)
(MOVWUreg (SLLconst [lc] x)) && lc < 32 => (UBFIZ [ssa.ArmBFAuxInt(lc, 32-lc)] x)
(MOVHUreg (SLLconst [lc] x)) && lc < 16 => (UBFIZ [ssa.ArmBFAuxInt(lc, 16-lc)] x)
(MOVBUreg (SLLconst [lc] x)) && lc < 8  => (UBFIZ [ssa.ArmBFAuxInt(lc,  8-lc)] x)

// merge ANDconst into ubfiz
// (x & ac) << sc
(SLLconst [sc] (ANDconst [ac] x)) && isARM64BFMask(sc, ac, 0)
	=> (UBFIZ [ssa.ArmBFAuxInt(sc, arm64BFWidth(ac, 0))] x)
// (x << sc) & ac
(ANDconst [ac] (SLLconst [sc] x)) && isARM64BFMask(sc, ac, sc)
	=> (UBFIZ [ssa.ArmBFAuxInt(sc, arm64BFWidth(ac, sc))] x)

// ubfx
// (x << lc) >> rc
(SRLconst [rc] (SLLconst [lc] x)) && lc < rc => (UBFX [ssa.ArmBFAuxInt(rc-lc, 64-rc)] x)
// uint64(x) >> rc
(SRLconst [rc] (MOVWUreg x)) && rc < 32 => (UBFX [ssa.ArmBFAuxInt(rc, 32-rc)] x)
(SRLconst [rc] (MOVHUreg x)) && rc < 16 => (UBFX [ssa.ArmBFAuxInt(rc, 16-rc)] x)
(SRLconst [rc] (MOVBUreg x)) && rc < 8  => (UBFX [ssa.ArmBFAuxInt(rc,  8-rc)] x)
// uint64(x >> rc)
(MOVWUreg (SRLconst [rc] x)) && rc < 32 => (UBFX [ssa.ArmBFAuxInt(rc, 32)] x)
(MOVHUreg (SRLconst [rc] x)) && rc < 16 => (UBFX [ssa.ArmBFAuxInt(rc, 16)] x)
(MOVBUreg (SRLconst [rc] x)) && rc < 8  => (UBFX [ssa.ArmBFAuxInt(rc,  8)] x)
// merge ANDconst into ubfx
// (x >> sc) & ac
(ANDconst [ac] (SRLconst [sc] x)) && isARM64BFMask(sc, ac, 0)
	=> (UBFX [ssa.ArmBFAuxInt(sc, arm64BFWidth(ac, 0))] x)
// (x & ac) >> sc
(SRLconst [sc] (ANDconst [ac] x)) && isARM64BFMask(sc, ac, sc)
	=> (UBFX [ssa.ArmBFAuxInt(sc, arm64BFWidth(ac, sc))] x)
// merge ANDconst and ubfx into ubfx
(ANDconst [c] (UBFX [bfc] x)) && isARM64BFMask(0, c, 0) =>
	(UBFX [ssa.ArmBFAuxInt(bfc.Lsb(), min(bfc.Width(), arm64BFWidth(c, 0)))] x)
(UBFX [bfc] (ANDconst [c] x)) && isARM64BFMask(0, c, 0) && bfc.Lsb() + bfc.Width() <= arm64BFWidth(c, 0) =>
	(UBFX [bfc] x)
// merge ubfx and zero-extension into ubfx
(MOVWUreg (UBFX [bfc] x)) && bfc.Width() <= 32 => (UBFX [bfc] x)
(MOVHUreg (UBFX [bfc] x)) && bfc.Width() <= 16 => (UBFX [bfc] x)
(MOVBUreg (UBFX [bfc] x)) && bfc.Width() <=  8 => (UBFX [bfc] x)

// Extracting bits from across a zero-extension boundary.
(UBFX [bfc] e:(MOVWUreg x))
	&& e.Uses == 1
	&& bfc.Lsb() < 32
	=> (UBFX [ssa.ArmBFAuxInt(bfc.Lsb(), min(bfc.Width(), 32-bfc.Lsb()))] x)
(UBFX [bfc] e:(MOVHUreg x))
	&& e.Uses == 1
	&& bfc.Lsb() < 16
	=> (UBFX [ssa.ArmBFAuxInt(bfc.Lsb(), min(bfc.Width(), 16-bfc.Lsb()))] x)
(UBFX [bfc] e:(MOVBUreg x))
	&& e.Uses == 1
	&& bfc.Lsb() < 8
	=> (UBFX [ssa.ArmBFAuxInt(bfc.Lsb(), min(bfc.Width(), 8-bfc.Lsb()))] x)

// ubfiz/ubfx combinations: merge shifts into bitfield ops
(SRLconst [sc] (UBFX [bfc] x)) && sc < bfc.Width()
	=> (UBFX [ssa.ArmBFAuxInt(bfc.Lsb()+sc, bfc.Width()-sc)] x)
(UBFX [bfc] (SRLconst [sc] x)) && sc+bfc.Width()+bfc.Lsb() < 64
	=> (UBFX [ssa.ArmBFAuxInt(bfc.Lsb()+sc, bfc.Width())] x)
(SLLconst [sc] (UBFIZ [bfc] x)) && sc+bfc.Width()+bfc.Lsb() < 64
	=> (UBFIZ [ssa.ArmBFAuxInt(bfc.Lsb()+sc, bfc.Width())] x)
(UBFIZ [bfc] (SLLconst [sc] x)) && sc < bfc.Width()
	=> (UBFIZ [ssa.ArmBFAuxInt(bfc.Lsb()+sc, bfc.Width()-sc)] x)
// ((x << c1) >> c2) >> c3
(SRLconst [sc] (UBFIZ [bfc] x)) && sc == bfc.Lsb()
	=> (ANDconst [1<<uint(bfc.Width())-1] x)
(SRLconst [sc] (UBFIZ [bfc] x)) && sc < bfc.Lsb()
	=> (UBFIZ [ssa.ArmBFAuxInt(bfc.Lsb()-sc, bfc.Width())] x)
(SRLconst [sc] (UBFIZ [bfc] x)) && sc > bfc.Lsb()
	&& sc < bfc.Lsb()+bfc.Width()
	=> (UBFX [ssa.ArmBFAuxInt(sc-bfc.Lsb(), bfc.Lsb()+bfc.Width()-sc)] x)
// ((x << c1) << c2) >> c3
(UBFX [bfc] (SLLconst [sc] x)) && sc == bfc.Lsb()
	=> (ANDconst [1<<uint(bfc.Width())-1] x)
(UBFX [bfc] (SLLconst [sc] x)) && sc < bfc.Lsb()
	=> (UBFX [ssa.ArmBFAuxInt(bfc.Lsb()-sc, bfc.Width())] x)
(UBFX [bfc] (SLLconst [sc] x)) && sc > bfc.Lsb()
	&& sc < bfc.Lsb()+bfc.Width()
	=> (UBFIZ [ssa.ArmBFAuxInt(sc-bfc.Lsb(), bfc.Lsb()+bfc.Width()-sc)] x)

// bfi
(OR (UBFIZ [bfc] x) (ANDconst [ac] y))
	&& ac == ^((1<<uint(bfc.Width())-1) << uint(bfc.Lsb()))
	=> (BFI [bfc] y x)
(ORshiftLL [s] (ANDconst [xc] x) (ANDconst [yc] y))
	&& xc == ^(yc << s)    // opposite masks
	&& yc & (yc+1) == 0    // power of 2 minus 1
	&& yc > 0              // not 0, not all 64 bits (there are better rewrites in that case)
	&& s+ssa.Log64(yc+1) <= 64 // shifted mask doesn't overflow
	=> (BFI [ssa.ArmBFAuxInt(s, ssa.Log64(yc+1))] x y)
(ORshiftRL [rc] (ANDconst [ac] x) (SLLconst [lc] y))
	&& lc > rc && ac == ^((1<<uint(64-lc)-1) << uint64(lc-rc))
	=> (BFI [ssa.ArmBFAuxInt(lc-rc, 64-lc)] x y)
// bfxil
(OR (UBFX [bfc] x) (ANDconst [ac] y)) && ac == ^(1<<uint(bfc.Width())-1)
	=> (BFXIL [bfc] y x)
(ORshiftLL [sc] (UBFX [bfc] x) (SRLconst [sc] y)) && sc == bfc.Width()
	=> (BFXIL [bfc] y x)
(ORshiftRL [rc] (ANDconst [ac] y) (SLLconst [lc] x)) && lc < rc && ac == ^((1<<uint(64-rc)-1))
	=> (BFXIL [ssa.ArmBFAuxInt(rc-lc, 64-rc)] y x)

// FP simplification
(FNEGS  (FMULS  x y)) => (FNMULS x y)
(FNEGD  (FMULD  x y)) => (FNMULD x y)
(FMULS  (FNEGS  x) y) => (FNMULS x y)
(FMULD  (FNEGD  x) y) => (FNMULD x y)
(FNEGS  (FNMULS x y)) => (FMULS  x y)
(FNEGD  (FNMULD x y)) => (FMULD  x y)
(FNMULS (FNEGS  x) y) => (FMULS  x y)
(FNMULD (FNEGD  x) y) => (FMULD  x y)

(FADDS a (FMULS  x y)) && a.Block.Func.UseFMA(v) => (FMADDS  a x y)
(FADDD a (FMULD  x y)) && a.Block.Func.UseFMA(v) => (FMADDD  a x y)
(FSUBS a (FMULS  x y)) && a.Block.Func.UseFMA(v) => (FMSUBS  a x y)
(FSUBD a (FMULD  x y)) && a.Block.Func.UseFMA(v) => (FMSUBD  a x y)
(FSUBS (FMULS  x y) a) && a.Block.Func.UseFMA(v) => (FNMSUBS a x y)
(FSUBD (FMULD  x y) a) && a.Block.Func.UseFMA(v) => (FNMSUBD a x y)
(FADDS a (FNMULS x y)) && a.Block.Func.UseFMA(v) => (FMSUBS  a x y)
(FADDD a (FNMULD x y)) && a.Block.Func.UseFMA(v) => (FMSUBD  a x y)
(FSUBS a (FNMULS x y)) && a.Block.Func.UseFMA(v) => (FMADDS  a x y)
(FSUBD a (FNMULD x y)) && a.Block.Func.UseFMA(v) => (FMADDD  a x y)
(FSUBS (FNMULS x y) a) && a.Block.Func.UseFMA(v) => (FNMADDS a x y)
(FSUBD (FNMULD x y) a) && a.Block.Func.UseFMA(v) => (FNMADDD a x y)

(MOVBUload [off] {sym} (SB) _) && ssa.SymIsRO(sym) => (MOVDconst [int64(ssa.Read8(sym, int64(off)))])
(MOVHUload [off] {sym} (SB) _) && ssa.SymIsRO(sym) => (MOVDconst [int64(ssa.Read16(sym, int64(off), config.Ctxt.Arch.ByteOrder))])
(MOVWUload [off] {sym} (SB) _) && ssa.SymIsRO(sym) => (MOVDconst [int64(ssa.Read32(sym, int64(off), config.Ctxt.Arch.ByteOrder))])
(MOVDload  [off] {sym} (SB) _) && ssa.SymIsRO(sym) => (MOVDconst [int64(ssa.Read64(sym, int64(off), config.Ctxt.Arch.ByteOrder))])
(MOVBload  [off] {sym} (SB) _) && ssa.SymIsRO(sym) => (MOVDconst [int64(int8(ssa.Read8(sym, int64(off))))])
(MOVHload  [off] {sym} (SB) _) && ssa.SymIsRO(sym) => (MOVDconst [int64(int16(ssa.Read16(sym, int64(off), config.Ctxt.Arch.ByteOrder)))])
(MOVWload  [off] {sym} (SB) _) && ssa.SymIsRO(sym) => (MOVDconst [int64(int32(ssa.Read32(sym, int64(off), config.Ctxt.Arch.ByteOrder)))])

// Prefetch instructions (aux is option: 0 - PLDL1KEEP; 1 - PLDL1STRM)
(PrefetchCache addr mem)         => (PRFM [0] addr mem)
(PrefetchCacheStreamed addr mem) => (PRFM [1] addr mem)

// Arch-specific inlining for small or disjoint runtime.memmove
(SelectN [0] call:(CALLstatic {sym} s1:(MOVDstore _ (MOVDconst [sz]) s2:(MOVDstore  _ src s3:(MOVDstore {t} _ dst mem)))))
	&& sz >= 0
	&& ssa.IsSameCall(sym, "runtime.memmove")
	&& s1.Uses == 1 && s2.Uses == 1 && s3.Uses == 1
	&& ssa.IsInlinableMemmove(dst, src, sz, config)
	&& ssa.Clobber(s1, s2, s3, call)
	=> (Move [sz] dst src mem)

// Match post-lowering calls, register version.
(SelectN [0] call:(CALLstatic {sym} dst src (MOVDconst [sz]) mem))
	&& sz >= 0
	&& ssa.IsSameCall(sym, "runtime.memmove")
	&& call.Uses == 1
	&& ssa.IsInlinableMemmove(dst, src, sz, config)
	&& ssa.Clobber(call)
	=> (Move [sz] dst src mem)

((REV|REVW) ((REV|REVW) p)) => p

// internal/runtime/math.MulUintptr intrinsics

(Select0 (Mul64uover x y)) => (MUL x y)
(Select1 (Mul64uover x y)) => (NotEqual (CMPconst (UMULH <typ.UInt64> x y) [0]))

// 32 mul 32 -> 64
(MUL r:(MOVWUreg x) s:(MOVWUreg y)) && r.Uses == 1 && s.Uses == 1 => (UMULL x y)
(MUL r:(MOVWreg  x) s:(MOVWreg  y)) && r.Uses == 1 && s.Uses == 1 =>  (MULL x y)

// ANDconst to zext
(ANDconst [0xffffffff] x) => (MOVWUreg x)
(ANDconst [0xffff    ] x) => (MOVHUreg x)
(ANDconst [0xff      ] x) => (MOVBUreg x)

// SIMD zero
(ZeroSIMD <t>) && t.Size() == 16 => (VMOVI16B [0] <t>)

// Neon peephole
(VEOR16B x x) => (VMOVI16B [0])
(VNOT16B (VCMEQ16B (VAND16B x y) (VMOVI16B [0]))) => (VCMTST16B x y)
(VNOT16B (VCMEQ8H (VAND16B x y) (VMOVI16B [0]))) => (VCMTST8H x y)
(VNOT16B (VCMEQ4S (VAND16B x y) (VMOVI16B [0]))) => (VCMTST4S x y)
(VNOT16B (VCMEQ2D (VAND16B x y) (VMOVI16B [0]))) => (VCMTST2D x y)
(ZeroSIMD <t>) && t.Size() == 32 && t.IsSIMD() => (ZDUPBconst [0])

// SVE scalable-vector load/store and runtime vector length.
(Load <t> ptr mem) && t.Size() == 32 && t.IsSIMD() => (ZLDRload ptr mem)
(Store {t} ptr val mem) && t.Size() == 32 && t.IsSIMD() => (ZSTRstore ptr val mem)
// SVE predicate (mask) load/store. A predicate is 8 bytes at the SSA level
// (types.TypeMask); it cannot move to a GP/FP register, so masks that cross a
// call or spill round-trip through memory via PLDR/PSTR. PSTR writes only VL/8
// bytes, so a store first zeroes the whole 8 bytes: the bits above the vector
// length read as zero, which is what lets user code read a mask's bits as a
// uint64.
(ZeroSIMD <t>) && t.Size() == 8 && t.IsSIMD() => (PPFALSEB)
(Load <t> ptr mem) && t.Size() == 8 && t.IsSIMD() => (PLDRload ptr mem)
(Store {t} ptr val mem) && t.Size() == 8 && t.IsSIMD() => (PSTRstore ptr val (MOVDstore ptr (MOVDconst [0]) mem))
(ScalableVectorLen) => (RDVL [1])

// SVE byte-granular predicated scalable load/store (LoadTs/Store slice APIs).
// These are written by hand until the predicated ops are generated by simdgen
// in the mask CL; at that point delete them and generate instead.
(Count8s r) => (Select0 <types.TypeMask> (PWHILELTB (MOVDconst [0]) r))

// SVE per-element select, backing IfElse and Masked.
(IfElseInt8s x mask y) => (ZSELB x y mask)
(IfElseUint8s x mask y) => (ZSELB x y mask)
(IfElseInt16s x mask y) => (ZSELH x y mask)
(IfElseUint16s x mask y) => (ZSELH x y mask)
(IfElseInt32s x mask y) => (ZSELS x y mask)
(IfElseUint32s x mask y) => (ZSELS x y mask)
(IfElseFloat32s x mask y) => (ZSELS x y mask)
(IfElseInt64s x mask y) => (ZSELD x y mask)
(IfElseUint64s x mask y) => (ZSELD x y mask)
(IfElseFloat64s x mask y) => (ZSELD x y mask)

// SVE scalar-to-vector broadcast. Integers come from a general register. A
// float scalar is held in element 0 of the vector register that shares its
// number, so a float broadcast is a broadcast of that element.
(BroadcastInt8s ...) => (ZDUPB ...)
(BroadcastUint8s ...) => (ZDUPB ...)
(BroadcastInt16s ...) => (ZDUPH ...)
(BroadcastUint16s ...) => (ZDUPH ...)
(BroadcastInt32s ...) => (ZDUPS ...)
(BroadcastUint32s ...) => (ZDUPS ...)
(BroadcastInt64s ...) => (ZDUPD ...)
(BroadcastUint64s ...) => (ZDUPD ...)
(BroadcastFloat32s x) => (ZDUPIS [0] x)
(BroadcastFloat64s x) => (ZDUPID [0] x)

// A broadcast of a small constant needs no general register: DUP (immediate)
// takes a signed 8-bit immediate.
(ZDUPB (MOVDconst [c])) && c == int64(int8(c)) => (ZDUPBconst [int8(c)])
(ZDUPH (MOVDconst [c])) && c == int64(int8(c)) => (ZDUPHconst [int8(c)])
(ZDUPS (MOVDconst [c])) && c == int64(int8(c)) => (ZDUPSconst [int8(c)])
(ZDUPD (MOVDconst [c])) && c == int64(int8(c)) => (ZDUPDconst [int8(c)])

// All-true predicate constructors: the same PWHILELT pattern the lowering
// synthesizes for implicitly predicated operations, so the select-fold
// peepholes recognize an explicit all-true mask and CSE dedups it with theirs.
(Mask8sAllTrue) => (Select0 <types.TypeMask> (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))
(Mask16sAllTrue) => (Select0 <types.TypeMask> (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))
(Mask32sAllTrue) => (Select0 <types.TypeMask> (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))
(Mask64sAllTrue) => (Select0 <types.TypeMask> (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))

// The first active lane is the one that follows the last active lane of an
// all-false mask.
(Mask8sFirst m) => (PPNEXTB (PPFALSEB) m)
(Mask16sFirst m) => (PPNEXTH (PPFALSEB) m)
(Mask32sFirst m) => (PPNEXTS (PPFALSEB) m)
(Mask64sFirst m) => (PPNEXTD (PPFALSEB) m)

(Mask8sNext m) => (PPNEXTB m (Mask8sAllTrue <types.TypeMask>))
(Mask16sNext m) => (PPNEXTH m (Mask16sAllTrue <types.TypeMask>))
(Mask32sNext m) => (PPNEXTS m (Mask32sAllTrue <types.TypeMask>))
(Mask64sNext m) => (PPNEXTD m (Mask64sAllTrue <types.TypeMask>))

// Every lane is active when no lane survives all-true AND NOT m.
(Mask8sAll m) => (Equal (Select1 <types.TypeFlags> (PBICSB t:(Mask8sAllTrue <types.TypeMask>) t m)))
(Mask16sAll m) => (Equal (Select1 <types.TypeFlags> (PBICSB t:(Mask16sAllTrue <types.TypeMask>) t m)))
(Mask32sAll m) => (Equal (Select1 <types.TypeFlags> (PBICSB t:(Mask32sAllTrue <types.TypeMask>) t m)))
(Mask64sAll m) => (Equal (Select1 <types.TypeFlags> (PBICSB t:(Mask64sAllTrue <types.TypeMask>) t m)))

(Mask8sNone m) => (Equal (PPTEST (Mask8sAllTrue <types.TypeMask>) m))
(Mask16sNone m) => (Equal (PPTEST (Mask16sAllTrue <types.TypeMask>) m))
(Mask32sNone m) => (Equal (PPTEST (Mask32sAllTrue <types.TypeMask>) m))
(Mask64sNone m) => (Equal (PPTEST (Mask64sAllTrue <types.TypeMask>) m))

(Mask8sAny m) => (NotEqual (PPTEST (Mask8sAllTrue <types.TypeMask>) m))
(Mask16sAny m) => (NotEqual (PPTEST (Mask16sAllTrue <types.TypeMask>) m))
(Mask32sAny m) => (NotEqual (PPTEST (Mask32sAllTrue <types.TypeMask>) m))
(Mask64sAny m) => (NotEqual (PPTEST (Mask64sAllTrue <types.TypeMask>) m))

(LoadMasked8 <t> ptr mask mem) && t.Size() == 32 => (ZLD1BPredload ptr mask mem)
(StoreMasked8 {t} ptr mask val mem) && t.Size() == 32 => (ZST1BPredstore ptr val mask mem)