Skip to content

Commit

Permalink
cmd/compile: move constant divide strength reduction to SSA rules
Browse files Browse the repository at this point in the history 
Currently the conversion from constant divides to multiplies is mostly
done during the walk pass.  This is suboptimal because SSA can
determine that the value being divided by is constant more often
(e.g. after inlining).

Change-Id: If1a9b993edd71be37396b9167f77da271966f85f
Reviewed-on: https://go-review.googlesource.com/37015
Run-TryBot: Keith Randall <khr@golang.org>
Reviewed-by: Josh Bleecher Snyder <josharian@gmail.com>
  • Loading branch information
@randall77
randall77 committed Feb 17, 2017
1 parent 794f1eb commit 708ba22
Show file tree
Hide file tree
Showing 26 changed files with 2,555 additions and 905 deletions.
1 change: 1 addition & 0 deletions src/cmd/compile/fmt_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,7 @@ var knownFormats = map[string]string{
"*cmd/internal/obj.Prog %s": "",
"*cmd/internal/obj.Prog %v": "",
"*math/big.Int %#x": "",
"*math/big.Int %s": "",
"[16]byte %x": "",
"[]*cmd/compile/internal/gc.Node %v": "",
"[]*cmd/compile/internal/gc.Sig %#v": "",
Expand Down
36 changes: 0 additions & 36 deletions src/cmd/compile/internal/gc/subr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1964,42 +1964,6 @@ func liststmt(l []*Node) *Node {
return n
}

// return power of 2 of the constant
// operand. -1 if it is not a power of 2.
// 1000+ if it is a -(power of 2)
func powtwo(n *Node) int {
if n == nil || n.Op != OLITERAL || n.Type == nil {
return -1
}
if !n.Type.IsInteger() {
return -1
}

v := uint64(n.Int64())
b := uint64(1)
for i := 0; i < 64; i++ {
if b == v {
return i
}
b = b << 1
}

if !n.Type.IsSigned() {
return -1
}

v = -v
b = 1
for i := 0; i < 64; i++ {
if b == v {
return i + 1000
}
b = b << 1
}

return -1
}

func ngotype(n *Node) *Sym {
if n.Type != nil {
return typenamesym(n.Type)
Expand Down
288 changes: 22 additions & 266 deletions src/cmd/compile/internal/gc/walk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1071,15 +1071,28 @@ opswitch:
break
}

// Try rewriting as shifts or magic multiplies.
n = walkdiv(n, init)

// rewrite 64-bit div and mod into function calls
// on 32-bit architectures.
switch n.Op {
case OMOD, ODIV:
if Widthreg >= 8 || (et != TUINT64 && et != TINT64) {
break opswitch
// rewrite 64-bit div and mod on 32-bit architectures.
// TODO: Remove this code once we can introduce
// runtime calls late in SSA processing.
if Widthreg < 8 && (et == TINT64 || et == TUINT64) {
if n.Right.Op == OLITERAL {
// Leave div/mod by constant powers of 2.
// The SSA backend will handle those.
switch et {
case TINT64:
c := n.Right.Int64()
if c < 0 {
c = -c
}
if c != 0 && c&(c-1) == 0 {
break opswitch
}
case TUINT64:
c := uint64(n.Right.Int64())
if c != 0 && c&(c-1) == 0 {
break opswitch
}
}
}
var fn string
if et == TINT64 {
Expand Down Expand Up @@ -3324,263 +3337,6 @@ func walkinrange(n *Node, init *Nodes) *Node {
return cmp
}

// walkdiv rewrites division by a constant as less expensive
// operations.
// The result of walkdiv MUST be assigned back to n, e.g.
// n.Left = walkdiv(n.Left, init)
func walkdiv(n *Node, init *Nodes) *Node {
// if >= 0, nr is 1<<pow // 1 if nr is negative.

if n.Right.Op != OLITERAL {
return n
}

// nr is a constant.
nl := cheapexpr(n.Left, init)

nr := n.Right

// special cases of mod/div
// by a constant
w := int(nl.Type.Width * 8)

s := 0 // 1 if nr is negative.
pow := powtwo(nr) // if >= 0, nr is 1<<pow
if pow >= 1000 {
// negative power of 2
s = 1

pow -= 1000
}

if pow+1 >= w {
// divisor too large.
return n
}

if pow < 0 {
// try to do division by multiply by (2^w)/d
// see hacker's delight chapter 10
// TODO: support 64-bit magic multiply here.
var m Magic
m.W = w

if nl.Type.IsSigned() {
m.Sd = nr.Int64()
smagic(&m)
} else {
m.Ud = uint64(nr.Int64())
umagic(&m)
}

if m.Bad != 0 {
return n
}

// We have a quick division method so use it
// for modulo too.
if n.Op == OMOD {
// rewrite as A%B = A - (A/B*B).
n1 := nod(ODIV, nl, nr)

n2 := nod(OMUL, n1, nr)
n = nod(OSUB, nl, n2)
goto ret
}

switch simtype[nl.Type.Etype] {
default:
return n

// n1 = nl * magic >> w (HMUL)
case TUINT8, TUINT16, TUINT32:
var nc Node

nodconst(&nc, nl.Type, int64(m.Um))
n1 := nod(OHMUL, nl, &nc)
n1 = typecheck(n1, Erv)
if m.Ua != 0 {
// Select a Go type with (at least) twice the width.
var twide *Type
switch simtype[nl.Type.Etype] {
default:
return n

case TUINT8, TUINT16:
twide = Types[TUINT32]

case TUINT32:
twide = Types[TUINT64]

case TINT8, TINT16:
twide = Types[TINT32]

case TINT32:
twide = Types[TINT64]
}

// add numerator (might overflow).
// n2 = (n1 + nl)
n2 := nod(OADD, conv(n1, twide), conv(nl, twide))

// shift by m.s
var nc Node

nodconst(&nc, Types[TUINT], int64(m.S))
n = conv(nod(ORSH, n2, &nc), nl.Type)
} else {
// n = n1 >> m.s
var nc Node

nodconst(&nc, Types[TUINT], int64(m.S))
n = nod(ORSH, n1, &nc)
}

// n1 = nl * magic >> w
case TINT8, TINT16, TINT32:
var nc Node

nodconst(&nc, nl.Type, m.Sm)
n1 := nod(OHMUL, nl, &nc)
n1 = typecheck(n1, Erv)
if m.Sm < 0 {
// add the numerator.
n1 = nod(OADD, n1, nl)
}

// shift by m.s
var ns Node

nodconst(&ns, Types[TUINT], int64(m.S))
n2 := conv(nod(ORSH, n1, &ns), nl.Type)

// add 1 iff n1 is negative.
var nneg Node

nodconst(&nneg, Types[TUINT], int64(w)-1)
n3 := nod(ORSH, nl, &nneg) // n4 = -1 iff n1 is negative.
n = nod(OSUB, n2, n3)

// apply sign.
if m.Sd < 0 {
n = nod(OMINUS, n, nil)
}
}

goto ret
}

switch pow {
case 0:
if n.Op == OMOD {
// nl % 1 is zero.
nodconst(n, n.Type, 0)
} else if s != 0 {
// divide by -1
n.Op = OMINUS

n.Right = nil
} else {
// divide by 1
n = nl
}

default:
if n.Type.IsSigned() {
if n.Op == OMOD {
// signed modulo 2^pow is like ANDing
// with the last pow bits, but if nl < 0,
// nl & (2^pow-1) is (nl+1)%2^pow - 1.
var nc Node

nodconst(&nc, Types[simtype[TUINT]], int64(w)-1)
n1 := nod(ORSH, nl, &nc) // n1 = -1 iff nl < 0.
if pow == 1 {
n1 = typecheck(n1, Erv)
n1 = cheapexpr(n1, init)

// n = (nl+ε)&1 -ε where ε=1 iff nl<0.
n2 := nod(OSUB, nl, n1)

var nc Node
nodconst(&nc, nl.Type, 1)
n3 := nod(OAND, n2, &nc)
n = nod(OADD, n3, n1)
} else {
// n = (nl+ε)&(nr-1) - ε where ε=2^pow-1 iff nl<0.
var nc Node

nodconst(&nc, nl.Type, (1<<uint(pow))-1)
n2 := nod(OAND, n1, &nc) // n2 = 2^pow-1 iff nl<0.
n2 = typecheck(n2, Erv)
n2 = cheapexpr(n2, init)

n3 := nod(OADD, nl, n2)
n4 := nod(OAND, n3, &nc)
n = nod(OSUB, n4, n2)
}

break
} else {
// arithmetic right shift does not give the correct rounding.
// if nl >= 0, nl >> n == nl / nr
// if nl < 0, we want to add 2^n-1 first.
var nc Node

nodconst(&nc, Types[simtype[TUINT]], int64(w)-1)
n1 := nod(ORSH, nl, &nc) // n1 = -1 iff nl < 0.
if pow == 1 {
// nl+1 is nl-(-1)
n.Left = nod(OSUB, nl, n1)
} else {
// Do a logical right right on -1 to keep pow bits.
var nc Node

nodconst(&nc, Types[simtype[TUINT]], int64(w)-int64(pow))
n2 := nod(ORSH, conv(n1, nl.Type.toUnsigned()), &nc)
n.Left = nod(OADD, nl, conv(n2, nl.Type))
}

// n = (nl + 2^pow-1) >> pow
n.Op = ORSH

var n2 Node
nodconst(&n2, Types[simtype[TUINT]], int64(pow))
n.Right = &n2
n.Typecheck = 0
}

if s != 0 {
n = nod(OMINUS, n, nil)
}
break
}

var nc Node
if n.Op == OMOD {
// n = nl & (nr-1)
n.Op = OAND

nodconst(&nc, nl.Type, nr.Int64()-1)
} else {
// n = nl >> pow
n.Op = ORSH

nodconst(&nc, Types[simtype[TUINT]], int64(pow))
}

n.Typecheck = 0
n.Right = &nc
}

goto ret

ret:
n = typecheck(n, Erv)
n = walkexpr(n, init)
return n
}

// return 1 if integer n must be in range [0, max), 0 otherwise
func bounded(n *Node, max int64) bool {
if n.Type == nil || !n.Type.IsInteger() {
Expand Down
2 changes: 2 additions & 0 deletions src/cmd/compile/internal/ssa/gen/386.rules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@

(Mul32uhilo x y) -> (MULLQU x y)

(Avg32u x y) -> (AVGLU x y)

(Div32F x y) -> (DIVSS x y)
(Div64F x y) -> (DIVSD x y)

Expand Down
2 changes: 2 additions & 0 deletions src/cmd/compile/internal/ssa/gen/386Ops.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,8 @@ func init() {

{name: "MULLQU", argLength: 2, reg: gp21mul, asm: "MULL", clobberFlags: true}, // arg0 * arg1, high 32 in result[0], low 32 in result[1]

{name: "AVGLU", argLength: 2, reg: gp21, commutative: true, resultInArg0: true, clobberFlags: true}, // (arg0 + arg1) / 2 as unsigned, all 32 result bits

{name: "DIVL", argLength: 2, reg: gp11div, asm: "IDIVL", clobberFlags: true}, // arg0 / arg1
{name: "DIVW", argLength: 2, reg: gp11div, asm: "IDIVW", clobberFlags: true}, // arg0 / arg1
{name: "DIVLU", argLength: 2, reg: gp11div, asm: "DIVL", clobberFlags: true}, // arg0 / arg1
Expand Down
3 changes: 3 additions & 0 deletions src/cmd/compile/internal/ssa/gen/ARM.rules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@
(Mod8 x y) -> (Mod32 (SignExt8to32 x) (SignExt8to32 y))
(Mod8u x y) -> (Mod32u (ZeroExt8to32 x) (ZeroExt8to32 y))

// (x + y) / 2 with x>=y -> (x - y) / 2 + y
(Avg32u <t> x y) -> (ADD (SRLconst <t> (SUB <t> x y) [1]) y)

(And32 x y) -> (AND x y)
(And16 x y) -> (AND x y)
(And8 x y) -> (AND x y)
Expand Down
3 changes: 2 additions & 1 deletion src/cmd/compile/internal/ssa/gen/ARM64.rules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@
(Mod8 x y) -> (MODW (SignExt8to32 x) (SignExt8to32 y))
(Mod8u x y) -> (UMODW (ZeroExt8to32 x) (ZeroExt8to32 y))

(Avg64u <t> x y) -> (ADD (ADD <t> (SRLconst <t> x [1]) (SRLconst <t> y [1])) (AND <t> (AND <t> x y) (MOVDconst [1])))
// (x + y) / 2 with x>=y -> (x - y) / 2 + y
(Avg64u <t> x y) -> (ADD (SRLconst <t> (SUB <t> x y) [1]) y)

(And64 x y) -> (AND x y)
(And32 x y) -> (AND x y)
Expand Down
Loading

0 comments on commit 708ba22

Please sign in to comment.