out of memory when docker image prune

[denverdino]

Description

Get out of memory exception when prune many images

Steps to reproduce the issue:

# docker image prune -a --force --filter "until=720h"

Describe the results you received:

fatal error: runtime: out of memory

runtime stack:
runtime.throw(0x17e114c, 0x16)
/usr/local/go/src/runtime/panic.go:616 +0x81
runtime.sysMap(0xc426800000, 0x100000, 0x590000, 0x2555db8)
/usr/local/go/src/runtime/mem_linux.go:216 +0x20a
runtime.(*mheap).sysAlloc(0x253bbe0, 0x100000, 0x7f69cbfa4fb0)
/usr/local/go/src/runtime/malloc.go:470 +0xd4
runtime.(*mheap).grow(0x253bbe0, 0x20, 0x0)
/usr/local/go/src/runtime/mheap.go:907 +0x60
runtime.(*mheap).allocSpanLocked(0x253bbe0, 0x20, 0x2555dc8, 0x7f69cbfa46c8)
/usr/local/go/src/runtime/mheap.go:820 +0x301
runtime.(*mheap).alloc_m(0x253bbe0, 0x20, 0x101, 0x7f69cbfa46c8)
/usr/local/go/src/runtime/mheap.go:686 +0x118
runtime.(*mheap).alloc.func1()
/usr/local/go/src/runtime/mheap.go:753 +0x4d
runtime.(*mheap).alloc(0x253bbe0, 0x20, 0x7f69cb000101, 0x7f69cbfa46c8)
/usr/local/go/src/runtime/mheap.go:752 +0x8a
runtime.largeAlloc(0x3e971, 0x450100, 0x7f69cbfa46c8)
/usr/local/go/src/runtime/malloc.go:826 +0x94
runtime.mallocgc.func1()
/usr/local/go/src/runtime/malloc.go:721 +0x46
runtime.systemstack(0x0)
/usr/local/go/src/runtime/asm_amd64.s:409 +0x79
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1175

goroutine 1 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:363 fp=0xc4211ff880 sp=0xc4211ff878 pc=0x457b50
runtime.mallocgc(0x3e971, 0x0, 0xc4211ff900, 0x41403d)
/usr/local/go/src/runtime/malloc.go:720 +0x8a2 fp=0xc4211ff920 sp=0xc4211ff880 pc=0x4143d2
runtime.rawstring(0x3e971, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/runtime/string.go:245 +0x4f fp=0xc4211ff950 sp=0xc4211ff920 pc=0x447d9f
runtime.rawstringtmp(0x0, 0x3e971, 0x7f69cbff5000, 0xc420000180, 0xc420685cf0, 0x76, 0xc4205d2080)
/usr/local/go/src/runtime/string.go:115 +0x6f fp=0xc4211ff990 sp=0xc4211ff950 pc=0x4477ff
runtime.concatstrings(0x0, 0xc4211ffa70, 0x2, 0x2, 0xc42049d0e0, 0xc420152000)
/usr/local/go/src/runtime/string.go:46 +0x10a fp=0xc4211ffa28 sp=0xc4211ff990 pc=0x44735a
runtime.concatstring2(0x0, 0xc426786000, 0x3e920, 0xc42049d0e0, 0x51, 0xc426786000, 0x3e920)
/usr/local/go/src/runtime/string.go:55 +0x47 fp=0xc4211ffa68 sp=0xc4211ffa28 pc=0x447547
github.com/docker/cli/cli/command/image.runPrune(0x1985860, 0xc42020b6c0, 0x1420101, 0xc420699b30, 0x3, 0xc420619b98, 0xc4203a8de0, 0xc420619c68, 0x590630)
/go/src/github.com/docker/cli/cli/command/image/prune.go:81 +0x463 fp=0xc4211ffba0 sp=0xc4211ffa68 pc=0xb14fd3
github.com/docker/cli/cli/command/image.NewPruneCommand.func1(0xc4206be000, 0xc420695540, 0x0, 0x4, 0x0, 0x0)
/go/src/github.com/docker/cli/cli/command/image/prune.go:29 +0x64 fp=0xc4211ffc78 sp=0xc4211ffba0 pc=0xb1d4c4
github.com/docker/cli/vendor/github.com/spf13/cobra.(*Command).execute(0xc4206be000, 0xc42003a0a0, 0x4, 0x4, 0xc4206be000, 0xc42003a0a0)
/go/src/github.com/docker/cli/vendor/github.com/spf13/cobra/command.go:762 +0x468 fp=0xc4211ffd68 sp=0xc4211ffc78 pc=0x58f678
github.com/docker/cli/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc42034a000, 0xc420685f10, 0x1536ca0, 0xc420685f20)
/go/src/github.com/docker/cli/vendor/github.com/spf13/cobra/command.go:852 +0x30a fp=0xc4211ffea8 sp=0xc4211ffd68 pc=0x59008a
github.com/docker/cli/vendor/github.com/spf13/cobra.(*Command).Execute(0xc42034a000, 0xc42034a000, 0x194a740)
/go/src/github.com/docker/cli/vendor/github.com/spf13/cobra/command.go:800 +0x2b fp=0xc4211ffed8 sp=0xc4211ffea8 pc=0x58fd5b
main.main()
/go/src/github.com/docker/cli/cmd/docker/docker.go:174 +0xd0 fp=0xc4211fff88 sp=0xc4211ffed8 pc=0x1422600
runtime.main()
/usr/local/go/src/runtime/proc.go:198 +0x212 fp=0xc4211fffe0 sp=0xc4211fff88 pc=0x42f572
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc4211fffe8 sp=0xc4211fffe0 pc=0x45a6e1

goroutine 5 [syscall, 5 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 9 [chan receive]:
github.com/docker/cli/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x2535180)
/go/src/github.com/docker/cli/vendor/github.com/golang/glog/glog.go:879 +0x8b
created by github.com/docker/cli/vendor/github.com/golang/glog.init.0
/go/src/github.com/docker/cli/vendor/github.com/golang/glog/glog.go:410 +0x203

goroutine 12 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7f69cbf99f00, 0x72, 0xc42006d9a8)
/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420130b98, 0x72, 0xffffffffffffff00, 0x194ca20, 0x24beb88)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc420130b98, 0xc420265000, 0x1000, 0x1000)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420130b80, 0xc420265000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc420130b80, 0xc420265000, 0x1000, 0x1000, 0x4565d0, 0xc420000180, 0x4)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4205e82c8, 0xc420265000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc42053ec60, 0xc420265000, 0x1000, 0x1000, 0xc42006db98, 0x4072a5, 0xc420046c00)
/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4202d67e0)
/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4202d67e0, 0x1, 0x0, 0x0, 0x0, 0xc420046b40, 0x0)
/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc42053ec60)
/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 13 [select, 5 minutes]:
net/http.(*persistConn).writeLoop(0xc42053ec60)
/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 15 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f69cbf99e30, 0x72, 0xc4200709a8)
/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420131898, 0x72, 0xffffffffffffff00, 0x194ca20, 0x24beb88)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc420131898, 0xc42015d000, 0x1000, 0x1000)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420131880, 0xc42015d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc420131880, 0xc42015d000, 0x1000, 0x1000, 0x4565d0, 0xc420000180, 0x4)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4205e8308, 0xc42015d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc42053eea0, 0xc42015d000, 0x1000, 0x1000, 0xc420070b98, 0x4072a5, 0xc420046f00)
/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4202d6c60)
/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4202d6c60, 0x1, 0x0, 0x0, 0x0, 0xc420047080, 0x0)
/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc42053eea0)
/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 16 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc42053eea0)
/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1238 +0x97f

Describe the results you expected:

Prune image successfully

Additional information you deem important (e.g. issue happens only occasionally):

I have tons of out of images

Output of docker version:

Client:
 Version:           18.06.0-ce
 API version:       1.38
 Go version:        go1.10.3
 Git commit:        0ffa825
 Built:             Wed Jul 18 19:10:22 2018
 OS/Arch:           linux/amd64
 Experimental:      false

Server:
 Engine:
  Version:          18.06.0-ce
  API version:      1.38 (minimum version 1.12)
  Go version:       go1.10.3
  Git commit:       0ffa825
  Built:            Wed Jul 18 19:08:26 2018
  OS/Arch:          linux/amd64
  Experimental:     false

Output of docker info:

Containers: 1
 Running: 1
 Paused: 0
 Stopped: 0
Images: 411
Server Version: 18.06.0-ce
Storage Driver: aufs
 Root Dir: /var/lib/docker/aufs
 Backing Filesystem: extfs
 Dirs: 1064
 Dirperm1 Supported: false
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
 Volume: local
 Network: bridge host macvlan null overlay
 Log: awslogs fluentd gcplogs gelf journald json-file logentries splunk syslog
Swarm: inactive
Runtimes: runc
Default Runtime: runc
Init Binary: docker-init
containerd version: d64c661f1d51c48782c9cec8fda7604785f93587
runc version: 69663f0bd4b60df09991c08812a60108003fa340
init version: fec3683
Security Options:
 apparmor
Kernel Version: 3.13.0-86-generic
Operating System: Ubuntu 14.04.5 LTS
OSType: linux
Architecture: x86_64
CPUs: 1
Total Memory: 993.9MiB
Name: iZ62lox823oZ
ID: CGJL:JQP3:BXST:KEYH:XO4G:Q2N4:IQ76:CIRN:BPJJ:TTVF:R7UU:CB4A
Docker Root Dir: /var/lib/docker
Debug Mode (client): false
Debug Mode (server): false
Registry: https://index.docker.io/v1/
Labels:
Experimental: false
Insecure Registries:
 127.0.0.0/8
Live Restore Enabled: false

WARNING: No swap limit support

Additional environment details (AWS, VirtualBox, physical, etc.):