Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

OOM while exporting a large conversation #184

Closed
rusq opened this issue Jan 29, 2023 · 0 comments · Fixed by #185
Closed

OOM while exporting a large conversation #184

rusq opened this issue Jan 29, 2023 · 0 comments · Fixed by #185
Labels
bug Something isn't working
Milestone
v2.2.9

Comments

@rusq
Copy link
Owner

rusq commented Jan 29, 2023 

  Great tool! Sorry if this is a noob question or something wrong with my system but not sure how to navigate this.

I was able to successfully dump a view channels but having issues running the workplace export itself.

I used this command ./slackdump -export Workspace_export.zip -export-type standard -download

and it seems to be successful but after about 15 minutes each time I get this error:

2023/01/28 23:20:34 messages fetch complete, total: 500365
runtime: out of memory: cannot allocate 4194304-byte block (1727692800 in use)
fatal error: out of memory

goroutine 1 [running]:
runtime.throw({0x1272cb1, 0xd})
        /usr/local/go/src/runtime/panic.go:992 +0x64 fp=0x142b0930 sp=0x142b091c pc=0xde56b4
runtime.(*mcache).allocLarge(0x1b40c18, 0x18000, 0x0)
        /usr/local/go/src/runtime/mcache.go:215 +0x1d9 fp=0x142b0958 sp=0x142b0930 pc=0xdc42e9
runtime.mallocgc(0x18000, 0x1243500, 0x1)
        /usr/local/go/src/runtime/malloc.go:1096 +0x446 fp=0x142b09a0 sp=0x142b0958 pc=0xdbb716
runtime.growslice(0x1243500, {0x7de000, 0x84, 0x84}, 0x85)
        /usr/local/go/src/runtime/slice.go:278 +0x413 fp=0x142b09cc sp=0x142b09a0 pc=0xdf95e3
github.com/rusq/slackdump/v2/export.populateMsgs(0x469d4ca0, {0x4ef80000, 0x7a28d, 0x7c633}, 0x142b16c4)
        /github/workspace/export/conversation.go:68 +0x201 fp=0x142b0e8c sp=0x142b09cc pc=0x11795a1
github.com/rusq/slackdump/v2/export.Export.byDate({{0x132af14, 0x12496000}, 0x12510600, {0x132bde0, 0x12404480}, {0x132b6f8, 0x12496018}, {{0x0, 0x0, 0x0}, ...}}, ...)
        /github/workspace/export/conversation.go:22 +0x51 fp=0x142b0ef4 sp=0x142b0e8c pc=0x11790e1
github.com/rusq/slackdump/v2/export.(*Export).exportConversation(0x12452000, {0x132b918, 0x12496120}, 0x142b16c4, {{{{0x131d7bb0, 0x9}, 0x54da8046, 0x0, {0x0, 0x0}, ...}, ...}, ...})
        /github/workspace/export/export.go:201 +0x235 fp=0x142b118c sp=0x142b0ef4 pc=0x117b9d5
github.com/rusq/slackdump/v2/export.(*Export).exclusiveExport.func1({{{{0x131d7bb0, 0x9}, 0x54da8046, 0x0, {0x0, 0x0}, 0x0, 0x0, 0x0, 0x0, ...}, ...}, ...})
        /github/workspace/export/export.go:131 +0x22f fp=0x142b1274 sp=0x142b118c pc=0x117ac5f
github.com/rusq/slackdump/v2.(*Session).StreamChannels.func1({0x1342e000, 0x63, 0x8d})
        /github/workspace/channels.go:36 +0xa7 fp=0x142b13e4 sp=0x142b1274 pc=0x116fc67
github.com/rusq/slackdump/v2.(*Session).getChannels(0x12510600, {0x132b918, 0x12496138}, {0x15c1100, 0x4, 0x4}, 0x142b152c)
        /github/workspace/channels.go:77 +0x320 fp=0x142b1508 sp=0x142b13e4 pc=0x116ffe0
github.com/rusq/slackdump/v2.(*Session).StreamChannels(0x12510600, {0x132b918, 0x12496120}, {0x15c1100, 0x4, 0x4}, 0x1243d5bc)
        /github/workspace/channels.go:34 +0x73 fp=0x142b1538 sp=0x142b1508 pc=0x116fb93
github.com/rusq/slackdump/v2/export.(*Export).exclusiveExport(0x12452000, {0x132b918, 0x12496090}, 0x1243d6c4, 0x1247e930)
        /github/workspace/export/export.go:125 +0x2c9 fp=0x142b1634 sp=0x142b1538 pc=0x117a849
github.com/rusq/slackdump/v2/export.(*Export).exportChannels(0x12452000, {0x132b918, 0x12496090}, 0x1243d6c4)
        /github/workspace/export/export.go:111 +0x9b fp=0x142b1660 sp=0x142b1634 pc=0x117a53b
github.com/rusq/slackdump/v2/export.(*Export).messages(0x12452000, {0x132b918, 0x12496030}, {0x13200000, 0xd80, 0x1094})        /github/workspace/export/export.go:88 +0x27b fp=0x142b1798 sp=0x142b1660 pc=0x117a08b
github.com/rusq/slackdump/v2/export.(*Export).Run(0x12452000, {0x132b918, 0x1247eab0})
        /github/workspace/export/export.go:65 +0x1aa fp=0x142b17ec sp=0x142b1798 pc=0x1179cba
github.com/rusq/slackdump/v2/internal/app.Export({0x132b918, 0x1247ea98}, {{0x0, 0x0}, {0x1247e930}, {{0x126ae9d, 0x1}, {0x0, 0x0}, {0x0, ...}}, ...}, ...)
        /github/workspace/internal/app/export.go:52 +0x5ed fp=0x142b1a00 sp=0x142b17ec pc=0x119ab5d
github.com/rusq/slackdump/v2/internal/app.Run({0x132ba58, 0x12423800}, {{0x0, 0x0}, {0x1247e930}, {{0x126ae9d, 0x1}, {0x0, 0x0}, {0x0, ...}}, ...}, ...)
        /github/workspace/internal/app/app.go:25 +0x21a fp=0x142b1b18 sp=0x142b1a00 pc=0x119781a
main.run({0x132b8d8, 0x1241a11c}, {{{0x0, 0x0}, {0x1247e930}, {{0x126ae9d, 0x1}, {0x0, 0x0}, {0x0, ...}}, ...}, ...})
        /github/workspace/cmd/slackdump/main.go:147 +0x47d fp=0x142b1d98 sp=0x142b1b18 pc=0x119d15d
main.main()
        /github/workspace/cmd/slackdump/main.go:103 +0x547 fp=0x142b1fc4 sp=0x142b1d98 pc=0x119cb87
runtime.main()
        /usr/local/go/src/runtime/proc.go:250 +0x231 fp=0x142b1ff0 sp=0x142b1fc4 pc=0xde8051
runtime.goexit()
        /usr/local/go/src/runtime/asm_386.s:1326 +0x1 fp=0x142b1ff4 sp=0x142b1ff0 pc=0xe13011

goroutine 6 [syscall, 14 minutes]:
os/signal.signal_recv()
        /usr/local/go/src/runtime/sigqueue.go:151 +0x3b
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:23 +0x1a
created by os/signal.Notify.func1.1
        /usr/local/go/src/os/signal/signal.go:151 +0x27

goroutine 7 [select, 14 minutes]:
os/signal.NotifyContext.func1()
        /usr/local/go/src/os/signal/signal.go:288 +0x95
created by os/signal.NotifyContext
        /usr/local/go/src/os/signal/signal.go:287 +0x187

goroutine 51 [chan receive, 14 minutes]:
github.com/rusq/slackdump/v2/downloader.(*Client).fltSeen.func1()
        /github/workspace/downloader/filters.go:15 +0xce
created by github.com/rusq/slackdump/v2/downloader.(*Client).fltSeen
        /github/workspace/downloader/filters.go:7 +0xa2

goroutine 15 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x1237cd0c, 0x72)
        /usr/local/go/src/runtime/netpoll.go:302 +0x52
internal/poll.(*pollDesc).wait(0x12467174, 0x72, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x37
internal/poll.execIO(0x12467094, 0x1295ed8)
        /usr/local/go/src/internal/poll/fd_windows.go:175 +0xfc
internal/poll.(*FD).Read(0x12467080, {0x14724000, 0x9f5a, 0x9f5a})
        /usr/local/go/src/internal/poll/fd_windows.go:441 +0x13b
net.(*netFD).Read(0x12467080, {0x14724000, 0x9f5a, 0x9f5a})
        /usr/local/go/src/net/fd_posix.go:55 +0x3f
net.(*conn).Read(0x12406f00, {0x14724000, 0x9f5a, 0x9f5a})
        /usr/local/go/src/net/net.go:183 +0x4f
crypto/tls.(*atLeastReader).Read(0x46f2b530, {0x14724000, 0x9f5a, 0x9f5a})
        /usr/local/go/src/crypto/tls/conn.go:784 +0x63
bytes.(*Buffer).ReadFrom(0x1256016c, {0x1329a84, 0x46f2b530})
        /usr/local/go/src/bytes/buffer.go:204 +0x95
crypto/tls.(*Conn).readFromUntil(0x12560000, {0x1237cda8, 0x12406f00}, 0x5)
        /usr/local/go/src/crypto/tls/conn.go:806 +0xdd
crypto/tls.(*Conn).readRecordOrCCS(0x12560000, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:613 +0x122
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:581
crypto/tls.(*Conn).Read(0x12560000, {0x1257d000, 0x1000, 0x1000})
        /usr/local/go/src/crypto/tls/conn.go:1284 +0x147
bufio.(*Reader).Read(0x12578150, {0x125105a4, 0x9, 0x9})
        /usr/local/go/src/bufio/bufio.go:236 +0x1ce
io.ReadAtLeast({0x13299a8, 0x12578150}, {0x125105a4, 0x9, 0x9}, 0x9)
        /usr/local/go/src/io/io.go:331 +0x97
io.ReadFull(...)
        /usr/local/go/src/io/io.go:350
net/http.http2readFrameHeader({0x125105a4, 0x9, 0x9}, {0x13299a8, 0x12578150})
        /usr/local/go/src/net/http/h2_bundle.go:1566 +0x6e
net/http.(*http2Framer).ReadFrame(0x12510580)
        /usr/local/go/src/net/http/h2_bundle.go:1830 +0x89
net/http.(*http2clientConnReadLoop).run(0x2cf03fd0)
        /usr/local/go/src/net/http/h2_bundle.go:8815 +0x15d
net/http.(*http2ClientConn).readLoop(0x124012c0)
        /usr/local/go/src/net/http/h2_bundle.go:8711 +0x6e
created by net/http.(*http2Transport).newClientConn
        /usr/local/go/src/net/http/h2_bundle.go:7439 +0xaa1

goroutine 20 [select, 14 minutes]:
github.com/rusq/slackdump/v2/downloader.(*Client).worker(0x125821c0, {0x132b918, 0x12496090}, 0x1257b100)
        /github/workspace/downloader/downloader.go:186 +0xd5
github.com/rusq/slackdump/v2/downloader.(*Client).startWorkers.func1(0x0)
        /github/workspace/downloader/downloader.go:174 +0x68
created by github.com/rusq/slackdump/v2/downloader.(*Client).startWorkers
        /github/workspace/downloader/downloader.go:173 +0x54

goroutine 21 [select, 14 minutes]:
github.com/rusq/slackdump/v2/downloader.(*Client).worker(0x125821c0, {0x132b918, 0x12496090}, 0x1257ab80)
        /github/workspace/downloader/downloader.go:186 +0xd5
github.com/rusq/slackdump/v2/downloader.(*Client).startWorkers.func1(0x1)
        /github/workspace/downloader/downloader.go:174 +0x68
created by github.com/rusq/slackdump/v2/downloader.(*Client).startWorkers
        /github/workspace/downloader/downloader.go:173 +0x54

goroutine 22 [select, 14 minutes]:
github.com/rusq/slackdump/v2/downloader.(*Client).worker(0x125821c0, {0x132b918, 0x12496090}, 0x1257b500)
        /github/workspace/downloader/downloader.go:186 +0xd5
github.com/rusq/slackdump/v2/downloader.(*Client).startWorkers.func1(0x2)
        /github/workspace/downloader/downloader.go:174 +0x68
created by github.com/rusq/slackdump/v2/downloader.(*Client).startWorkers
        /github/workspace/downloader/downloader.go:173 +0x54

goroutine 23 [select, 14 minutes]:
github.com/rusq/slackdump/v2/downloader.(*Client).worker(0x125821c0, {0x132b918, 0x12496090}, 0x12416200)
        /github/workspace/downloader/downloader.go:186 +0xd5
github.com/rusq/slackdump/v2/downloader.(*Client).startWorkers.func1(0x3)
        /github/workspace/downloader/downloader.go:174 +0x68
created by github.com/rusq/slackdump/v2/downloader.(*Client).startWorkers
        /github/workspace/downloader/downloader.go:173 +0x54

goroutine 52 [chan receive, 14 minutes]:
github.com/rusq/slackdump/v2/downloader.(*Client).fltSeen.func1()
        /github/workspace/downloader/filters.go:15 +0xce
created by github.com/rusq/slackdump/v2/downloader.(*Client).fltSeen
        /github/workspace/downloader/filters.go:7 +0xa2

goroutine 53 [chan receive, 14 minutes]:
github.com/rusq/slackdump/v2/downloader.(*Client).fltSeen.func1()
        /github/workspace/downloader/filters.go:15 +0xce
created by github.com/rusq/slackdump/v2/downloader.(*Client).fltSeen
        /github/workspace/downloader/filters.go:7 +0xa2

goroutine 39 [chan receive, 14 minutes]:
github.com/rusq/slackdump/v2/downloader.(*Client).fltSeen.func1()
        /github/workspace/downloader/filters.go:15 +0xce
created by github.com/rusq/slackdump/v2/downloader.(*Client).fltSeen
        /github/workspace/downloader/filters.go:7 +0xa2

Originally posted by @DAJA48 in #183
@rusq rusq added the bug Something isn't working label Jan 29, 2023
@rusq rusq added this to the v2.2.9 milestone Jan 29, 2023
@rusq rusq mentioned this issue Jan 30, 2023
Merged
@rusq rusq closed this as completed in #185 Feb 2, 2023
rusq added a commit that referenced this issue Feb 2, 2023
@rusq
Merge pull request #185 from rusq/i184-oom
d4b7c94 
Fixing #184 out of memory
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
bug Something isn't working
Projects
None yet
Milestone
v2.2.9
Development

Successfully merging a pull request may close this issue.

Fixing #184 out of memory rusq/slackdump
1 participant
@rusq