Skip to content

runtime: gc goroutine occasionally use 100% CPU #61046

Open
@lixin1234qqq

Description

@lixin1234qqq

What version of Go are you using (go version)?

go version 1.19.4
linux
amd64

Does this issue reproduce with the latest release?

not sure

What operating system and processor architecture are you using (go env)?

GOARCH="amd64"
GOOS="linux"

What did you do?

run my application(using cgo) bind to last processor using taskset -c xx
(I'm looking for a way to reproduce this issue stably...)

What did you expect to see?

the cpu usage is stable

What did you see instead?

the cpu usage increased to 100% occasionally sometime
If I a) attached and detached the process using gdb or b) sent SIGSTOP and SIGCONT,the cpu usage was restored

top -Hp pid
one thread used 100% cpu

perf record -F 999 -t tid
result:

44.63%  my-application  [.] runtime.unlock2
37.76%  my-application  [.] runtime.lock2
 6.40%  my-application  [.] runtime.sweepone
 5.02%  my-application  [.] runtime.freeSomeWbufs
 4.91%  my-application  [.] runtime.bgsweep
 0.04%  my-application  [.] runtime.cgoSigtramp.abi0
 0.02%  my-application  [.] runtime.findfunc
 0.01%  my-application  [.] runtime.sigsend
 0.01%  my-application  [.] runtime.findRunnable
 0.01%  my-application  [.] runtime.adjustSignalStack
 0.01%  my-application  [.] runtime.sigfwdgo
 0.01%  my-application  [.] runtime.morestack.abi0
 0.01%  my-application  [.] runtime.doSigPreempt
 0.01%  my-application  [.] runtime.casgstatus
 0.01%  my-application  [.] runtime.isAsyncSafePoint
 0.01%  my-application  [.] runtime.sigtramp.abi0
 0.01%  my-application  [.] runtime.sigtrampgo
Percent│     Disassembly of section .text
       │
       │     000000000083d280 <runtime.bgsweep@@Base>:
       │  0:   cmp    0x10(%r14),%rsp
       │     ↓ jbe    12e
       │       sub    $0x30,%rsp
       │       mov    %rbp,0x28(%rsp
       │       lea    0x28(%rsp),%rbp
       │       mov    %rax,0x20(%rsp
       │       cmpl   $0x0,runtime.writeBarrier@@Base
       │     ↓ jne    32
       │       mov    %r14,%rcx
       │       mov    %rcx,runtime.sweep@@Base+0x8
       │     ↓ jmp    45
       │ 32:   lea    runtime.sweep@@Base+0x8,%rdi
       │       mov    %r14,%rcx
       │       nop
       │     → callq  runtime.gcWriteBarrierCX
       │ 45:   nop
       │       nop
       │       lea    runtime.sweep@@Base,%rax
       │     → callq  runtime.lock2
       │       movb   $0x1,runtime.sweep@@Base+0x10
       │       mov    0x20(%rsp),%rax
       │       lea    go.link.pkghashbytes.vendor/golang.org/x/text/unicode/norm@@Base+0x120,%rbx
       │     → callq  runtime.chansend1
       │       nop
       │       lea    runtime.parkunlock_c·f@@Base,%rax
       │       lea    runtime.sweep@@Base,%rbx
       │       mov    $0xc,%ecx
       │       mov    $0x14,%edi
       │       mov    $0x1,%esi
       │     → callq  runtime.gopark
       │     ↓ jmp    100
       │ 90:   nop
       │       nop
       │       lea    runtime.sweep@@Base,%rax
 15.82 │     → callq  runtime.lock2
  0.10 │       nop
       │       mov    runtime.sweep@@Base+0x1c,%ecx
 17.25 │       cmp    $0x80000000,%ecx
       │     ↓ jne    d9
       │       movb   $0x1,runtime.sweep@@Base+0x
       │       nop
       │       lea    runtime.parkunlock_c·f@@Base,%rax
       │       lea    runtime.sweep@@Base,%rbx
       │       mov    $0xc,%ecx
       │       mov    $0x14,%edi
       │       mov    $0x1,%esi
       │     → callq  runtime.gopark
       │     ↓ jmp    100
       │ d9:   nop
       │       nop
       │       lea    runtime.sweep@@Base,%rax
 17.14 │     → callq  runtime.unlock2
       │     ↓ jmp    100
       │ e9:   incl   runtime.sweep@@Base+0x14
       │       nop
       │       lea    runtime.gosched_m·f@@Base,%rax
       │     → callq  runtime.mcall
       │       nop
 16.74 │100: → callq  runtime.sweepone
       │       cmp    $0xffffffffffffffff,%rax
       │     ↑ jne    e9
       │     ↓ jmp    11a
       │10d:   nop
       │       lea    runtime.gosched_m·f@@Base,%rax
       │     → callq  runtime.mcall
 17.75 │11a:   mov    $0x1,%eax
       │       nop
       │     → callq  runtime.freeSomeWbufs
       │       test   %al,%al
       │     ↑ jne    10d
 15.21 │     ↑ jmpq   90
       │12e:   mov    %rax,0x8(%rsp
       │     → callq  runtime.morestack_noctxt.abi0
       │       mov    0x8(%rsp),%rax
       │       nop
       │       jmpq   0




Percent│     Disassembly of section .text
       │
       │     000000000083d4e0 <runtime.sweepone@@Base>:
 12.27 │  0:   cmp    0x10(%r14),%rsp
       │     ↓ jbe    24c
       │       sub    $0x48,%rsp
       │       mov    %rbp,0x40(%rsp
 13.23 │       lea    0x40(%rsp),%rbp
       │       mov    %r14,0x38(%rsp
       │       mov    0x30(%r14),%rcx
       │       incl   0x108(%rcx
 12.05 │ 27:   nop
       │       mov    runtime.sweep@@Base+0x1c,%eax
 11.58 │       mov    %eax,%ecx
  0.08 │       and    $0x80000000,%eax
       │       test   %eax,%eax
       │     ↓ jne    5c
       │       lea    0x1(%rcx),%edx
       │       mov    %eax,%ebx
       │       mov    %ecx,%eax
       │       lea    runtime.sweep@@Base+0x1c,%rsi
       │       lock   cmpxchg %edx,(%rsi
       │       sete   %cl
       │       test   %cl,%cl
       │     ↑ je     27
       │       mov    runtime.mheap_@@Base+0x10138,%ecx
       │       test   %ebx,%ebx
       │     ↓ jmp    69
       │ 5c:   mov    runtime.mheap_@@Base+0x10138,%ecx
 12.04 │       lea    runtime.sweep@@Base+0x1c,%rsi
       │ 69:   mov    %ecx,0x20(%rsp
       │       sete   0x24(%rsp
 12.45 │     ↓ je     94
  0.16 │       mov    0x38(%rsp),%rcx
       │       mov    0x30(%rcx),%rcx
       │       decl   0x108(%rcx
 13.70 │       mov    $0xffffffffffffffff,%rax
       │       mov    0x40(%rsp),%rbp
 12.44 │       add    $0x48,%rsp
       │     ← retq        
       │ 94:   lea    runtime.mheap_@@Base,%rax
       │       nop
       │     → callq  runtime.(*mheap).nextSpanForSweep
       │       test   %rax,%rax
       │     ↓ je     115
       │       mov    0x63(%rax),%cl
       │       cmp    $0x1,%cl
       │     ↓ je     c9
       │       mov    0x58(%rax),%edx
       │       mov    0x20(%rsp),%ebx
       │       cmp    %edx,%ebx
       │     ↑ je     94
       │       lea    0x3(%rbx),%esi
       │       cmp    %esi,%edx
       │     ↑ je     94
       │     ↓ jmpq   1cd
       │ c9:   mov    %rax,%rbx
       │       lea    0x20(%rsp),%rax
       │     → callq  runtime.(*sweepLocker).tryAcquire
       │       mov    %rax,0x28(%rsp
       │       test   %bl,%bl
       │     ↑ je     94
       │       mov    0x20(%rax),%rcx
       │       mov    %rcx,0x30(%rsp
       │       lea    0x28(%rsp),%rax
       │       xor    %ebx,%ebx
       │     → callq  runtime.(*sweepLocked).sweep
       │       test   %al,%al
       │     ↓ je     10f
       │       nop
       │       mov    0x30(%rsp),%rcx
       │       lea    runtime.mheap_@@Base+0x10188,%rdx
       │       mov    %rcx,%rsi
       │       lock   xadd   %rcx,(%rdx
       │     ↓ jmp    111
       │10f:   xor    %esi,%esi
       │111:   xor    %edx,%edx
       │     ↓ jmp    150    	                              	   
       │115:   nop
       │       mov    runtime.sweep@@Base+0x1c,%eax
       │       mov    %eax,%edx
       │       and    $0x80000000,%eax
       │       test   %eax,%eax
       │     ↓ jne    146
       │       mov    %edx,%esi
       │       bts    $0x1f,%edx
       │       mov    %eax,%ecx
       │       mov    %esi,%eax
       │       lea    runtime.sweep@@Base+0x1c,%rdi
       │       lock   cmpxchg %edx,(%rdi
       │       sete   %dl
       │       nop
       │       test   %dl,%dl
       │     ↑ je     115
       │       test   %ecx,%ecx
       │146:   sete   %dl
       │       mov    $0xffffffffffffffff,%rsi
       │150:   mov    %dl,0x16(%rsp
       │       mov    %rsi,0x30(%rsp
       │       mov    0x20(%rsp),%ebx
       │       movzbl 0x24(%rsp),%ecx
       │       lea    runtime.sweep@@Base+0x1c,%rax
       │     → callq  runtime.(*activeSweep).end
       │       movzbl 0x16(%rsp),%edx
       │       test   %dl,%dl
       │     ↓ je     1af
       │       cmpl   $0x0,runtime.debug@@Base+0x28
       │       xchg   %ax,%ax
       │     ↓ jle    19f
       │       lea    runtime.sweepone.func1·f@@Base,%rax
       │       mov    %rax,(%rsp
       │     → callq  runtime.systemstack.abi0
       │       xorps  %xmm15,%xmm15
       │       mov    %fs:0xfffffffffffffff8,%r14
       │19f:   nop
       │       nop
       │       mov    $0x1,%ecx
       │       lea    runtime.scavenger@@Base+0x20,%rdx
       │       xchg   %ecx,(%rdx)  	 
       │1af:   mov    0x38(%rsp),%rcx
       │       mov    0x30(%rcx),%rcx
       │       decl   0x108(%rcx
       │       mov    0x30(%rsp),%rax
       │       mov    0x40(%rsp),%rbp
       │       add    $0x48,%rsp
       │     ← retq
       │1cd:   mov    %cl,0x17(%rsp
       │       mov    %edx,0x1c(%rsp
       │       mov    %ebx,0x18(%rsp
       │     → callq  runtime.printlock
       │       lea    type.RMd5Pq1o@@Base+0x18daa,%rax
       │       mov    $0x1a,%ebx
       │     → callq  runtime.printstring
       │       movzbl 0x17(%rsp),%eax
       │       movzbl %al,%eax
       │     → callq  runtime.printuint
       │       lea    type.RMd5Pq1o@@Base+0x85f9,%rax
       │       mov    $0xc,%ebx
       │     → callq  runtime.printstring
       │       mov    0x1c(%rsp),%eax
       │     → callq  runtime.printuint
       │       lea    type.RMd5Pq1o@@Base+0x61c5,%rax
       │       mov    $0xa,%ebx
       │     → callq  runtime.printstring
       │       mov    0x18(%rsp),%eax
       │     → callq  runtime.printuint
       │     → callq  runtime.printnl
       │     → callq  runtime.printunlock
       │       lea    type.RMd5Pq1o@@Base+0x1f1d9,%rax
       │       mov    $0x1f,%ebx
       │     → callq  runtime.throw
       │       nop
       │24c: → callq  runtime.morestack_noctxt.abi0
       │     ↑ jmpq   0	     




Percent│    Disassembly of section .text
       │
       │    000000000083f6e0 <runtime.freeSomeWbufs@@Base>:
 16.47 │ 0:   cmp    0x10(%r14),%rsp
       │    ↓ jbe    c3
       │      sub    $0x28,%rsp
       │      mov    %rbp,0x20(%rsp
 11.97 │      lea    0x20(%rsp),%rbp
  6.99 │      mov    %al,0x30(%rsp
       │      nop
       │      lea    runtime.work@@Base+0x50,%rax
 10.88 │    → callq  runtime.lock2
       │      cmpl   $0x0,runtime.gcphase@@Base
 18.66 │    ↓ jne    3c
       │      cmpq   $0x0,runtime.work@@Base+0x58
       │    ↓ jne    56
 18.66 │3c:   nop
       │      nop
       │      lea    runtime.work@@Base+0x50,%rax
       │    → callq  runtime.unlock2
       │      xor    %eax,%eax
       │      mov    0x20(%rsp),%rbp
 16.36 │      add    $0x28,%rsp
       │    ← retq
       │56:   movq   $0x0,0x10(%rsp
       │      movb   $0x0,0x18(%rsp
       │      lea    runtime.freeSomeWbufs.func1@@Base,%rax
       │      mov    %rax,0x10(%rsp
       │      movzbl 0x30(%rsp),%eax
       │      mov    %al,0x18(%rsp
       │      lea    0x10(%rsp),%rax
       │      mov    %rax,(%rsp
       │    → callq  runtime.systemstack.abi0
       │      xorps  %xmm15,%xmm15
       │      mov    %fs:0xfffffffffffffff8,%r14
       │      nop
       │      nop
       │      mov    runtime.work@@Base+0x58,%rax
       │      mov    %rax,0x8(%rsp
       │      lea    runtime.work@@Base+0x50,%rax
       │    → callq  runtime.unlock2
       │      mov    0x8(%rsp),%rax
       │      test   %rax,%rax
       │      setne  %al
       │      mov    0x20(%rsp),%rbp
       │      add    $0x28,%rsp
       │    ← retq
       │c3:   mov    %al,0x8(%rsp
       │    → callq  runtime.morestack_noctxt.abi0
       │      movzbl 0x8(%rsp),%eax
       │    ↑ jmpq   0 

Metadata

Metadata

Assignees

No one assigned

    Labels

    NeedsInvestigationSomeone must examine and confirm this is a valid issue and not a duplicate of an existing one.compiler/runtimeIssues related to the Go compiler and/or runtime.

    Type

    No type

    Projects

    Status

    Todo

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions