Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Deadlock while styling some pages #28659

Open
jdm opened this issue Jan 1, 2022 · 2 comments
Open

Deadlock while styling some pages #28659

jdm opened this issue Jan 1, 2022 · 2 comments
Labels
A-layout/uncategorized I-perf-slow Unnecessary performance degredation.

Comments

@jdm
Copy link
Member

jdm commented Jan 1, 2022

I can trigger a deadlock in a debug build visiting https://github.com/ethereum/go-ethereum/wiki/Geth and then clicking a link to another wiki page and resizing the page. The new layout thread is sitting in:

* thread #72, name = 'Layout(1,2)'
  * frame #0: 0x00007fff70315882 libsystem_kernel.dylib`__psynch_cvwait + 10
    frame #1: 0x00007fff703d6425 libsystem_pthread.dylib`_pthread_cond_wait + 698
    frame #2: 0x000000010ef04f46 servo`std::sys::unix::condvar::Condvar::wait::h8bfa6fc5f8adbe95(self=0x0000000143ee03b0, mutex=0x000000014238f500) at condvar.rs:82:17
    frame #3: 0x000000010ef24526 servo`std::sys_common::condvar::Condvar::wait::hb2c61e7a5f831fe0(self=0x00007ff8d1a4b468, mutex=0x00007ff8d1a4b458) at condvar.rs:44:9
    frame #4: 0x000000010ef051ce servo`std::sync::condvar::Condvar::wait::hbdf5e22aa730cf38(self=0x00007ff8d1a4b468, guard=MutexGuard<bool> @ 0x000070000db32d10) at condvar.rs:189:13
    frame #5: 0x000000010eef8e64 servo`rayon_core::latch::LockLatch::wait_and_reset::h4c5c6523ed629fc1(self=0x00007ff8d1a4b458) at latch.rs:240:21
    frame #6: 0x0000000109cee95d servo`rayon_core::registry::Registry::in_worker_cold::_$u7b$$u7b$closure$u7d$$u7d$::h42430639cbecff44(l=0x00007ff8d1a4b458) at registry.rs:470:13
    frame #7: 0x0000000109dfa57d servo`std::thread::local::LocalKey$LT$T$GT$::try_with::h4b3cae8c8d4f9ce2(self=0x00000001128dcef0, f={closure#0} @ 0x000070000db33100) at local.rs:399:16
    frame #8: 0x0000000109df9aee servo`std::thread::local::LocalKey$LT$T$GT$::with::h59e0bfa75e360297(self=0x00000001128dcef0, f=<unavailable>) at local.rs:375:9
    frame #9: 0x000000010fb497a2 servo`rayon_core::registry::Registry::in_worker_cold::h8754c17a48f402e5(self=0x0000000137a40e00, op=<unavailable>) at registry.rs:458:9
    frame #10: 0x0000000109cef04d servo`rayon_core::registry::Registry::in_worker::ha3eedb86091d9dac(self=0x0000000137a40e00, op={closure#0} @ 0x000070000db333f0) at registry.rs:438:17
    frame #11: 0x0000000109db7fcd servo`rayon_core::thread_pool::ThreadPool::install::h363b5a0f9cb14b2a(self=0x0000000112c4d1a0, op={closure#1} @ 0x000070000db33ab8) at mod.rs:111:9
    frame #12: 0x0000000109d7dfd6 servo`style::driver::traverse_dom::h6b8abf2ba6d3e0a9(traversal=0x000070000db350a8, token=(__0 = core::option::Option<layout_thread::dom_wrapper::ServoLayoutElement> @ 0x000070000db33c48), pool=Option<&rayon_core::thread_pool::ThreadPool> @ 0x000070000db335c8) at driver.rs:136:17
    frame #13: 0x0000000109d1d732 servo`layout_thread::LayoutThread::handle_reflow::_$u7b$$u7b$closure$u7d$$u7d$::h6decd735238dd89a at lib.rs:1436:32
    frame #14: 0x0000000109d44a64 servo`profile_traits::time::profile::he626dcc8006b0295(category=LayoutStyleRecalc, meta=Option<profile_traits::time::TimerMetadata> @ 0x000070000db352b8, profiler_chan=ProfilerChan @ 0x000070000db33d44, callback={closure#3} @ 0x000070000db352d8) at time.rs:141:15
    frame #15: 0x0000000109d1c27c servo`layout_thread::LayoutThread::handle_reflow::h8b448d7a85e75c65(self=0x000070000db37a18, data=0x000070000db35cf8, possibly_locked_rw_data=0x000070000db379e0) at lib.rs:1430:13
    frame #16: 0x0000000109d1601d servo`layout_thread::LayoutThread::handle_request_helper::_$u7b$$u7b$closure$u7d$$u7d$::h1f44e4edfbc4c4c9 at lib.rs:751:24
    frame #17: 0x0000000109d44164 servo`profile_traits::time::profile::h7b8e94a0575b1dc5(category=LayoutPerform, meta=Option<profile_traits::time::TimerMetadata> @ 0x000070000db35f10, profiler_chan=ProfilerChan @ 0x000070000db358f4, callback={closure#0} @ 0x000070000db35f30) at time.rs:141:15
    frame #18: 0x0000000109d1599b servo`layout_thread::LayoutThread::handle_request_helper::h716625da0575f847(self=0x000070000db37a18, request=Msg @ 0x000070000db37458, possibly_locked_rw_data=0x000070000db379e0) at lib.rs:747:17
    frame #19: 0x0000000109d144f6 servo`layout_thread::LayoutThread::handle_request::h18bcbb91f30e0ed9(self=0x000070000db37a18, possibly_locked_rw_data=0x000070000db379e0) at lib.rs:695:41
    frame #20: 0x0000000109d13682 servo`layout_thread::LayoutThread::start::h2679f8fdbfb81d73(self=LayoutThread @ 0x000070000db37a18) at lib.rs:592:15
    frame #21: 0x0000000109d10e3b servo`_$LT$layout_thread..LayoutThread$u20$as$u20$layout_traits..LayoutThreadFactory$GT$::create::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h3c506d32eb5605b7 at lib.rs:339:29
    frame #22: 0x0000000109ca30bb servo`profile_traits::mem::ProfilerChan::run_with_memory_reporting::h0b5469c294d260b4(self=0x000070000db3b0a4, f={closure#0} @ 0x000070000db3a308, reporter_name=String @ 0x000070000db3af10, channel_for_reporter=(flavor = crossbeam_channel::channel::SenderFlavor<script_layout_interface::message::Msg> @ 0x000070000db38690), msg=(servo`_$LT$msg..constellation_msg..PipelineId$u20$as$u20$core..fmt..Display$GT$::fmt::h2f6f7ba5d5193a12 at constellation_msg.rs:232)) at mem.rs:88:9
    frame #23: 0x0000000109d11695 servo`_$LT$layout_thread..LayoutThread$u20$as$u20$layout_traits..LayoutThreadFactory$GT$::create::_$u7b$$u7b$closure$u7d$$u7d$::hb6c4cb57a4161ad2 at lib.rs:337:21
    frame #24: 0x0000000109dc40bb servo`std::sys_common::backtrace::__rust_begin_short_backtrace::h419a84b813fd0da6(f=<unavailable>) at backtrace.rs:123:18
    frame #25: 0x0000000109ca652b servo`std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::hc305369c6ced6d75 at mod.rs:484:17
    frame #26: 0x0000000109d7388b servo`_$LT$core..panic..unwind_safe..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h92b1b57ad7ce7ef1(self=<unavailable>, _args=<unavailable>) at unwind_safe.rs:271:9
    frame #27: 0x0000000109d7849a servo`std::panicking::try::do_call::h6578162d2805dd1c(data="\x01") at panicking.rs:406:40
    frame #28: 0x0000000109d93efd servo`__rust_try + 29
    frame #29: 0x0000000109d781e8 servo`std::panicking::try::hfe2a9c0cb5a9870f(f=<unavailable>) at panicking.rs:370:19
    frame #30: 0x0000000109ca59cb servo`std::panic::catch_unwind::h38011a9e01305672(f=<unavailable>) at panic.rs:133:14
    frame #31: 0x0000000109ca6384 servo`std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::he18f88e552cd4ec5 at mod.rs:483:30
    frame #32: 0x0000000109dc48c1 servo`core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::hd3f2b4d1869ec70a((null)=0x0000000137959d00, (null)=<unavailable>) at function.rs:227:5
    frame #33: 0x000000010fb1a997 servo`std::sys::unix::thread::Thread::new::thread_start::h0190e77bc7633265 [inlined] _$LT$alloc..boxed..Box$LT$F$C$A$GT$$u20$as$u20$core..ops..function..FnOnce$LT$Args$GT$$GT$::call_once::h579f3b223634893c at boxed.rs:1694:9 [opt]
    frame #34: 0x000000010fb1a991 servo`std::sys::unix::thread::Thread::new::thread_start::h0190e77bc7633265 [inlined] _$LT$alloc..boxed..Box$LT$F$C$A$GT$$u20$as$u20$core..ops..function..FnOnce$LT$Args$GT$$GT$::call_once::h805b8d91b6f45bdd at boxed.rs:1694 [opt]
    frame #35: 0x000000010fb1a98a servo`std::sys::unix::thread::Thread::new::thread_start::h0190e77bc7633265 at thread.rs:108 [opt]
    frame #36: 0x00007fff703d6109 libsystem_pthread.dylib`_pthread_start + 148
    frame #37: 0x00007fff703d1b8b libsystem_pthread.dylib`thread_start + 15

Meanwhile, there are two rayon registry threads like this:

* thread #40
  * frame #0: 0x00007fff70315882 libsystem_kernel.dylib`__psynch_cvwait + 10
    frame #1: 0x00007fff703d6425 libsystem_pthread.dylib`_pthread_cond_wait + 698
    frame #2: 0x000000010ef04f46 servo`std::sys::unix::condvar::Condvar::wait::h8bfa6fc5f8adbe95(self=0x000000012b7e18d0, mutex=0x000000012b769c00) at condvar.rs:82:17
    frame #3: 0x000000010ef24526 servo`std::sys_common::condvar::Condvar::wait::hb2c61e7a5f831fe0(self=0x000000012b7e7790, mutex=0x000000012b7e7780) at condvar.rs:44:9
    frame #4: 0x000000010ef051ce servo`std::sync::condvar::Condvar::wait::hbdf5e22aa730cf38(self=0x000000012b7e7790, guard=MutexGuard<bool> @ 0x000070000a0eae70) at condvar.rs:189:13
    frame #5: 0x000000010fbc055a servo`rayon_core::sleep::Sleep::sleep::h2c459b8a83f28b92(self=0x000000012b7f11a8, idle_state=0x000070000a0eb1e0, latch=0x000000012b7da5e0, has_injected_jobs={closure#2} @ 0x000070000a0eb028) at mod.rs:226:30
    frame #6: 0x000000010ef138d0 servo`rayon_core::sleep::Sleep::no_work_found::h819564110ff224b6(self=0x000000012b7f11a8, idle_state=0x000070000a0eb1e0, latch=0x000000012b7da5e0, has_injected_jobs={closure#2} @ 0x000070000a0eb118) at mod.rs:120:13
    frame #7: 0x000000010fbc0979 servo`rayon_core::registry::WorkerThread::wait_until_cold::hfc488ae5f310be02(self=0x000070000a0eb400, latch=0x000000012b7da5e0) at registry.rs:729:17
    frame #8: 0x000000010ef231f4 servo`rayon_core::registry::WorkerThread::wait_until::h64370dce30fbe405(self=0x000070000a0eb400, latch=0x000000012b7da5e0) at registry.rs:700:13
    frame #9: 0x000000010ef23c5f servo`rayon_core::registry::main_loop::h39c8f3e73fcc85cc(worker=<unavailable>, registry=Arc<rayon_core::registry::Registry> @ 0x000070000a0eb3f0, index=15) at registry.rs:833:5
    frame #10: 0x000000010ef20add servo`rayon_core::registry::ThreadBuilder::run::h6c6a224c8e3cf6b1(self=ThreadBuilder @ 0x000070000a0eb898) at registry.rs:55:18
    frame #11: 0x000000010ef20fa2 servo`_$LT$rayon_core..registry..DefaultSpawn$u20$as$u20$rayon_core..registry..ThreadSpawn$GT$::spawn::_$u7b$$u7b$closure$u7d$$u7d$::hde0bb9d23ad9c395 at registry.rs:100:20
    frame #12: 0x000000010ef0fd02 servo`std::sys_common::backtrace::__rust_begin_short_backtrace::hbb36aa13dd8728d2(f=<unavailable>) at backtrace.rs:123:18
    frame #13: 0x000000010eef14d2 servo`std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h9c7a21353f3f3045 at mod.rs:484:17
    frame #14: 0x000000010ef2b412 servo`_$LT$core..panic..unwind_safe..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h1f4a59d7e89eb18b(self=<unavailable>, _args=<unavailable>) at unwind_safe.rs:271:9
    frame #15: 0x000000010ef24c64 servo`std::panicking::try::do_call::h4f546feabe04fb08(data="") at panicking.rs:406:40
    frame #16: 0x000000010ef2744d servo`__rust_try + 29
    frame #17: 0x000000010ef247ef servo`std::panicking::try::h9ea1399623c68786(f=<unavailable>) at panicking.rs:370:19
    frame #18: 0x000000010eee75b2 servo`std::panic::catch_unwind::h0492f166f141dbf6(f=<unavailable>) at panic.rs:133:14
    frame #19: 0x000000010eef0c70 servo`std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::hd8adee284344f88c at mod.rs:483:30
    frame #20: 0x000000010eee7b11 servo`core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h8015d9eaf2d31d10((null)=0x000000012da00690, (null)=<unavailable>) at function.rs:227:5
    frame #21: 0x000000010fb1a997 servo`std::sys::unix::thread::Thread::new::thread_start::h0190e77bc7633265 [inlined] _$LT$alloc..boxed..Box$LT$F$C$A$GT$$u20$as$u20$core..ops..function..FnOnce$LT$Args$GT$$GT$::call_once::h579f3b223634893c at boxed.rs:1694:9 [opt]
    frame #22: 0x000000010fb1a991 servo`std::sys::unix::thread::Thread::new::thread_start::h0190e77bc7633265 [inlined] _$LT$alloc..boxed..Box$LT$F$C$A$GT$$u20$as$u20$core..ops..function..FnOnce$LT$Args$GT$$GT$::call_once::h805b8d91b6f45bdd at boxed.rs:1694 [opt]
    frame #23: 0x000000010fb1a98a servo`std::sys::unix::thread::Thread::new::thread_start::h0190e77bc7633265 at thread.rs:108 [opt]
    frame #24: 0x00007fff703d6109 libsystem_pthread.dylib`_pthread_start + 148
    frame #25: 0x00007fff703d1b8b libsystem_pthread.dylib`thread_start + 15

* thread #39
  * frame #0: 0x00007fff70315882 libsystem_kernel.dylib`__psynch_cvwait + 10
    frame #1: 0x00007fff703d6425 libsystem_pthread.dylib`_pthread_cond_wait + 698
    frame #2: 0x000000010ef04f46 servo`std::sys::unix::condvar::Condvar::wait::h8bfa6fc5f8adbe95(self=0x000000012b7e18a0, mutex=0x000000012b769bc0) at condvar.rs:82:17
    frame #3: 0x000000010ef24526 servo`std::sys_common::condvar::Condvar::wait::hb2c61e7a5f831fe0(self=0x000000012b7e7710, mutex=0x000000012b7e7700) at condvar.rs:44:9
    frame #4: 0x000000010ef051ce servo`std::sync::condvar::Condvar::wait::hbdf5e22aa730cf38(self=0x000000012b7e7710, guard=MutexGuard<bool> @ 0x0000700009ee7e70) at condvar.rs:189:13
    frame #5: 0x000000010fbc055a servo`rayon_core::sleep::Sleep::sleep::h2c459b8a83f28b92(self=0x000000012b7f11a8, idle_state=0x0000700009ee81e0, latch=0x000000012b7da580, has_injected_jobs={closure#2} @ 0x0000700009ee8028) at mod.rs:226:30
    frame #6: 0x000000010ef138d0 servo`rayon_core::sleep::Sleep::no_work_found::h819564110ff224b6(self=0x000000012b7f11a8, idle_state=0x0000700009ee81e0, latch=0x000000012b7da580, has_injected_jobs={closure#2} @ 0x0000700009ee8118) at mod.rs:120:13
    frame #7: 0x000000010fbc0979 servo`rayon_core::registry::WorkerThread::wait_until_cold::hfc488ae5f310be02(self=0x0000700009ee8400, latch=0x000000012b7da580) at registry.rs:729:17
    frame #8: 0x000000010ef231f4 servo`rayon_core::registry::WorkerThread::wait_until::h64370dce30fbe405(self=0x0000700009ee8400, latch=0x000000012b7da580) at registry.rs:700:13
    frame #9: 0x000000010ef23c5f servo`rayon_core::registry::main_loop::h39c8f3e73fcc85cc(worker=<unavailable>, registry=Arc<rayon_core::registry::Registry> @ 0x0000700009ee83f0, index=14) at registry.rs:833:5
    frame #10: 0x000000010ef20add servo`rayon_core::registry::ThreadBuilder::run::h6c6a224c8e3cf6b1(self=ThreadBuilder @ 0x0000700009ee8898) at registry.rs:55:18
    frame #11: 0x000000010ef20fa2 servo`_$LT$rayon_core..registry..DefaultSpawn$u20$as$u20$rayon_core..registry..ThreadSpawn$GT$::spawn::_$u7b$$u7b$closure$u7d$$u7d$::hde0bb9d23ad9c395 at registry.rs:100:20
    frame #12: 0x000000010ef0fd02 servo`std::sys_common::backtrace::__rust_begin_short_backtrace::hbb36aa13dd8728d2(f=<unavailable>) at backtrace.rs:123:18
    frame #13: 0x000000010eef14d2 servo`std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h9c7a21353f3f3045 at mod.rs:484:17
    frame #14: 0x000000010ef2b412 servo`_$LT$core..panic..unwind_safe..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h1f4a59d7e89eb18b(self=<unavailable>, _args=<unavailable>) at unwind_safe.rs:271:9
    frame #15: 0x000000010ef24c64 servo`std::panicking::try::do_call::h4f546feabe04fb08(data="") at panicking.rs:406:40
    frame #16: 0x000000010ef2744d servo`__rust_try + 29
    frame #17: 0x000000010ef247ef servo`std::panicking::try::h9ea1399623c68786(f=<unavailable>) at panicking.rs:370:19
    frame #18: 0x000000010eee75b2 servo`std::panic::catch_unwind::h0492f166f141dbf6(f=<unavailable>) at panic.rs:133:14
    frame #19: 0x000000010eef0c70 servo`std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::hd8adee284344f88c at mod.rs:483:30
    frame #20: 0x000000010eee7b11 servo`core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h8015d9eaf2d31d10((null)=0x000000012da00620, (null)=<unavailable>) at function.rs:227:5
    frame #21: 0x000000010fb1a997 servo`std::sys::unix::thread::Thread::new::thread_start::h0190e77bc7633265 [inlined] _$LT$alloc..boxed..Box$LT$F$C$A$GT$$u20$as$u20$core..ops..function..FnOnce$LT$Args$GT$$GT$::call_once::h579f3b223634893c at boxed.rs:1694:9 [opt]
    frame #22: 0x000000010fb1a991 servo`std::sys::unix::thread::Thread::new::thread_start::h0190e77bc7633265 [inlined] _$LT$alloc..boxed..Box$LT$F$C$A$GT$$u20$as$u20$core..ops..function..FnOnce$LT$Args$GT$$GT$::call_once::h805b8d91b6f45bdd at boxed.rs:1694 [opt]
    frame #23: 0x000000010fb1a98a servo`std::sys::unix::thread::Thread::new::thread_start::h0190e77bc7633265 at thread.rs:108 [opt]
    frame #24: 0x00007fff703d6109 libsystem_pthread.dylib`_pthread_start + 148
    frame #25: 0x00007fff703d1b8b libsystem_pthread.dylib`thread_start + 15
@jdm jdm added A-layout/uncategorized I-perf-slow Unnecessary performance degredation. labels Jan 1, 2022
@jdm
Copy link
Member Author

jdm commented Jan 1, 2022

Running with RAYON_LOG=tail:rayon.log doesn't work because the log is only dumped when the program quits, and the deadlock prevents the program from quitting. RAYON_LOG=all does show output, but I'm not sure how to interpret it.

@jdm
Copy link
Member Author

jdm commented Jan 1, 2022 •
edited

I'm trying to see if I can replicate the same deadlock in a test program, no success yet:

use rayon::ThreadPool;
use std::sync::mpsc::channel;
use std::sync::Arc;
use std::thread::sleep_ms;
use rand::thread_rng;
use rand::RngCore;

pub const STYLE_THREAD_STACK_SIZE_KB: usize = 256;

fn do_install(idx: u32, pool: &ThreadPool) {
    let mut i = 0;
    loop {
        println!("({}) {} installing", idx, i);
        pool.install(move || {
            let ms = thread_rng().next_u32() % 5000;
            println!("({}) Sleeping for {}ms", idx, ms);
            sleep_ms(ms);
            println!("({}) {} done", idx, i);
        });
        i += 1;
    }
}

fn main() {
    let pool = Arc::new(rayon::ThreadPoolBuilder::new()
        .num_threads(3)
        .stack_size(STYLE_THREAD_STACK_SIZE_KB * 1024)
        .build()
        .unwrap());
    let pool1 = pool.clone();
    std::thread::spawn(move || {
        do_install(0, &pool1);
    });
    do_install(1, &pool);
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
A-layout/uncategorized I-perf-slow Unnecessary performance degredation.
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
1 participant
@jdm