OOM crash on iOS #511

elan · 2021-05-10T20:39:34Z

Bug Description

This started showing up on BugSnag after release, no obvious correlation with particular device/OS version. Looks like an out-of-memory issue, but free memory shows as considerable in all the reports and didn't have any issues with JSC.

EXC_BAD_ACCESShermes.cpp:146
Attempted to dereference null pointer.

0	Plexamp	
hermes.cpp:146:32
facebook::hermes::detail::hermesFatalErrorHandler(void*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, bool)
1	Plexamp	
ErrorHandling.cpp:108:5
llvh::report_fatal_error(llvh::Twine const&, bool)
2	Plexamp	
ErrorHandling.cpp:83:3
llvh::report_fatal_error(char const*, bool)
3	Plexamp	
ErrorHandling.cpp:61:3
hermes::hermes_fatal(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)
4	Plexamp	
GCBase.cpp:726:3
hermes::vm::GCBase::oom(std::__1::error_code)
5	Plexamp	
OldGenNC.cpp:630:8
hermes::vm::OldGen::fullCollectThenAlloc(unsigned int, hermes::vm::HasFinalizer)
6	Plexamp	
OldGenNC.h:488:10
hermes::vm::OldGen::alloc(unsigned int, hermes::vm::HasFinalizer)
7	GenGCNC.h	
void* hermes::vm::Runtime::allocLongLived<(hermes::vm::HasFinalizer)1>(unsigned int)
8	Plexamp	
HiddenClass.cpp:259:16
hermes::vm::HiddenClass::addProperty(hermes::vm::Handle<hermes::vm::HiddenClass>, hermes::vm::Runtime*, hermes::vm::SymbolID, hermes::vm::PropertyFlags)
9	Plexamp	
JSObject.cpp:2698:20
hermes::vm::JSObject::addOwnPropertyImpl(hermes::vm::Handle<hermes::vm::JSObject>, hermes::vm::Runtime*, hermes::vm::SymbolID, hermes::vm::PropertyFlags, hermes::vm::Handle<hermes::vm::HermesValue>)
10	Plexamp	
JSObject.cpp:2676:7
hermes::vm::JSObject::addOwnProperty(hermes::vm::Handle<hermes::vm::JSObject>, hermes::vm::Runtime*, hermes::vm::SymbolID, hermes::vm::DefinePropertyFlags, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::PropOpFlags)
11	Plexamp	
JSObject.cpp:2027:10
hermes::vm::JSObject::defineOwnProperty(hermes::vm::Handle<hermes::vm::JSObject>, hermes::vm::Runtime*, hermes::vm::SymbolID, hermes::vm::DefinePropertyFlags, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::PropOpFlags)
12	Plexamp	
RegExp.cpp:530:35
hermes::vm::directRegExpExec(hermes::vm::Handle<hermes::vm::JSRegExp>, hermes::vm::Runtime*, hermes::vm::Handle<hermes::vm::StringPrimitive>)
13	Plexamp	
RegExp.cpp:676:40
hermes::vm::regExpPrototypeExec(void*, hermes::vm::Runtime*, hermes::vm::NativeArgs)
14	Plexamp	
Callable.h:539:9
hermes::vm::NativeFunction::_nativeCall(hermes::vm::NativeFunction*, hermes::vm::Runtime*)
15	Plexamp	
Callable.h:273:12
hermes::vm::Callable::executeCall1(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::HermesValue, bool)
16	Plexamp	
RegExp.cpp:635:9
hermes::vm::regExpExec(hermes::vm::Runtime*, hermes::vm::Handle<hermes::vm::JSObject>, hermes::vm::Handle<hermes::vm::StringPrimitive>)
17	Plexamp	
RegExp.cpp:1431:20
hermes::vm::regExpPrototypeSymbolReplace(void*, hermes::vm::Runtime*, hermes::vm::NativeArgs)
18	Plexamp	
Callable.h:539:9
hermes::vm::NativeFunction::_nativeCall(hermes::vm::NativeFunction*, hermes::vm::Runtime*)
19	Plexamp	
Callable.h:273:12
hermes::vm::Callable::executeCall2(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::HermesValue, hermes::vm::HermesValue, bool)
20	Plexamp	
String.cpp:1910:14
hermes::vm::stringPrototypeReplace(void*, hermes::vm::Runtime*, hermes::vm::NativeArgs)
21	Plexamp	
Callable.h:539:9
hermes::vm::NativeFunction::_nativeCall(hermes::vm::NativeFunction*, hermes::vm::Runtime*)
22	Plexamp	
Interpreter.cpp:318:12
hermes::vm::Interpreter::handleCallSlowPath(hermes::vm::Runtime*, hermes::vm::PinnedHermesValue*)
23	Plexamp	
Interpreter.cpp:1690:7
hermes::vm::CallResult<hermes::vm::HermesValue, (hermes::vm::detail::CallResultSpecialize)2> hermes::vm::Interpreter::interpretFunction<false>(hermes::vm::Runtime*, hermes::vm::InterpreterState&)
24	Plexamp	
Interpreter.cpp:862:10
hermes::vm::Runtime::interpretFunctionImpl(hermes::vm::CodeBlock*)
25	Plexamp	
Callable.cpp:1295:23
hermes::vm::JSFunction::_callImpl(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*)
26	Plexamp	
Callable.h:273:12
hermes::vm::Callable::executeCall(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::Handle<hermes::vm::JSObject>)
27	Plexamp	
Function.cpp:212:10
hermes::vm::functionPrototypeApply(void*, hermes::vm::Runtime*, hermes::vm::NativeArgs)
28	Plexamp	
Callable.h:539:9
hermes::vm::NativeFunction::_nativeCall(hermes::vm::NativeFunction*, hermes::vm::Runtime*)
29	Plexamp	
Interpreter.cpp:318:12
hermes::vm::Interpreter::handleCallSlowPath(hermes::vm::Runtime*, hermes::vm::PinnedHermesValue*)
30	Plexamp	
Interpreter.cpp:1690:7
hermes::vm::CallResult<hermes::vm::HermesValue, (hermes::vm::detail::CallResultSpecialize)2> hermes::vm::Interpreter::interpretFunction<false>(hermes::vm::Runtime*, hermes::vm::InterpreterState&)
31	Plexamp	
Interpreter.cpp:862:10
hermes::vm::Runtime::interpretFunctionImpl(hermes::vm::CodeBlock*)
32	Plexamp	
Callable.cpp:1295:23
hermes::vm::JSFunction::_callImpl(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*)
33	Plexamp	
Callable.h:273:12
hermes::vm::Callable::executeCall(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::Handle<hermes::vm::JSObject>)
34	Plexamp	
Function.cpp:212:10
hermes::vm::functionPrototypeApply(void*, hermes::vm::Runtime*, hermes::vm::NativeArgs)
35	Plexamp	
Callable.h:539:9
hermes::vm::NativeFunction::_nativeCall(hermes::vm::NativeFunction*, hermes::vm::Runtime*)
36	Plexamp	
Interpreter.cpp:318:12
hermes::vm::Interpreter::handleCallSlowPath(hermes::vm::Runtime*, hermes::vm::PinnedHermesValue*)
37	Plexamp	
Interpreter.cpp:1690:7
hermes::vm::CallResult<hermes::vm::HermesValue, (hermes::vm::detail::CallResultSpecialize)2> hermes::vm::Interpreter::interpretFunction<false>(hermes::vm::Runtime*, hermes::vm::InterpreterState&)
38	Plexamp	
Interpreter.cpp:862:10
hermes::vm::Runtime::interpretFunctionImpl(hermes::vm::CodeBlock*)
39	Plexamp	
Callable.cpp:1295:23
hermes::vm::JSFunction::_callImpl(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*)
40	Plexamp	
Callable.h:273:12
hermes::vm::Callable::executeCall(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::Handle<hermes::vm::JSObject>)
41	Plexamp	
Function.cpp:212:10
hermes::vm::functionPrototypeApply(void*, hermes::vm::Runtime*, hermes::vm::NativeArgs)
42	Plexamp	
Callable.h:539:9
hermes::vm::NativeFunction::_nativeCall(hermes::vm::NativeFunction*, hermes::vm::Runtime*)
43	Plexamp	
Interpreter.cpp:318:12
hermes::vm::Interpreter::handleCallSlowPath(hermes::vm::Runtime*, hermes::vm::PinnedHermesValue*)
44	Plexamp	
Interpreter.cpp:1690:7
hermes::vm::CallResult<hermes::vm::HermesValue, (hermes::vm::detail::CallResultSpecialize)2> hermes::vm::Interpreter::interpretFunction<false>(hermes::vm::Runtime*, hermes::vm::InterpreterState&)
45	Plexamp	
Interpreter.cpp:862:10
hermes::vm::Runtime::interpretFunctionImpl(hermes::vm::CodeBlock*)
46	Plexamp	
Callable.cpp:1295:23
hermes::vm::JSFunction::_callImpl(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*)
47	Plexamp	
Callable.h:273:12
hermes::vm::Callable::executeCall(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::Handle<hermes::vm::HermesValue>, hermes::vm::Handle<hermes::vm::JSObject>)
48	Plexamp	
Function.cpp:212:10
hermes::vm::functionPrototypeApply(void*, hermes::vm::Runtime*, hermes::vm::NativeArgs)
49	Plexamp	
Callable.h:539:9
hermes::vm::NativeFunction::_nativeCall(hermes::vm::NativeFunction*, hermes::vm::Runtime*)
50	Plexamp	
Interpreter.cpp:318:12
hermes::vm::Interpreter::handleCallSlowPath(hermes::vm::Runtime*, hermes::vm::PinnedHermesValue*)
51	Plexamp	
Interpreter.cpp:1690:7
hermes::vm::CallResult<hermes::vm::HermesValue, (hermes::vm::detail::CallResultSpecialize)2> hermes::vm::Interpreter::interpretFunction<false>(hermes::vm::Runtime*, hermes::vm::InterpreterState&)
52	Plexamp	
Interpreter.cpp:862:10
hermes::vm::Runtime::interpretFunctionImpl(hermes::vm::CodeBlock*)
53	Plexamp	
Callable.cpp:1295:23
hermes::vm::JSFunction::_callImpl(hermes::vm::Handle<hermes::vm::Callable>, hermes::vm::Runtime*)
54	Plexamp	
Callable.h:273:12
hermes::vm::BoundFunction::_boundCall(hermes::vm::BoundFunction*, hermes::inst::Inst const*, hermes::vm::Runtime*)
55	Plexamp	
Callable.h:273:12
facebook::hermes::HermesRuntimeImpl::call(facebook::jsi::Function const&, facebook::jsi::Value const&, facebook::jsi::Value const*, unsigned long)
56	Plexamp	
decorator.h:294:19
facebook::jsi::WithRuntimeDecorator<facebook::react::(anonymous namespace)::ReentrancyCheck, facebook::jsi::Runtime, facebook::jsi::Runtime>::call(facebook::jsi::Function const&, facebook::jsi::Value const&, facebook::jsi::Value const*, unsigned long)
57	Plexamp	
jsi-inl.h:228:18
facebook::jsi::Value facebook::jsi::Function::call<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, facebook::jsi::Value>(facebook::jsi::Runtime&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, facebook::jsi::Value&&) const
58	Plexamp	
JSIExecutor.cpp:231:50
facebook::react::JSIExecutor::callFunction(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, folly::dynamic const&)::$_4::operator()() const
59	Plexamp	
type_traits:3545:1
_ZNSt3__1L8__invokeIRPFvRKNS_8functionIFvvEEENS1_IFNS_12basic_stringIcNS_11char_traitsIcEENS_9allocatorIcEEEEvEEEEJS5_SD_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSH_DpOSI_
60	Plexamp	
functional:1873:16
facebook::react::JSIExecutor::callFunction(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, folly::dynamic const&)
61	Plexamp	
functional:1873:16
std::__1::__function::__func<facebook::react::NativeToJsBridge::runOnExecutorQueue(std::__1::function<void (facebook::react::JSExecutor*)>)::$_8, std::__1::allocator<facebook::react::NativeToJsBridge::runOnExecutorQueue(std::__1::function<void (facebook::react::JSExecutor*)>)::$_8>, void ()>::operator()()
62	Plexamp	
functional:1873:16
facebook::react::tryAndReturnError(std::__1::function<void ()> const&)
63	Plexamp	
RCTMessageThread.mm:69:20
facebook::react::RCTMessageThread::tryFunc(std::__1::function<void ()> const&)
64	Plexamp	
functional:1873:16
___ZN8facebook5react16RCTMessageThread8runAsyncENSt3__18functionIFvvEEE_block_invoke
65	CoreFoundation	
___CFRUNLOOP_IS_CALLING_OUT_TO_A_BLOCK__
66	CoreFoundation	
___CFRunLoopDoBlocks
67	CoreFoundation	
___CFRunLoopRun
68	CoreFoundation	
_CFRunLoopRunSpecific
69	Plexamp	
RCTCxxBridge.mm:308:12
+[RCTCxxBridge runRunLoop]
70	Foundation	
___NSThread__start__
71	libsystem_pthread.dylib	
__pthread_start

I have run gradle clean and confirmed this bug does not occur with JSC

Hermes version: 0.7.2
React Native version (if any): 0.64.1
OS version (if any): iOS (various)
Platform (most likely one of arm64-v8a, armeabi-v7a, x86, x86_64): arm64-v8a

Steps To Reproduce

Unfortunately don't have any yet.

The Expected Behavior

The text was updated successfully, but these errors were encountered:

tmikov · 2021-05-10T21:21:40Z

Looks like the maximum heap size has been exceeded. It is a configuration value, set to 512MB by default:

hermes/public/hermes/Public/GCConfig.h

Line 154 in f3b1f0c

F(constexpr, gcheapsize_t, MaxHeapSize, 512 << 20) \

I am not sure whether ReactNative overrides it on initialization or whether it provides a mechanism to change it.

elan · 2021-05-10T21:29:35Z

One curious bit is that at least for one user, this crash happens very shortly after launch, and I'd be amazed if the app was able to use that much memory so quickly. I can't find it now, but I thought I saw some reference around Hermes having another limit around number of properties on an object or something like that that (limiting them to a single page of memory?) but I might be misremembering.

tmikov · 2021-05-10T22:20:05Z

Hermes does have a restriction in number of properties in an object (around 200,000), but in this case the stack strongly suggests that the garbage collector thinks this is a real OOM (the null pointer dereference is deliberate, in order to cause a crash):

4	Plexamp	
GCBase.cpp:726:3
hermes::vm::GCBase::oom(std::__1::error_code)
5	Plexamp	
OldGenNC.cpp:630:8
hermes::vm::OldGen::fullCollectThenAlloc(unsigned int, hermes::vm::HasFinalizer)
6	Plexamp	
OldGenNC.h:488:10
hermes::vm::OldGen::alloc(unsigned int, hermes::vm::HasFinalizer)

Ashoat · 2021-07-20T08:01:42Z

On iOS the limit appears to be 512 MiIB, as MaxHeapSize is not overriden. On Android it appears to be overriden to 1024 MiB.

If you're directly constructing your JSExecutorFactory via jsExecutorFactoryForBridge, you should be able to pass in a hermes::vm::RuntimeConfig as the third parameter.

It's also not super clear to me why a GCConfig is generated for Android but not for iOS.

I've been able to directly copy the makeRuntimeConfig from stock React Native Android into our React Native iOS config with (so far) positive results.

…Android Summary: I was noticing OOM crashes because the app uses just about 500 MiB. I investigated some more and realized that the Hermes default is 512 MiB, which Android overrides to 1024 MiB, and iOS does not override. I further investigated the Android Hermes config and figured it wouldn't hurt to copy it over to iOS. More context in [this GitHub comment](facebook/hermes#511 (comment)). Test Plan: Make sure iOS stops crashing when logged in with my user Reviewers: karol-bisztyga, palys-swm Reviewed By: palys-swm Subscribers: KatPo, Adrian, atul Differential Revision: https://phabricator.ashoat.com/D1764

Huxpro · 2021-07-20T22:35:08Z

@Ashoat yup I agreed that we should increase the MaxHeapSize to at least 1GiB for whom using the default set up. Maybe even 3GiB which is "unlimited" in practice.

It's also not super clear to me why a GCConfig is generated for Android but not for iOS.

We only support Android back to that time. It was increased to mediate a similar issue #295.

Ashoat · 2021-07-25T09:08:21Z

As a starting point to pulling this stuff out to be shared with iOS, I put up this PR: Remove custom Hermes config for Android react-native#31900
It moves the Hermes GC config from ReactAndroid to a new file in ReactCommon that can be accessed by iOS too
Once this is landed I'll put up a PR on React/CxxBridge/RCTCxxBridge.mm to call into that when no jsExecutorFactoryForBridge is found (I'd put it up now if GitHub has better support for stacked PRs...)
After that I can put up a diff to move us from 1 GiB to 3 GiB. We can discuss there but I think it's a good idea given JSC isn't limited this way

Huxpro · 2021-07-28T00:54:08Z

@Ashoat Thanks for pulling off this.

I'm wondering would it make more sense to just increase the default number within Hermes and remove the React-Android specific GCConfig in React Native instead? As more platforms (Windows, macOS) adopting Hermes, it seems suboptimal to glue this default GCConfig again and again.

vladamx · 2021-08-06T21:31:31Z

For context, are there any additional changes needed to this PR for heap size increase to be actually applied on IOS?

Ashoat · 2021-08-06T22:35:12Z

The goal of my work here is to eventually update the default heap size on iOS, but the above PR is only the first part. If you want to customize the heap size of a Hermes iOS app you should implement a custom jsExecutorFactoryForBridge in your AppDelegate, eg. like this.

jaydgoss · 2021-08-06T23:00:20Z

After implementing a custom jsExecutorFactoryForBridge, do you know of a way to verify or log that the max heap size has been set as expected for the runtime in use?

Huxpro · 2021-08-07T00:27:20Z

I'll update the default heap size but that will only be available until the next Hermes (and React Native) release (presumably 0.9 unless people felt like they really need a patch release). In the meantime, feel free to override the maxHeapSize GCConfig as @Ashoat suggested.

@jaydgoss After implementing a custom jsExecutorFactoryForBridge, do you know of a way to verify or log that the max heap size has been set as expected for the runtime in use?

Unfortunately there isn't an easy way that I'm aware of. The closest one that we exposed to JavaScript is HermesInternal.getInstrumentedStats().js_heapSize which will give you the current heap size in byte.

Huxpro · 2021-09-03T01:00:52Z

Closing since this iOS OOM issue should be solved by upgrading to Hermes 0.9 (shipped with the upcoming RN 0.66) which includes the fix 5f2b47d that increase the default max heap size from 512MiB to 3GiB (which is "unlimited" in practice and make us more or less on par with the JSC unlimited behavior). (I made a typo in the commit summary so it's not linked to this issue 😅 )

Big shout out to @Ashoat to all the investigation and effort he made that helped us resolving this and similar memory issue going forward!

elan added the bug Something isn't working label May 10, 2021

Huxpro mentioned this issue Jul 28, 2021

Remove custom Hermes config for Android facebook/react-native#31900

Closed

Huxpro closed this as completed Sep 3, 2021

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

OOM crash on iOS #511

OOM crash on iOS #511

elan commented May 10, 2021

tmikov commented May 10, 2021

elan commented May 10, 2021

tmikov commented May 10, 2021

Ashoat commented Jul 20, 2021

Huxpro commented Jul 20, 2021 •

edited

Ashoat commented Jul 25, 2021 •

edited

Huxpro commented Jul 28, 2021 •

edited

vladamx commented Aug 6, 2021

Ashoat commented Aug 6, 2021

jaydgoss commented Aug 6, 2021

Huxpro commented Aug 7, 2021 •

edited

Huxpro commented Sep 3, 2021 •

edited

OOM crash on iOS #511

OOM crash on iOS #511

Comments

elan commented May 10, 2021

Bug Description

Steps To Reproduce

The Expected Behavior

tmikov commented May 10, 2021

elan commented May 10, 2021

tmikov commented May 10, 2021

Ashoat commented Jul 20, 2021

Huxpro commented Jul 20, 2021 • edited

Ashoat commented Jul 25, 2021 • edited

Huxpro commented Jul 28, 2021 • edited

vladamx commented Aug 6, 2021

Ashoat commented Aug 6, 2021

jaydgoss commented Aug 6, 2021

Huxpro commented Aug 7, 2021 • edited

Huxpro commented Sep 3, 2021 • edited

Huxpro commented Jul 20, 2021 •

edited

Ashoat commented Jul 25, 2021 •

edited

Huxpro commented Jul 28, 2021 •

edited

Huxpro commented Aug 7, 2021 •

edited

Huxpro commented Sep 3, 2021 •

edited