Lots of OutOfMemory #1852

markuswahl-2 · 2022-05-25T07:22:32Z

Platform

iOS

Installed

CocoaPods

Version

7.11.0

Steps to Reproduce

We do not know. These are the only breadcrumbs from sentry.io:

CATEGORY	DESCRIPTION	LEVEL	TIME
sentry.event	A value set to the context or sdk is not serializable. Dropping context and sdk.	Error	21:07:28
exception	OutOfMemory: The OS most likely terminated your app because it overused RAM.	Error	21:07:28

We are seeing this with sentry.cocoa 7.9.0, 7.10.2 and 7.11.0. One difference we see when we upgraded from 7.9.0 to 7.11.0 is that we no longer get the information about APP Start Time. With 7.9.0 we see that all such OOM events occur in the same second that the app was started. After having upgraded to 7.10.2 or 7.11.0 we no longer get that piece of information.

Expected Result

No crash

Actual Result

with-context-information-prior-to-7.11.0-redacted.json

{
  "event_id": "REDACTED",
  "project": REDACTED,
  "release": "REDACTED",
  "dist": "580",
  "platform": "cocoa",
  "message": "",
  "datetime": "2022-05-23T08:12:56.779649Z",
  "tags": [
    [
      "app.device",
      "REDACTED"
    ],
    [
      "device",
      "iPhone12,8"
    ],
    [
      "device.family",
      "iOS"
    ],
    [
      "environment",
      "production"
    ],
    [
      "event.environment",
      "native"
    ],
    [
      "event.origin",
      "ios"
    ],
    [
      "handled",
      "no"
    ],
    [
      "level",
      "fatal"
    ],
    [
      "mechanism",
      "out_of_memory"
    ],
    [
      "os",
      "iOS 15.4.1"
    ],
    [
      "os.name",
      "iOS"
    ],
    [
      "os.rooted",
      "no"
    ],
    [
      "dist",
      "580"
    ],
    [
      "release",
      "REDACTED"
    ],
    [
      "user",
      "id:REDACTED"
    ]
  ],
  "_metrics": {
    "bytes.ingested.event": 1410,
    "bytes.stored.event": 2623
  },
  "contexts": {
    "app": {
      "app_start_time": "2022-05-23T08:12:56Z",
      "device_app_hash": "REDACTED",
      "build_type": "app store",
      "app_identifier": "com.org.org",
      "app_name": "Organization Name App 2.0",
      "app_version": "1.0.10",
      "app_build": "580",
      "app_id": "REDACTED",
      "type": "app"
    },
    "device": {
      "family": "iOS",
      "model": "iPhone12,8",
      "model_id": "D79AP",
      "memory_size": 3100426240,
      "usable_memory": 1877671936,
      "storage_size": 63966400512,
      "boot_time": "2022-05-07T21:36:47Z",
      "timezone": "CEST",
      "locale": "en_SE",
      "type": "device"
    },
    "os": {
      "name": "iOS",
      "version": "15.4.1",
      "build": "19E258",
      "kernel_version": "Darwin Kernel Version 21.4.0: Mon Feb 21 21:27:53 PST 2022; root:xnu-8020.102.3~1/RELEASE_ARM64_T8030",
      "rooted": false,
      "type": "os"
    }
  },
  "culprit": "",
  "environment": "production",
  "exception": {
    "values": [
      {
        "type": "OutOfMemory",
        "value": "The OS most likely terminated your app because it overused RAM.",
        "mechanism": {
          "type": "out_of_memory",
          "handled": false
        }
      }
    ]
  },
  "fingerprint": [
    "{{ default }}"
  ],
  "grouping_config": {
    "enhancements": "eJybzDRxY3J-bm5-npWRgaGlroGxrpHxBABcYgcZ",
    "id": "newstyle:2019-05-08"
  },
  "hashes": [
    "REDACTED"
  ],
  "key_id": "REDACTED",
  "level": "fatal",
  "logger": "",
  "metadata": {
    "display_title_with_tree_label": true,
    "type": "OutOfMemory",
    "value": "The OS most likely terminated your app because it overused RAM."
  },
  "nodestore_insert": 1653293579.902627,
  "received": 1653293577.576998,
  "sdk": {
    "name": "sentry.cocoa",
    "version": "7.9.0",
    "integrations": [
      "isolateErrorIntegration",
      "runZonedGuardedIntegration",
      "widgetsFlutterBindingIntegration",
      "flutterErrorIntegration",
      "widgetsBindingIntegration"
    ]
  },
  "timestamp": 1653293576.779649,
  "title": "OutOfMemory: The OS most likely terminated your app because it overused RAM.",
  "type": "error",
  "user": {
    "id": "REDACTED",
    "ip_address": "REDACTED",
    "geo": {
      "country_code": "SE",
      "region": "Sweden"
    }
  },
  "version": "7",
  "location": null
}

markuswahl-2 · 2022-05-30T10:31:53Z

without-context-information-using-7.11.0-redacted.json

{
  "event_id": "REDACTED",
  "project": REDACTED,
  "release": "REDACTED",
  "dist": "805",
  "platform": "cocoa",
  "message": "",
  "datetime": "2022-05-30T08:51:06.743513Z",
  "tags": [
    [
      "environment",
      "production"
    ],
    [
      "event.environment",
      "native"
    ],
    [
      "event.origin",
      "ios"
    ],
    [
      "handled",
      "no"
    ],
    [
      "level",
      "fatal"
    ],
    [
      "mechanism",
      "out_of_memory"
    ],
    [
      "dist",
      "805"
    ],
    [
      "release",
      "REDACTED"
    ],
    [
      "user",
      "id:REDACTED"
    ]
  ],
  "_metrics": {
    "bytes.ingested.event": 701,
    "bytes.stored.event": 1796
  },
  "breadcrumbs": {
    "values": [
      {
        "timestamp": 1653900666.744,
        "type": "error",
        "category": "sentry.event",
        "level": "error",
        "message": "A value set to the context or sdk is not serializable. Dropping context and sdk."
      }
    ]
  },
  "culprit": "",
  "environment": "production",
  "exception": {
    "values": [
      {
        "type": "OutOfMemory",
        "value": "The OS most likely terminated your app because it overused RAM.",
        "mechanism": {
          "type": "out_of_memory",
          "handled": false
        }
      }
    ]
  },
  "fingerprint": [
    "{{ default }}"
  ],
  "grouping_config": {
    "enhancements": "REDACTED",
    "id": "newstyle:2019-05-08"
  },
  "hashes": [
    "REDACTED"
  ],
  "key_id": "REDACTED",
  "level": "fatal",
  "logger": "",
  "metadata": {
    "display_title_with_tree_label": true,
    "type": "OutOfMemory",
    "value": "The OS most likely terminated your app because it overused RAM."
  },
  "nodestore_insert": 1653900670.293765,
  "received": 1653900667.589878,
  "sdk": {
    "name": "sentry.cocoa",
    "version": "7.11.0"
  },
  "timestamp": 1653900666.743513,
  "title": "OutOfMemory: The OS most likely terminated your app because it overused RAM.",
  "type": "error",
  "user": {
    "id": "REDACTED",
    "ip_address": "REDACTED",
    "geo": {
      "country_code": "SE",
      "city": "Gothenburg",
      "region": "Sweden"
    }
  },
  "version": "7",
  "location": null
}

philipphofmann · 2022-05-31T07:54:57Z

FYI, the Cocoa SDK creates the OOM errors on heuristics. There is no 100% guarantee that your app really crashed. Especially on hybrid platforms such as Flutter and RN, we had issues in the past that we created OOMs by mistake. You are not the first person to be confused by these OOM errors.

Can you provide a repro for an OOM event maybe?

I don't think that the Cocoa SDK causes the missing APP data. I think that one of the recent changes to the dart SDK causes this:

Is the APP data only missing for OOM errors or other events as well?

markuswahl-2 · 2022-06-02T13:20:13Z

I see. Thanks for your feedback.

We can see reports in sentry for other events where the APP data section is present.

In regards to the APP data section and the field "Start Time": After reading the heuristics section you mentioned, I understand that the OOM is reported the next time the app starts. If so, what does the "Start Time" field in the APP section indicate? Is it the start time of the app for the session that had an OOM or is it the start time of the app that reports the OOM (i e the next time the app starts)? If it is the latter, I understand that we can not infer how long the app had been live when (if) it experienced OOM, from the start time field in the APP section. Right?

With repro, you mean if we can reproduce it, right? I am afraid not.

philipphofmann · 2022-06-10T08:59:14Z

This could be related to #1877

philipphofmann · 2022-06-22T13:11:42Z

Hey @markuswahl-2, can you try to update to 7.18.0, because #1877 might have fixed the issue.

markuswahl-2 · 2022-06-27T08:07:28Z

Will do, thanks!

markuswahl-2 · 2022-06-29T06:32:04Z

I am afraid we still get those errors even after bumping to 7.18.1 of sentry.cocoa. Also we see a sudden other error (but in android of NoClassDefFoundError: io.sentry.flutter.f) after having bumped to 7.18.1.

philipphofmann · 2022-06-29T07:33:07Z

I'm sorry to hear that, @markuswahl-2. Does your app do any background work? Could it be that it runs in the background and then goes into the foreground when the user actively opens it?

markuswahl-2 · 2022-06-29T07:34:21Z

I think that is possible. (That's normal for most apps, no?)

philipphofmann · 2022-06-29T07:57:38Z

Yes, that's normal for sure, but I'm just asking because it could be that we don't detect this properly.

Do you maybe have enableAutoSessionTracking disabled, @markuswahl-2? If so, getsentry/sentry-dart#905 would fix the issue.

markuswahl-2 · 2022-06-29T11:51:36Z

OK, thanks.

It is this one, right?
https://pub.dev/documentation/sentry_flutter/latest/sentry_flutter/SentryFlutterOptions/enableAutoSessionTracking.html

It is no entirely clear to me from the docs what enableAutoSessionTracking does.

Do I understand it correctly that setting enableAutoSessionTracking to false (while keeping the default value of enableOutOfMemoryTracking of true), and waiting for getsentry/sentry-dart#905 to become available, I might expect improvements?

It goes without saying that we are very much still interested in getting reports about true OOM, it is just the possibly false OOM reports that we want to skip. 😃

markuswahl-2 · 2022-06-29T11:58:19Z

(And to answer your question, no we did not have enableAutoSessionTracking disabled, but we are considering to do it, especially once getsentry/sentry-dart#905 becomes available).

philipphofmann · 2022-06-29T14:26:31Z

Do I understand it correctly that setting enableAutoSessionTracking to false (while keeping the default value of enableOutOfMemoryTracking of true), and waiting for getsentry/sentry-dart#905 to become available, I might expect improvements?

No, if you set enableAutoSessionTracking to false, it might get worse. That's why I asked. The default is true. If you didn't touch the option getsentry/sentry-dart#905 should not have an impact on you.

(And to answer your question, no we did not have enableAutoSessionTracking disabled, but we are considering to do it, especially once getsentry/sentry-dart#905 becomes available).

Why do you plan on disabling enableAutoSessionTracking? If you had it disabled in the past it could lead to wrongly reported OOMs. getsentry/sentry-dart#905 fixes this problem.

philipphofmann · 2022-06-29T14:33:55Z

@markuswahl-2, I'm out of ideas. If you want, can you maybe send me the link to your project to philipp.hofmann@sentry.io so that I can look at your data? Maybe I spot something.

markuswahl-2 · 2022-06-30T11:18:06Z

No, I meant that we were eager to try setting enableAutoSessionTracking to false because I, incorrectly, understood from you that that might correct it. The setting enableAutoSessionTracking is and has always been unmodified in our source code. And now that I understand you, we will of course not disable that setting.

markuswahl-2 · 2022-07-01T10:59:36Z

Here are the five files in our app that touches upon Sentry. (I added suffix .txt because github did not allow them otherwise):
Podfile.lock.txt
leading_icon.dart.txt
main.dart.txt
pubspec.yaml.txt
pubspec.lock.txt

philipphofmann · 2022-07-05T15:26:47Z

Thanks a lot for your update, @markuswahl-2. I didn't have time yet to look at your setup. I will get back to you later.

github-actions · 2022-07-27T00:08:33Z

This issue has gone three weeks without activity. In another week, I will close it.

But! If you comment or otherwise update it, I will reset the clock, and if you label it Status: Backlog or Status: In Progress, I will leave it alone ... forever!

"A weed is but an unloved flower." ― Ella Wheeler Wilcox 🥀

markuswahl-2 · 2022-08-15T06:45:43Z

Hi Philipp! I was away from keyboard during the summer. I am back now. Have you had some insight into this?

Best regards

brustolin · 2022-08-16T08:04:59Z

Hello @markuswahl-2 Philipp is in PTO. Sorry but we need to wait for him to came back to look into this. We don't have enough bandwidth right now.

markuswahl-2 · 2022-09-06T05:42:52Z

Hello @philipphofmann
Have you had time to look at the files?
Best regards,
Markus

philipphofmann · 2022-09-07T13:47:07Z

Hey, @markuswahl-2,

sorry, no didn't get to that yet. Try to do it this week.

markuswahl-2 · 2022-09-28T05:14:37Z

Hi @kevinrenskers
Have you have had a chance to look into this?

best regards,
Markus

kevinrenskers · 2022-10-03T14:34:03Z

I am looking into this now.

philipphofmann · 2022-10-11T13:33:10Z

Hey, @markuswahl-2,

Could it be that some parts of your app use exit(code)? We have an issue on React-Native for creating wrong OOMs when some code calls exit(code). getsentry/sentry-docs#4350.

We don't have any other ideas what could cause your OOMs at the moment. Do you by any chance have a way to reproduce it now?

marandaneto · 2022-10-13T08:56:50Z

@philipphofmann I can also see that if I just stop the debugger and open it again, but this is likely an exit(0) example, will try a release mode.

marandaneto · 2022-10-13T11:42:25Z

@markuswahl-2 I could not reproduce it using a release build, can you provide a minimal reproducible example and the steps to reproduce it?

markuswahl-2 · 2022-10-18T11:45:10Z

Hi! I am afraid we do not know how to reproduce it. We see it amongst our users, but we have not experienced it ourselves. No user has complained. This leads us to believe that we should ourselves have been exposed to the problem but that it has not manifested itself to us in a visible way. Or that it is a case of a false negative, i e that it is the heuristic-detection of OOM that falsely reports this as an OOM.

And no, we do not order the exit of the app.

We don't have a way to reproduce it. But our users reproduce it all the time. We do not see a pattern though...

markuswahl-2 · 2022-10-18T11:48:15Z

I understand that OOM is reported the next time the app starts. Is there a way for us to get some insights from sentry about the previous session, ie the session of the app that the OOM is reporting about? Maybe that way we could start to look for some patterns among the sessions that would have experienced the actual OOM.

philipphofmann · 2022-10-18T16:19:41Z

Thanks for the update again, @markuswahl-2.

Is there a way for us to get some insights from sentry about the previous session, ie the session of the app that the OOM is reporting about?

No, not yet, but we have an issue for that #1759. It has a high priority, but I can't give you an ETA for it. This feature would definitely help to get a better understanding of what's going on for you.

We might also tackle MetricKit (#1661) in the following months, giving you OOMs with a stacktrace.

github-actions · 2022-11-09T00:09:32Z

This issue has gone three weeks without activity. In another week, I will close it.

But! If you comment or otherwise update it, I will reset the clock, and if you label it Status: Backlog or Status: In Progress, I will leave it alone ... forever!

"A weed is but an unloved flower." ― Ella Wheeler Wilcox 🥀

kevinrenskers · 2022-11-14T10:26:24Z

@markuswahl-2 With #2347 merged, please try out the next release of the SDK to see if the added breadcrumbs for OOM events help to make sense of them.

kahest · 2022-11-29T13:40:58Z

@markuswahl-2 did you get round to test this with a current release?

github-actions · 2022-12-29T00:07:11Z

This issue has gone three weeks without activity. In another week, I will close it.

But! If you comment or otherwise update it, I will reset the clock, and if you label it Status: Backlog or Status: In Progress, I will leave it alone ... forever!

"A weed is but an unloved flower." ― Ella Wheeler Wilcox 🥀

markuswahl-2 added Platform: Cocoa Type: Bug labels May 25, 2022

markuswahl-2 mentioned this issue May 25, 2022

iOS Native OOM getsentry/sentry-dart#814

Closed

11 tasks

brustolin assigned brustolin and philipphofmann and unassigned brustolin Jun 8, 2022

This was referenced Jun 10, 2022

False OOM when calling SentrySDK.start twice #1877

Closed

Fix reporting wrong OOM when starting SDK twice #1878

Merged

This was referenced Jun 29, 2022

Send DidBecomeActiveNotification when OOM enabled getsentry/sentry-dart#905

Merged

Send DidBecomeActiveNotification when OOM enabled getsentry/sentry-react-native#2326

Merged

buenaflor mentioned this issue Jul 19, 2022

iOS throws an OutOfMemory after initializing the application getsentry/sentry-kotlin-multiplatform#15

Closed

github-actions bot added the Status: Stale label Jul 27, 2022

brustolin removed the Status: Stale label Jul 27, 2022

philipphofmann assigned kevinrenskers Sep 15, 2022

github-actions bot added the Status: Stale label Nov 9, 2022

philipphofmann mentioned this issue Nov 11, 2022

feat: Store breadcrumbs to disk for OOM events #2347

Merged

5 tasks

brustolin added Status: Backlog and removed Status: Stale labels Nov 14, 2022

kevinrenskers removed their assignment Nov 22, 2022

github-actions bot added the Status: Stale label Dec 29, 2022

github-actions bot closed this as not planned Won't fix, can't repro, duplicate, stale Jan 5, 2023

Lots of OutOfMemory #1852

Lots of OutOfMemory #1852

Comments

markuswahl-2 commented May 25, 2022

Platform

Installed

Version

Steps to Reproduce

Expected Result

Actual Result

Tags

philipphofmann commented May 27, 2022 • edited Loading

markuswahl-2 commented May 30, 2022

markuswahl-2 commented May 30, 2022 • edited Loading

with-context-information-prior-to-7.11.0-redacted.json

markuswahl-2 commented May 30, 2022

without-context-information-using-7.11.0-redacted.json

philipphofmann commented May 31, 2022

markuswahl-2 commented Jun 2, 2022 • edited Loading

philipphofmann commented Jun 10, 2022

philipphofmann commented Jun 22, 2022

markuswahl-2 commented Jun 27, 2022

markuswahl-2 commented Jun 29, 2022

philipphofmann commented Jun 29, 2022

markuswahl-2 commented Jun 29, 2022

philipphofmann commented Jun 29, 2022 • edited Loading

markuswahl-2 commented Jun 29, 2022

markuswahl-2 commented Jun 29, 2022

philipphofmann commented Jun 29, 2022 • edited Loading

philipphofmann commented Jun 29, 2022

markuswahl-2 commented Jun 30, 2022

markuswahl-2 commented Jul 1, 2022

philipphofmann commented Jul 5, 2022

github-actions bot commented Jul 27, 2022

markuswahl-2 commented Aug 15, 2022

brustolin commented Aug 16, 2022

markuswahl-2 commented Sep 6, 2022

philipphofmann commented Sep 7, 2022

markuswahl-2 commented Sep 28, 2022

kevinrenskers commented Oct 3, 2022

philipphofmann commented Oct 11, 2022

marandaneto commented Oct 13, 2022

marandaneto commented Oct 13, 2022

markuswahl-2 commented Oct 18, 2022 • edited Loading

markuswahl-2 commented Oct 18, 2022

philipphofmann commented Oct 18, 2022

github-actions bot commented Nov 9, 2022

kevinrenskers commented Nov 14, 2022

kahest commented Nov 29, 2022

github-actions bot commented Dec 29, 2022

philipphofmann commented May 27, 2022 •

edited

Loading

markuswahl-2 commented May 30, 2022 •

edited

Loading

markuswahl-2 commented Jun 2, 2022 •

edited

Loading

philipphofmann commented Jun 29, 2022 •

edited

Loading

philipphofmann commented Jun 29, 2022 •

edited

Loading

markuswahl-2 commented Oct 18, 2022 •

edited

Loading