Skip to content
Browse files

changed to call delete_project_iteratively() to avoid running out of …

…memory when deleting large projects on Heroku
  • Loading branch information...
matthewcornell committed Nov 5, 2019
1 parent e0d651e commit a66c5636130ae59b6ac46b73139459ef16bd2673
Showing with 52 additions and 5 deletions.
  1. +52 −5 utils/
@@ -3,6 +3,7 @@

import click
import django
from django.db import transaction

logger = logging.getLogger(__name__)
@@ -22,14 +23,60 @@
def delete_project_app(name):
project = Project.objects.filter(name=name).first() # None if doesn't exist
if not project:
logger.error("Project not found: '{}'".format(name))
logger.error(f"Project not found: '{name}'")
return"Deleting project: {}".format(project))
logger.debug("memory before: {}".format(sizeof_fmt(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)))
print(f"delete_project_app(): project={project}")
# delete_project_single_call(project)
print(f"\ndelete_project_app(): done!")

def delete_project_iteratively(project):
Deletes the passed Project, but unlike `delete_project_single_call()`, does so by iterating over objects that refer
to the project before deleting the project itself. This apparently reduces the memory usage enough to allow the
below Heroku deletion.
print(f"\n* models and forecasts")
for forecast_model in project.models.iterator():
print(f"- {}")
for forecast in forecast_model.forecasts.iterator():
print(f" = {}")

print(f"\n* locations")
for location in project.locations.iterator():
print(f"- {}")

print(f"\n* targets")
for target in project.targets.iterator():
print(f"- {}")

print(f"\n* timezeros")
for timezero in project.timezeros.iterator():
print(f"- {}")

project.delete() # deletes remaining references: RowCountCache, ScoreCsvFileCache

def delete_project_single_call(project):
Deletes the passed Project via a single `delete()` call. This is fine for small projects, but it soon uses a lot of
memory as project size (# models and # forecasts) increases. On Heroku, deleting the 'CDC Flu challenge' project
caused "Error R14 (Memory quota exceeded)" and "Error R15 (Memory quota vastly exceeded)" errors, leading to
"Stopping process with SIGKILL".
""""Deleting project: {project}")
logger.debug(f"memory before: {sizeof_fmt(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)}")
project.delete()"Delete: done")
logger.debug("memory after: {}".format(sizeof_fmt(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)))"Delete: done")
logger.debug(f"memory after: {sizeof_fmt(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)}")


0 comments on commit a66c563

Please sign in to comment.
You can’t perform that action at this time.