Skip to content

Retry recreate_index subtasks on worker exit #1615

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Oct 1, 2024
Merged

Retry recreate_index subtasks on worker exit #1615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d466a8e
retry_on_celery_worker_lost for all recreate_index subtasks
mbertrand Sep 27, 2024
a662e5c
Add SystemExit retry
mbertrand Sep 27, 2024
0a8ab87
experiment
mbertrand Sep 30, 2024
cc3d081
workign
mbertrand Sep 30, 2024
b8f5acf
working2
mbertrand Sep 30, 2024
ef10fd7
Working but only with cold shutdowns and short sleep inetrvals in eac…
mbertrand Sep 30, 2024
270b988
Get rid of custom redis command, still works
mbertrand Sep 30, 2024
fdb93cf
Working without cold shutdown
mbertrand Oct 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions docker-compose.apps.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,7 @@ services:
command: >
/bin/bash -c '
sleep 3;
celery -A main.celery:app worker -Q default -B -l ${MITOL_LOG_LEVEL:-INFO} &
celery -A main.celery:app worker -Q edx_content,default -l ${MITOL_LOG_LEVEL:-INFO}'
celery -A main.celery:app worker -E -Q default,edx_content -B -l ${MITOL_LOG_LEVEL:-INFO}'
depends_on:
db:
condition: service_healthy
Expand Down
32 changes: 25 additions & 7 deletions learning_resources_search/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,13 @@ def send_subscription_emails(self, subscription_type, period="daily"):
return self.replace(email_tasks)


@app.task(autoretry_for=(RetryError,), retry_backoff=True, rate_limit="600/m")
@app.task(
acks_late=True,
reject_on_worker_lost=True,
autoretry_for=(RetryError, SystemExit),
retry_backoff=True,
rate_limit="600/m",
)
def index_learning_resources(ids, resource_type, index_types):
"""
Index courses
Expand All @@ -290,7 +296,7 @@ def index_learning_resources(ids, resource_type, index_types):
try:
with wrap_retry_exception(*SEARCH_CONN_EXCEPTIONS):
api.index_learning_resources(ids, resource_type, index_types)
except (RetryError, Ignore):
except (RetryError, Ignore, SystemExit):
raise
except: # noqa: E722
error = "index_courses threw an error"
Expand Down Expand Up @@ -348,7 +354,13 @@ def bulk_deindex_percolators(ids):
return error


@app.task(autoretry_for=(RetryError,), retry_backoff=True, rate_limit="600/m")
@app.task(
acks_late=True,
reject_on_worker_lost=True,
autoretry_for=(RetryError, SystemExit),
retry_backoff=True,
rate_limit="600/m",
)
def bulk_index_percolate_queries(percolate_ids, index_types):
"""
Bulk index percolate queries for provided percolate query Ids
Expand All @@ -366,7 +378,7 @@ def bulk_index_percolate_queries(percolate_ids, index_types):
PERCOLATE_INDEX_TYPE,
index_types,
)
except (RetryError, Ignore):
except (RetryError, Ignore, SystemExit):
raise
except: # noqa: E722
error = "bulk_index_percolate_queries threw an error"
Expand Down Expand Up @@ -397,7 +409,13 @@ def index_course_content_files(course_ids, index_types):
return error


@app.task(autoretry_for=(RetryError,), retry_backoff=True, rate_limit="600/m")
@app.task(
acks_late=True,
reject_on_worker_lost=True,
autoretry_for=(RetryError, SystemExit),
retry_backoff=True,
rate_limit="600/m",
)
def index_content_files(
content_file_ids,
learning_resource_id,
Expand All @@ -418,7 +436,7 @@ def index_content_files(
api.index_content_files(
content_file_ids, learning_resource_id, index_types=index_types
)
except (RetryError, Ignore):
except (RetryError, Ignore, SystemExit):
raise
except: # noqa: E722
error = "index_content_files threw an error"
Expand Down Expand Up @@ -843,7 +861,7 @@ def get_update_learning_resource_tasks(resource_type):
@app.task(
acks_late=True,
reject_on_worker_lost=True,
autoretry_for=(RetryError,),
autoretry_for=(RetryError, SystemExit),
retry_backoff=True,
rate_limit="600/m",
)
Expand Down