Skip to content

perf: bulk INSERT secret events to reduce DB round-trips #784

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rohan-chaturvedi merged 8 commits into from
May 11, 2026
+188 −103
Merged

perf: bulk INSERT secret events to reduce DB round-trips #784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3579d33
perf: bulk INSERT secret events to reduce DB round-trips
nimish-ks Feb 24, 2026
af50e9d
Merge branch 'main' into perf/bulk-secret-event-logging
rohan-chaturvedi Mar 9, 2026
daf8cad
Merge branch 'main' into perf/bulk-secret-event-logging
nimish-ks Apr 30, 2026
ff1f7a1
Merge branch 'main' into perf/bulk-secret-event-logging
nimish-ks May 1, 2026
7e4b37e
Merge branch 'main' into perf/bulk-secret-event-logging
nimish-ks May 11, 2026
f18ba8d
fix: propagate type, prefetch tags, and atomic-wrap bulk audit events
rohan-chaturvedi May 11, 2026
18830c6
fix: guard BulkCreateSecret org lookup against empty input
rohan-chaturvedi May 11, 2026
ef7e936
perf: cap bulk audit insert batch size to 1000
rohan-chaturvedi May 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions backend/api/utils/audit_logging.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
from django.utils import timezone
from django.db import transaction
from django.db.models import prefetch_related_objects

from api.models import SecretEvent

Expand Down Expand Up @@ -42,3 +44,65 @@ def log_secret_event(
)

event.tags.set(secret.tags.all())


def log_secret_events_bulk(
secrets,
event_type,
user=None,
service_token=None,
service_account_token=None,
ip_address=None,
user_agent=None,
):
"""
Bulk version of log_secret_event. Logs events for multiple secrets
using bulk_create to reduce database round-trips.
"""

if not secrets:
return []

service_account = None
if service_account_token is not None:
service_account = service_account_token.service_account

# Load tags for all secrets in one query, regardless of caller's prefetch state.
prefetch_related_objects(secrets, "tags")

events = [
SecretEvent(
secret=secret,
environment=secret.environment,
folder=secret.folder,
path=secret.path,
user=user,
service_token=service_token,
service_account=service_account,
service_account_token=service_account_token,
key=secret.key,
key_digest=secret.key_digest,
value=secret.value,
version=secret.version,
comment=secret.comment,
type=secret.type,
event_type=event_type,
ip_address=ip_address,
user_agent=user_agent,
)
for secret in secrets
]

with transaction.atomic():
created = SecretEvent.objects.bulk_create(events, batch_size=1000)

through_model = SecretEvent.tags.through
m2m_rows = [
through_model(secretevent_id=event.pk, secrettag_id=tag.pk)
for event, secret in zip(created, secrets)
for tag in secret.tags.all()
]
if m2m_rows:
through_model.objects.bulk_create(m2m_rows, batch_size=1000)

return created
182 changes: 97 additions & 85 deletions backend/api/views/secrets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from api.utils.access.permissions import (
user_has_permission,
)
from api.utils.audit_logging import log_secret_event
from api.utils.audit_logging import log_secret_event, log_secret_events_bulk

from api.utils.crypto import encrypt_asymmetric, validate_encrypted_string
from api.utils.rest import (
Expand Down Expand Up @@ -137,18 +137,17 @@ def get(self, request, *args, **kwargs):
except:
pass

secrets = Secret.objects.filter(**secrets_filter)
secrets = Secret.objects.filter(**secrets_filter).prefetch_related('tags')

for secret in secrets:
log_secret_event(
secret,
SecretEvent.READ,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
log_secret_events_bulk(
list(secrets),
SecretEvent.READ,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

serializer = SecretSerializer(
secrets, many=True, context={"org_member": request.auth["org_member"]}
Expand Down Expand Up @@ -294,6 +293,8 @@ def post(self, request, *args, **kwargs):
):
return JsonResponse({"error": "Duplicate secret found"}, status=409)

created_secrets = []

for secret in request_body["secrets"]:

# Check that all encrypted fields are valid
Expand Down Expand Up @@ -335,16 +336,7 @@ def post(self, request, *args, **kwargs):

secret_obj = Secret.objects.create(**secret_data)
secret_obj.tags.set(tags)

log_secret_event(
secret_obj,
SecretEvent.CREATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
created_secrets.append(secret_obj)

# If the request is authenticated as a user and an override is supplied
if request.auth["org_member"] and "override" in secret:
Expand All @@ -354,6 +346,16 @@ def post(self, request, *args, **kwargs):
value=secret["override"]["value"],
)

log_secret_events_bulk(
created_secrets,
SecretEvent.CREATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

return Response(status=status.HTTP_200_OK)

def put(self, request, *args, **kwargs):
Expand All @@ -372,6 +374,8 @@ def put(self, request, *args, **kwargs):
):
return JsonResponse({"error": "Duplicate secret found"}, status=409)

updated_secrets = []

for secret in request_body["secrets"]:

secret_obj = Secret.objects.get(id=secret["id"])
Expand Down Expand Up @@ -448,16 +452,7 @@ def put(self, request, *args, **kwargs):
secret_obj.updated_at = timezone.now()
secret_obj.tags.set(tags)
secret_obj.save()

log_secret_event(
secret_obj,
SecretEvent.UPDATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
updated_secrets.append(secret_obj)

# If the request is authenticated as a user and an override is supplied
if request.auth["org_member"] and "override" in secret:
Expand All @@ -471,6 +466,16 @@ def put(self, request, *args, **kwargs):
},
)

log_secret_events_bulk(
updated_secrets,
SecretEvent.UPDATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

return Response(status=status.HTTP_200_OK)

def delete(self, request, *args, **kwargs):
Expand All @@ -479,27 +484,31 @@ def delete(self, request, *args, **kwargs):

ip_address, user_agent = get_resolver_request_meta(request)

secrets_to_delete = Secret.objects.filter(id__in=request_body["secrets"])
secrets_to_delete = Secret.objects.filter(
id__in=request_body["secrets"]
).prefetch_related('tags')

if not secrets_to_delete.exists():
return Response(status=status.HTTP_200_OK)

env = secrets_to_delete[0].environment

deleted_secrets = []
for secret in secrets_to_delete:
secret.updated_at = timezone.now()
secret.deleted_at = timezone.now()
secret.save()

log_secret_event(
secret,
SecretEvent.DELETE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
deleted_secrets.append(secret)

log_secret_events_bulk(
deleted_secrets,
SecretEvent.DELETE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

return Response(status=status.HTTP_200_OK)

Expand Down Expand Up @@ -583,18 +592,17 @@ def get(self, request, *args, **kwargs):
# Filter secrets based on these tags
secrets_filter["tags__in"] = tags

secrets = Secret.objects.filter(**secrets_filter)
secrets = Secret.objects.filter(**secrets_filter).prefetch_related('tags')

for secret in secrets:
log_secret_event(
secret,
SecretEvent.READ,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
log_secret_events_bulk(
list(secrets),
SecretEvent.READ,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

# Pre-compute crypto context for N+1 optimization
crypto_context = get_environment_crypto_context(env)
Expand Down Expand Up @@ -814,16 +822,6 @@ def post(self, request, *args, **kwargs):
)
secret_obj.tags.set(tags)

log_secret_event(
secret_obj,
SecretEvent.CREATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

# If the request is authenticated as a user and an override is supplied
if request.auth["org_member"] and "override" in secret:
PersonalSecret.objects.create(
Expand All @@ -834,6 +832,16 @@ def post(self, request, *args, **kwargs):

created_secrets.append(secret_obj)

log_secret_events_bulk(
created_secrets,
SecretEvent.CREATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

# Pre-compute crypto context for N+1 optimization
crypto_context = get_environment_crypto_context(env)
context_cache = {}
Expand Down Expand Up @@ -955,16 +963,6 @@ def put(self, request, *args, **kwargs):

secret_obj.save()

log_secret_event(
secret_obj,
SecretEvent.UPDATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

# If the request is authenticated as a user and an override is supplied
if request.auth["org_member"] and "override" in secret:
PersonalSecret.objects.update_or_create(
Expand All @@ -979,6 +977,16 @@ def put(self, request, *args, **kwargs):

updated_secrets.append(secret_obj)

log_secret_events_bulk(
updated_secrets,
SecretEvent.UPDATE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

# Pre-compute crypto context for N+1 optimization
crypto_context = get_environment_crypto_context(env)
context_cache = {}
Expand Down Expand Up @@ -1008,26 +1016,30 @@ def delete(self, request, *args, **kwargs):

ip_address, user_agent = get_resolver_request_meta(request)

secrets_to_delete = Secret.objects.filter(id__in=request_body["secrets"])
secrets_to_delete = Secret.objects.filter(
id__in=request_body["secrets"]
).prefetch_related('tags')

for secret in secrets_to_delete:
if not Secret.objects.filter(id=secret.id).exists():
return JsonResponse({"error": "Secret does not exist"}, status=404)

deleted_secrets = []
for secret in secrets_to_delete:
secret.updated_at = timezone.now()
secret.deleted_at = timezone.now()
secret.save()

log_secret_event(
secret,
SecretEvent.DELETE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)
deleted_secrets.append(secret)

log_secret_events_bulk(
deleted_secrets,
SecretEvent.DELETE,
request.auth["org_member"],
request.auth["service_token"],
request.auth["service_account_token"],
ip_address,
user_agent,
)

return Response(
{"message": f"Deleted {len(secrets_to_delete)} secrets"},
Expand Down
Loading
Loading