From 3579d339b54b36ac6a655f54200364564738f5c7 Mon Sep 17 00:00:00 2001 From: Nimish Date: Tue, 24 Feb 2026 22:08:05 +0530 Subject: [PATCH 1/4] perf: bulk INSERT secret events to reduce DB round-trips Replace per-secret log_secret_event() loops with a single log_secret_events_bulk() call using bulk_create() for both SecretEvent rows and M2M tag associations. Add prefetch_related('tags') on READ/DELETE querysets to eliminate N+1 tag queries. Updated across all 3 API surfaces: - E2EE REST (E2EESecretsView): GET, POST, PUT, DELETE - Public REST (PublicSecretsView): GET, POST, PUT, DELETE - GraphQL: BulkCreate, BulkEdit, BulkDelete, ReadSecret mutations --- backend/api/utils/audit_logging.py | 62 ++++++ backend/api/views/secrets.py | 182 ++++++++++-------- .../backend/graphene/mutations/environment.py | 64 +++--- 3 files changed, 195 insertions(+), 113 deletions(-) diff --git a/backend/api/utils/audit_logging.py b/backend/api/utils/audit_logging.py index e70d3d480..d93c01a30 100644 --- a/backend/api/utils/audit_logging.py +++ b/backend/api/utils/audit_logging.py @@ -41,3 +41,65 @@ def log_secret_event( ) event.tags.set(secret.tags.all()) + + +def log_secret_events_bulk( + secrets, + event_type, + user=None, + service_token=None, + service_account_token=None, + ip_address=None, + user_agent=None, +): + """ + Bulk version of log_secret_event. Logs events for multiple secrets + using bulk_create to reduce database round-trips. + """ + + if not secrets: + return [] + + service_account = None + if service_account_token is not None: + service_account = service_account_token.service_account + + now = timezone.now() + + events = [ + SecretEvent( + secret=secret, + environment=secret.environment, + folder=secret.folder, + path=secret.path, + user=user, + service_token=service_token, + service_account=service_account, + service_account_token=service_account_token, + key=secret.key, + key_digest=secret.key_digest, + value=secret.value, + version=secret.version, + comment=secret.comment, + event_type=event_type, + timestamp=now, + ip_address=ip_address, + user_agent=user_agent, + ) + for secret in secrets + ] + + created = SecretEvent.objects.bulk_create(events) + + # Bulk M2M: collect all tag associations and insert at once + through_model = SecretEvent.tags.through + m2m_rows = [] + for event, secret in zip(created, secrets): + for tag in secret.tags.all(): + m2m_rows.append( + through_model(secretevent_id=event.pk, secrettag_id=tag.pk) + ) + if m2m_rows: + through_model.objects.bulk_create(m2m_rows) + + return created diff --git a/backend/api/views/secrets.py b/backend/api/views/secrets.py index 4a9981632..24721e528 100644 --- a/backend/api/views/secrets.py +++ b/backend/api/views/secrets.py @@ -22,7 +22,7 @@ from api.utils.access.permissions import ( user_has_permission, ) -from api.utils.audit_logging import log_secret_event +from api.utils.audit_logging import log_secret_event, log_secret_events_bulk from api.utils.crypto import encrypt_asymmetric, validate_encrypted_string from api.utils.rest import ( @@ -137,18 +137,17 @@ def get(self, request, *args, **kwargs): except: pass - secrets = Secret.objects.filter(**secrets_filter) + secrets = Secret.objects.filter(**secrets_filter).prefetch_related('tags') - for secret in secrets: - log_secret_event( - secret, - SecretEvent.READ, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + log_secret_events_bulk( + list(secrets), + SecretEvent.READ, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) serializer = SecretSerializer( secrets, many=True, context={"org_member": request.auth["org_member"]} @@ -294,6 +293,8 @@ def post(self, request, *args, **kwargs): ): return JsonResponse({"error": "Duplicate secret found"}, status=409) + created_secrets = [] + for secret in request_body["secrets"]: # Check that all encrypted fields are valid @@ -334,16 +335,7 @@ def post(self, request, *args, **kwargs): secret_obj = Secret.objects.create(**secret_data) secret_obj.tags.set(tags) - - log_secret_event( - secret_obj, - SecretEvent.CREATE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + created_secrets.append(secret_obj) # If the request is authenticated as a user and an override is supplied if request.auth["org_member"] and "override" in secret: @@ -353,6 +345,16 @@ def post(self, request, *args, **kwargs): value=secret["override"]["value"], ) + log_secret_events_bulk( + created_secrets, + SecretEvent.CREATE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) + return Response(status=status.HTTP_200_OK) def put(self, request, *args, **kwargs): @@ -371,6 +373,8 @@ def put(self, request, *args, **kwargs): ): return JsonResponse({"error": "Duplicate secret found"}, status=409) + updated_secrets = [] + for secret in request_body["secrets"]: secret_obj = Secret.objects.get(id=secret["id"]) @@ -432,16 +436,7 @@ def put(self, request, *args, **kwargs): secret_obj.updated_at = timezone.now() secret_obj.tags.set(tags) secret_obj.save() - - log_secret_event( - secret_obj, - SecretEvent.UPDATE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + updated_secrets.append(secret_obj) # If the request is authenticated as a user and an override is supplied if request.auth["org_member"] and "override" in secret: @@ -455,6 +450,16 @@ def put(self, request, *args, **kwargs): }, ) + log_secret_events_bulk( + updated_secrets, + SecretEvent.UPDATE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) + return Response(status=status.HTTP_200_OK) def delete(self, request, *args, **kwargs): @@ -463,27 +468,31 @@ def delete(self, request, *args, **kwargs): ip_address, user_agent = get_resolver_request_meta(request) - secrets_to_delete = Secret.objects.filter(id__in=request_body["secrets"]) + secrets_to_delete = Secret.objects.filter( + id__in=request_body["secrets"] + ).prefetch_related('tags') if not secrets_to_delete.exists(): return Response(status=status.HTTP_200_OK) env = secrets_to_delete[0].environment + deleted_secrets = [] for secret in secrets_to_delete: secret.updated_at = timezone.now() secret.deleted_at = timezone.now() secret.save() - - log_secret_event( - secret, - SecretEvent.DELETE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + deleted_secrets.append(secret) + + log_secret_events_bulk( + deleted_secrets, + SecretEvent.DELETE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) return Response(status=status.HTTP_200_OK) @@ -567,18 +576,17 @@ def get(self, request, *args, **kwargs): # Filter secrets based on these tags secrets_filter["tags__in"] = tags - secrets = Secret.objects.filter(**secrets_filter) + secrets = Secret.objects.filter(**secrets_filter).prefetch_related('tags') - for secret in secrets: - log_secret_event( - secret, - SecretEvent.READ, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + log_secret_events_bulk( + list(secrets), + SecretEvent.READ, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) # Pre-compute crypto context for N+1 optimization crypto_context = get_environment_crypto_context(env) @@ -797,16 +805,6 @@ def post(self, request, *args, **kwargs): ) secret_obj.tags.set(tags) - log_secret_event( - secret_obj, - SecretEvent.CREATE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) - # If the request is authenticated as a user and an override is supplied if request.auth["org_member"] and "override" in secret: PersonalSecret.objects.create( @@ -817,6 +815,16 @@ def post(self, request, *args, **kwargs): created_secrets.append(secret_obj) + log_secret_events_bulk( + created_secrets, + SecretEvent.CREATE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) + # Pre-compute crypto context for N+1 optimization crypto_context = get_environment_crypto_context(env) context_cache = {} @@ -923,16 +931,6 @@ def put(self, request, *args, **kwargs): secret_obj.save() - log_secret_event( - secret_obj, - SecretEvent.UPDATE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) - # If the request is authenticated as a user and an override is supplied if request.auth["org_member"] and "override" in secret: PersonalSecret.objects.update_or_create( @@ -947,6 +945,16 @@ def put(self, request, *args, **kwargs): updated_secrets.append(secret_obj) + log_secret_events_bulk( + updated_secrets, + SecretEvent.UPDATE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) + # Pre-compute crypto context for N+1 optimization crypto_context = get_environment_crypto_context(env) context_cache = {} @@ -976,26 +984,30 @@ def delete(self, request, *args, **kwargs): ip_address, user_agent = get_resolver_request_meta(request) - secrets_to_delete = Secret.objects.filter(id__in=request_body["secrets"]) + secrets_to_delete = Secret.objects.filter( + id__in=request_body["secrets"] + ).prefetch_related('tags') for secret in secrets_to_delete: if not Secret.objects.filter(id=secret.id).exists(): return JsonResponse({"error": "Secret does not exist"}, status=404) + deleted_secrets = [] for secret in secrets_to_delete: secret.updated_at = timezone.now() secret.deleted_at = timezone.now() secret.save() - - log_secret_event( - secret, - SecretEvent.DELETE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + deleted_secrets.append(secret) + + log_secret_events_bulk( + deleted_secrets, + SecretEvent.DELETE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) return Response( {"message": f"Deleted {len(secrets_to_delete)} secrets"}, diff --git a/backend/backend/graphene/mutations/environment.py b/backend/backend/graphene/mutations/environment.py index fd4b4cc28..20bfc8fb9 100644 --- a/backend/backend/graphene/mutations/environment.py +++ b/backend/backend/graphene/mutations/environment.py @@ -9,7 +9,7 @@ user_has_permission, user_is_org_member, ) -from api.utils.audit_logging import log_secret_event +from api.utils.audit_logging import log_secret_event, log_secret_events_bulk from api.utils.secrets import create_environment_folder_structure, normalize_path_string from backend.quotas import can_add_environment, can_use_custom_envs import graphene @@ -814,19 +814,21 @@ def mutate(cls, root, info, secrets_data): secret.tags.set(tags) created_secrets.append(secret) - ip_address, user_agent = get_resolver_request_meta(info.context) - org_member = OrganisationMember.objects.get( - user=info.context.user, organisation=org, deleted_at=None - ) - log_secret_event( - secret, - SecretEvent.CREATE, - org_member, - None, - None, - ip_address, - user_agent, - ) + ip_address, user_agent = get_resolver_request_meta(info.context) + org_member = OrganisationMember.objects.get( + user=info.context.user, + organisation=Environment.objects.get(id=secrets_data[0].env_id).app.organisation, + deleted_at=None, + ) + log_secret_events_bulk( + created_secrets, + SecretEvent.CREATE, + org_member, + None, + None, + ip_address, + user_agent, + ) return BulkCreateSecretMutation(secrets=created_secrets) @@ -939,12 +941,15 @@ def mutate(cls, root, info, secrets_data): secret.save() updated_secrets.append(secret) + if updated_secrets: ip_address, user_agent = get_resolver_request_meta(info.context) org_member = OrganisationMember.objects.get( - user=info.context.user, organisation=org, deleted_at=None + user=info.context.user, + organisation=updated_secrets[0].environment.app.organisation, + deleted_at=None, ) - log_secret_event( - secret, + log_secret_events_bulk( + updated_secrets, SecretEvent.UPDATE, org_member, None, @@ -1020,12 +1025,15 @@ def mutate(cls, root, info, ids): secret.save() deleted_secrets.append(secret) + if deleted_secrets: ip_address, user_agent = get_resolver_request_meta(info.context) org_member = OrganisationMember.objects.get( - user=info.context.user, organisation=org, deleted_at=None + user=info.context.user, + organisation=deleted_secrets[0].environment.app.organisation, + deleted_at=None, ) - log_secret_event( - secret, + log_secret_events_bulk( + deleted_secrets, SecretEvent.DELETE, org_member, None, @@ -1045,24 +1053,24 @@ class Arguments: @classmethod def mutate(cls, root, info, ids): + secrets = [] for id in ids: secret = Secret.objects.get(id=id) if not user_can_access_environment( info.context.user.userId, secret.environment.id ): raise GraphQLError("You don't have permission to perform this action") + secrets.append(secret) - env = secret.environment - org = env.app.organisation - + if secrets: ip_address, user_agent = get_resolver_request_meta(info.context) - org_member = OrganisationMember.objects.get( - user=info.context.user, organisation=org, deleted_at=None + user=info.context.user, + organisation=secrets[0].environment.app.organisation, + deleted_at=None, ) - - log_secret_event( - secret, + log_secret_events_bulk( + secrets, SecretEvent.READ, org_member, None, From f18ba8dc3db4f9792fe0aa735a75624fccc8f050 Mon Sep 17 00:00:00 2001 From: rohan Date: Mon, 11 May 2026 15:48:46 +0530 Subject: [PATCH 2/4] fix: propagate type, prefetch tags, and atomic-wrap bulk audit events --- backend/api/utils/audit_logging.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/backend/api/utils/audit_logging.py b/backend/api/utils/audit_logging.py index 335ea1823..f81b8e303 100644 --- a/backend/api/utils/audit_logging.py +++ b/backend/api/utils/audit_logging.py @@ -1,4 +1,6 @@ from django.utils import timezone +from django.db import transaction +from django.db.models import prefetch_related_objects from api.models import SecretEvent @@ -65,7 +67,8 @@ def log_secret_events_bulk( if service_account_token is not None: service_account = service_account_token.service_account - now = timezone.now() + # Load tags for all secrets in one query, regardless of caller's prefetch state. + prefetch_related_objects(secrets, "tags") events = [ SecretEvent( @@ -82,25 +85,24 @@ def log_secret_events_bulk( value=secret.value, version=secret.version, comment=secret.comment, + type=secret.type, event_type=event_type, - timestamp=now, ip_address=ip_address, user_agent=user_agent, ) for secret in secrets ] - created = SecretEvent.objects.bulk_create(events) + with transaction.atomic(): + created = SecretEvent.objects.bulk_create(events) - # Bulk M2M: collect all tag associations and insert at once - through_model = SecretEvent.tags.through - m2m_rows = [] - for event, secret in zip(created, secrets): - for tag in secret.tags.all(): - m2m_rows.append( - through_model(secretevent_id=event.pk, secrettag_id=tag.pk) - ) - if m2m_rows: - through_model.objects.bulk_create(m2m_rows) + through_model = SecretEvent.tags.through + m2m_rows = [ + through_model(secretevent_id=event.pk, secrettag_id=tag.pk) + for event, secret in zip(created, secrets) + for tag in secret.tags.all() + ] + if m2m_rows: + through_model.objects.bulk_create(m2m_rows) return created From 18830c6b188249c586705960dd3299e4881e8de8 Mon Sep 17 00:00:00 2001 From: rohan Date: Mon, 11 May 2026 15:48:47 +0530 Subject: [PATCH 3/4] fix: guard BulkCreateSecret org lookup against empty input --- .../backend/graphene/mutations/environment.py | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/backend/backend/graphene/mutations/environment.py b/backend/backend/graphene/mutations/environment.py index 01650d0ab..d4188ea61 100644 --- a/backend/backend/graphene/mutations/environment.py +++ b/backend/backend/graphene/mutations/environment.py @@ -821,21 +821,22 @@ def mutate(cls, root, info, secrets_data): secret.tags.set(tags) created_secrets.append(secret) - ip_address, user_agent = get_resolver_request_meta(info.context) - org_member = OrganisationMember.objects.get( - user=info.context.user, - organisation=Environment.objects.get(id=secrets_data[0].env_id).app.organisation, - deleted_at=None, - ) - log_secret_events_bulk( - created_secrets, - SecretEvent.CREATE, - org_member, - None, - None, - ip_address, - user_agent, - ) + if created_secrets: + ip_address, user_agent = get_resolver_request_meta(info.context) + org_member = OrganisationMember.objects.get( + user=info.context.user, + organisation=created_secrets[0].environment.app.organisation, + deleted_at=None, + ) + log_secret_events_bulk( + created_secrets, + SecretEvent.CREATE, + org_member, + None, + None, + ip_address, + user_agent, + ) return BulkCreateSecretMutation(secrets=created_secrets) From ef7e93615e46009644193b3f62afb94154aa526e Mon Sep 17 00:00:00 2001 From: rohan Date: Mon, 11 May 2026 15:58:17 +0530 Subject: [PATCH 4/4] perf: cap bulk audit insert batch size to 1000 --- backend/api/utils/audit_logging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/api/utils/audit_logging.py b/backend/api/utils/audit_logging.py index f81b8e303..29548c253 100644 --- a/backend/api/utils/audit_logging.py +++ b/backend/api/utils/audit_logging.py @@ -94,7 +94,7 @@ def log_secret_events_bulk( ] with transaction.atomic(): - created = SecretEvent.objects.bulk_create(events) + created = SecretEvent.objects.bulk_create(events, batch_size=1000) through_model = SecretEvent.tags.through m2m_rows = [ @@ -103,6 +103,6 @@ def log_secret_events_bulk( for tag in secret.tags.all() ] if m2m_rows: - through_model.objects.bulk_create(m2m_rows) + through_model.objects.bulk_create(m2m_rows, batch_size=1000) return created