diff --git a/backend/api/utils/audit_logging.py b/backend/api/utils/audit_logging.py index 11e20c611..29548c253 100644 --- a/backend/api/utils/audit_logging.py +++ b/backend/api/utils/audit_logging.py @@ -1,4 +1,6 @@ from django.utils import timezone +from django.db import transaction +from django.db.models import prefetch_related_objects from api.models import SecretEvent @@ -42,3 +44,65 @@ def log_secret_event( ) event.tags.set(secret.tags.all()) + + +def log_secret_events_bulk( + secrets, + event_type, + user=None, + service_token=None, + service_account_token=None, + ip_address=None, + user_agent=None, +): + """ + Bulk version of log_secret_event. Logs events for multiple secrets + using bulk_create to reduce database round-trips. + """ + + if not secrets: + return [] + + service_account = None + if service_account_token is not None: + service_account = service_account_token.service_account + + # Load tags for all secrets in one query, regardless of caller's prefetch state. + prefetch_related_objects(secrets, "tags") + + events = [ + SecretEvent( + secret=secret, + environment=secret.environment, + folder=secret.folder, + path=secret.path, + user=user, + service_token=service_token, + service_account=service_account, + service_account_token=service_account_token, + key=secret.key, + key_digest=secret.key_digest, + value=secret.value, + version=secret.version, + comment=secret.comment, + type=secret.type, + event_type=event_type, + ip_address=ip_address, + user_agent=user_agent, + ) + for secret in secrets + ] + + with transaction.atomic(): + created = SecretEvent.objects.bulk_create(events, batch_size=1000) + + through_model = SecretEvent.tags.through + m2m_rows = [ + through_model(secretevent_id=event.pk, secrettag_id=tag.pk) + for event, secret in zip(created, secrets) + for tag in secret.tags.all() + ] + if m2m_rows: + through_model.objects.bulk_create(m2m_rows, batch_size=1000) + + return created diff --git a/backend/api/views/secrets.py b/backend/api/views/secrets.py index 4f44773bf..e9569cb4f 100644 --- a/backend/api/views/secrets.py +++ b/backend/api/views/secrets.py @@ -22,7 +22,7 @@ from api.utils.access.permissions import ( user_has_permission, ) -from api.utils.audit_logging import log_secret_event +from api.utils.audit_logging import log_secret_event, log_secret_events_bulk from api.utils.crypto import encrypt_asymmetric, validate_encrypted_string from api.utils.rest import ( @@ -137,18 +137,17 @@ def get(self, request, *args, **kwargs): except: pass - secrets = Secret.objects.filter(**secrets_filter) + secrets = Secret.objects.filter(**secrets_filter).prefetch_related('tags') - for secret in secrets: - log_secret_event( - secret, - SecretEvent.READ, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + log_secret_events_bulk( + list(secrets), + SecretEvent.READ, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) serializer = SecretSerializer( secrets, many=True, context={"org_member": request.auth["org_member"]} @@ -294,6 +293,8 @@ def post(self, request, *args, **kwargs): ): return JsonResponse({"error": "Duplicate secret found"}, status=409) + created_secrets = [] + for secret in request_body["secrets"]: # Check that all encrypted fields are valid @@ -335,16 +336,7 @@ def post(self, request, *args, **kwargs): secret_obj = Secret.objects.create(**secret_data) secret_obj.tags.set(tags) - - log_secret_event( - secret_obj, - SecretEvent.CREATE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + created_secrets.append(secret_obj) # If the request is authenticated as a user and an override is supplied if request.auth["org_member"] and "override" in secret: @@ -354,6 +346,16 @@ def post(self, request, *args, **kwargs): value=secret["override"]["value"], ) + log_secret_events_bulk( + created_secrets, + SecretEvent.CREATE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) + return Response(status=status.HTTP_200_OK) def put(self, request, *args, **kwargs): @@ -372,6 +374,8 @@ def put(self, request, *args, **kwargs): ): return JsonResponse({"error": "Duplicate secret found"}, status=409) + updated_secrets = [] + for secret in request_body["secrets"]: secret_obj = Secret.objects.get(id=secret["id"]) @@ -448,16 +452,7 @@ def put(self, request, *args, **kwargs): secret_obj.updated_at = timezone.now() secret_obj.tags.set(tags) secret_obj.save() - - log_secret_event( - secret_obj, - SecretEvent.UPDATE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + updated_secrets.append(secret_obj) # If the request is authenticated as a user and an override is supplied if request.auth["org_member"] and "override" in secret: @@ -471,6 +466,16 @@ def put(self, request, *args, **kwargs): }, ) + log_secret_events_bulk( + updated_secrets, + SecretEvent.UPDATE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) + return Response(status=status.HTTP_200_OK) def delete(self, request, *args, **kwargs): @@ -479,27 +484,31 @@ def delete(self, request, *args, **kwargs): ip_address, user_agent = get_resolver_request_meta(request) - secrets_to_delete = Secret.objects.filter(id__in=request_body["secrets"]) + secrets_to_delete = Secret.objects.filter( + id__in=request_body["secrets"] + ).prefetch_related('tags') if not secrets_to_delete.exists(): return Response(status=status.HTTP_200_OK) env = secrets_to_delete[0].environment + deleted_secrets = [] for secret in secrets_to_delete: secret.updated_at = timezone.now() secret.deleted_at = timezone.now() secret.save() - - log_secret_event( - secret, - SecretEvent.DELETE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + deleted_secrets.append(secret) + + log_secret_events_bulk( + deleted_secrets, + SecretEvent.DELETE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) return Response(status=status.HTTP_200_OK) @@ -583,18 +592,17 @@ def get(self, request, *args, **kwargs): # Filter secrets based on these tags secrets_filter["tags__in"] = tags - secrets = Secret.objects.filter(**secrets_filter) + secrets = Secret.objects.filter(**secrets_filter).prefetch_related('tags') - for secret in secrets: - log_secret_event( - secret, - SecretEvent.READ, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + log_secret_events_bulk( + list(secrets), + SecretEvent.READ, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) # Pre-compute crypto context for N+1 optimization crypto_context = get_environment_crypto_context(env) @@ -814,16 +822,6 @@ def post(self, request, *args, **kwargs): ) secret_obj.tags.set(tags) - log_secret_event( - secret_obj, - SecretEvent.CREATE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) - # If the request is authenticated as a user and an override is supplied if request.auth["org_member"] and "override" in secret: PersonalSecret.objects.create( @@ -834,6 +832,16 @@ def post(self, request, *args, **kwargs): created_secrets.append(secret_obj) + log_secret_events_bulk( + created_secrets, + SecretEvent.CREATE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) + # Pre-compute crypto context for N+1 optimization crypto_context = get_environment_crypto_context(env) context_cache = {} @@ -955,16 +963,6 @@ def put(self, request, *args, **kwargs): secret_obj.save() - log_secret_event( - secret_obj, - SecretEvent.UPDATE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) - # If the request is authenticated as a user and an override is supplied if request.auth["org_member"] and "override" in secret: PersonalSecret.objects.update_or_create( @@ -979,6 +977,16 @@ def put(self, request, *args, **kwargs): updated_secrets.append(secret_obj) + log_secret_events_bulk( + updated_secrets, + SecretEvent.UPDATE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) + # Pre-compute crypto context for N+1 optimization crypto_context = get_environment_crypto_context(env) context_cache = {} @@ -1008,26 +1016,30 @@ def delete(self, request, *args, **kwargs): ip_address, user_agent = get_resolver_request_meta(request) - secrets_to_delete = Secret.objects.filter(id__in=request_body["secrets"]) + secrets_to_delete = Secret.objects.filter( + id__in=request_body["secrets"] + ).prefetch_related('tags') for secret in secrets_to_delete: if not Secret.objects.filter(id=secret.id).exists(): return JsonResponse({"error": "Secret does not exist"}, status=404) + deleted_secrets = [] for secret in secrets_to_delete: secret.updated_at = timezone.now() secret.deleted_at = timezone.now() secret.save() - - log_secret_event( - secret, - SecretEvent.DELETE, - request.auth["org_member"], - request.auth["service_token"], - request.auth["service_account_token"], - ip_address, - user_agent, - ) + deleted_secrets.append(secret) + + log_secret_events_bulk( + deleted_secrets, + SecretEvent.DELETE, + request.auth["org_member"], + request.auth["service_token"], + request.auth["service_account_token"], + ip_address, + user_agent, + ) return Response( {"message": f"Deleted {len(secrets_to_delete)} secrets"}, diff --git a/backend/backend/graphene/mutations/environment.py b/backend/backend/graphene/mutations/environment.py index c01af8da3..d4188ea61 100644 --- a/backend/backend/graphene/mutations/environment.py +++ b/backend/backend/graphene/mutations/environment.py @@ -10,7 +10,7 @@ user_has_permission, user_is_org_member, ) -from api.utils.audit_logging import log_secret_event +from api.utils.audit_logging import log_secret_event, log_secret_events_bulk from api.utils.secrets import create_environment_folder_structure, normalize_path_string from backend.quotas import can_add_environment, can_use_custom_envs import graphene @@ -821,12 +821,15 @@ def mutate(cls, root, info, secrets_data): secret.tags.set(tags) created_secrets.append(secret) + if created_secrets: ip_address, user_agent = get_resolver_request_meta(info.context) org_member = OrganisationMember.objects.get( - user=info.context.user, organisation=org, deleted_at=None + user=info.context.user, + organisation=created_secrets[0].environment.app.organisation, + deleted_at=None, ) - log_secret_event( - secret, + log_secret_events_bulk( + created_secrets, SecretEvent.CREATE, org_member, None, @@ -970,12 +973,15 @@ def mutate(cls, root, info, secrets_data): secret.save() updated_secrets.append(secret) + if updated_secrets: ip_address, user_agent = get_resolver_request_meta(info.context) org_member = OrganisationMember.objects.get( - user=info.context.user, organisation=org, deleted_at=None + user=info.context.user, + organisation=updated_secrets[0].environment.app.organisation, + deleted_at=None, ) - log_secret_event( - secret, + log_secret_events_bulk( + updated_secrets, SecretEvent.UPDATE, org_member, None, @@ -1051,12 +1057,15 @@ def mutate(cls, root, info, ids): secret.save() deleted_secrets.append(secret) + if deleted_secrets: ip_address, user_agent = get_resolver_request_meta(info.context) org_member = OrganisationMember.objects.get( - user=info.context.user, organisation=org, deleted_at=None + user=info.context.user, + organisation=deleted_secrets[0].environment.app.organisation, + deleted_at=None, ) - log_secret_event( - secret, + log_secret_events_bulk( + deleted_secrets, SecretEvent.DELETE, org_member, None, @@ -1080,24 +1089,24 @@ class Arguments: @classmethod def mutate(cls, root, info, ids): + secrets = [] for id in ids: secret = Secret.objects.get(id=id) if not user_can_access_environment( info.context.user.userId, secret.environment.id ): raise GraphQLError("You don't have permission to perform this action") + secrets.append(secret) - env = secret.environment - org = env.app.organisation - + if secrets: ip_address, user_agent = get_resolver_request_meta(info.context) - org_member = OrganisationMember.objects.get( - user=info.context.user, organisation=org, deleted_at=None + user=info.context.user, + organisation=secrets[0].environment.app.organisation, + deleted_at=None, ) - - log_secret_event( - secret, + log_secret_events_bulk( + secrets, SecretEvent.READ, org_member, None,