From ee4d738870dcfae5660935f18458f992548cff78 Mon Sep 17 00:00:00 2001 From: Vignesh Aigal Date: Sun, 17 Mar 2024 10:11:18 -0700 Subject: [PATCH 1/3] Update file --- .../migrations/0004_alter_userfiles_user.py | 21 +++++++++++++++++++ llmstack/datasources/models.py | 10 ++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 llmstack/datasources/migrations/0004_alter_userfiles_user.py diff --git a/llmstack/datasources/migrations/0004_alter_userfiles_user.py b/llmstack/datasources/migrations/0004_alter_userfiles_user.py new file mode 100644 index 00000000000..0ebfea1f681 --- /dev/null +++ b/llmstack/datasources/migrations/0004_alter_userfiles_user.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.10 on 2024-03-17 16:02 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('datasources', '0003_userfiles'), + ] + + operations = [ + migrations.AlterField( + model_name='userfiles', + name='user', + field=models.ForeignKey(help_text='User this asset belongs to', on_delete=django.db.models.deletion.DO_NOTHING, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/llmstack/datasources/models.py b/llmstack/datasources/models.py index 506ec8aeff0..5799824b34f 100644 --- a/llmstack/datasources/models.py +++ b/llmstack/datasources/models.py @@ -4,6 +4,8 @@ from django.contrib.auth.models import User from django.db import models +from django.db.models.signals import pre_delete +from django.dispatch import receiver from django.utils.timezone import now from llmstack.base.models import Profile @@ -198,7 +200,7 @@ def upload_to(instance, filename): class UserFiles(models.Model): uuid = models.UUIDField(default=uuid.uuid4, editable=False, help_text="UUID of the asset") - user = models.OneToOneField(User, on_delete=models.DO_NOTHING, help_text="User this asset belongs to") + user = models.ForeignKey(User, on_delete=models.DO_NOTHING, help_text="User this asset belongs to") path = "" file = models.FileField( storage=select_storage, @@ -223,6 +225,12 @@ def profile_uuid(self): return Profile.objects.get(user=self.user).uuid +@receiver(pre_delete, sender=UserFiles) +def delete_file_on_delete(sender, instance, **kwargs): + if instance.file: + instance.file.delete(False) + + def create_from_bytes(user, file_bytes, filename, metadata=None): from django.core.files.base import ContentFile From 7bd6928a5b1603fa54e7dea04c29c283feb37741 Mon Sep 17 00:00:00 2001 From: Vignesh Aigal Date: Sun, 17 Mar 2024 20:53:01 -0700 Subject: [PATCH 2/3] Add a checksume generator file --- llmstack/common/utils/utils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/llmstack/common/utils/utils.py b/llmstack/common/utils/utils.py index dbf0b5696b5..37b6225d919 100644 --- a/llmstack/common/utils/utils.py +++ b/llmstack/common/utils/utils.py @@ -1,4 +1,5 @@ import copy +import hashlib import logging import re import time @@ -103,6 +104,22 @@ def validate_parse_data_uri( return (mime_type, file_name, data) +def generate_checksum(file_data, algorithm="sha256"): + """ + Generates a checksum of file data using the specified algorithm. + + Parameters: + file_data (bytes): The data of the file as bytes. + algorithm (str): The hashing algorithm to use (default is SHA-256). + + Returns: + str: The checksum of the file data. + """ + checksum = hashlib.new(algorithm) + checksum.update(file_data) + return checksum.hexdigest() + + def sanitize_dict_values(value): if isinstance(value, dict): return {k: sanitize_dict_values(v) for k, v in value.items()} From d7af8d3491b135e86337e3429c290cbeae5e9bfa Mon Sep 17 00:00:00 2001 From: Vignesh Aigal Date: Sun, 17 Mar 2024 21:40:59 -0700 Subject: [PATCH 3/3] Make methods take path param --- llmstack/datasources/models.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/llmstack/datasources/models.py b/llmstack/datasources/models.py index 5799824b34f..d12f154ce35 100644 --- a/llmstack/datasources/models.py +++ b/llmstack/datasources/models.py @@ -231,10 +231,10 @@ def delete_file_on_delete(sender, instance, **kwargs): instance.file.delete(False) -def create_from_bytes(user, file_bytes, filename, metadata=None): +def create_from_bytes(user, file_bytes, filename, metadata=None, path=""): from django.core.files.base import ContentFile - asset = UserFiles(user=user) + asset = UserFiles(user=user, path=path) asset.file.save( filename, ContentFile(file_bytes), @@ -245,7 +245,9 @@ def create_from_bytes(user, file_bytes, filename, metadata=None): return asset -def create_from_data_uri(user, data_uri, metadata={}): +def create_from_data_uri(user, data_uri, metadata={}, path=""): mime_type, file_name, file_data = validate_parse_data_uri(data_uri) file_bytes = base64.b64decode(file_data) - return create_from_bytes(user, file_bytes, file_name, {**metadata, "mime_type": mime_type, "file_name": file_name}) + return create_from_bytes( + user, file_bytes, file_name, {**metadata, "mime_type": mime_type, "file_name": file_name}, path=path + )