From 216dc0867e938aed779b824de1e7b73e1745e1d9 Mon Sep 17 00:00:00 2001 From: Vignesh Aigal Date: Fri, 15 Mar 2024 01:37:42 -0700 Subject: [PATCH 1/3] Add a new model for user assets --- llmstack/datasources/admin.py | 3 +- .../datasources/migrations/0003_userfiles.py | 28 ++++++++++ llmstack/datasources/models.py | 51 +++++++++++++++++++ llmstack/server/settings.py | 3 ++ 4 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 llmstack/datasources/migrations/0003_userfiles.py diff --git a/llmstack/datasources/admin.py b/llmstack/datasources/admin.py index 151dd86bdbe..bada131ed85 100644 --- a/llmstack/datasources/admin.py +++ b/llmstack/datasources/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from .models import DataSource, DataSourceEntry, DataSourceType +from .models import DataSource, DataSourceEntry, DataSourceType, UserFiles class DataSourceEntryAdmin(admin.ModelAdmin): @@ -15,3 +15,4 @@ class DataSourceAdmin(admin.ModelAdmin): admin.site.register(DataSourceEntry, DataSourceEntryAdmin) admin.site.register(DataSourceType, DataSourceAdmin) admin.site.register(DataSource) +admin.site.register(UserFiles) diff --git a/llmstack/datasources/migrations/0003_userfiles.py b/llmstack/datasources/migrations/0003_userfiles.py new file mode 100644 index 00000000000..074630869ad --- /dev/null +++ b/llmstack/datasources/migrations/0003_userfiles.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.10 on 2024-03-15 08:33 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import llmstack.datasources.models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('datasources', '0002_datasource_config_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='UserFiles', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('path', models.CharField(blank=True, help_text='Path to the asset', max_length=256, null=True)), + ('file', models.FileField(blank=True, null=True, storage=llmstack.datasources.models.select_storage, upload_to=llmstack.datasources.models.upload_to)), + ('metadata', models.JSONField(blank=True, default=dict, help_text='Metadata for the asset', null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('user', models.OneToOneField(help_text='User this asset belongs to', on_delete=django.db.models.deletion.DO_NOTHING, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/llmstack/datasources/models.py b/llmstack/datasources/models.py index 8e27f6e2476..f3411fae754 100644 --- a/llmstack/datasources/models.py +++ b/llmstack/datasources/models.py @@ -176,3 +176,54 @@ def user_can_read(self, user) -> bool: ).organization ) return False + + +def select_storage(): + from django.core.files.storage import storages + + return storages["useruploads"] + + +def upload_to(instance, filename): + return "/".join( + [ + str(instance.profile_uuid), + instance.path, + filename, + ] + ) + + +class UserFiles(models.Model): + user = models.OneToOneField(User, on_delete=models.DO_NOTHING, help_text="User this asset belongs to") + path = models.CharField(max_length=256, help_text="Path to the asset", null=True, blank=True) + file = models.FileField( + storage=select_storage, + upload_to=upload_to, + null=True, + blank=True, + ) + metadata = models.JSONField( + default=dict, + help_text="Metadata for the asset", + null=True, + blank=True, + ) + created_at = models.DateTimeField(auto_now_add=True) + + @property + def profile_uuid(self): + return Profile.objects.get(user=self.user).uuid + + +def create_from_bytes(user, file_bytes, filename, metadata=None): + from django.core.files.base import ContentFile + + asset = UserFiles(user=user) + asset.file.save( + filename, + ContentFile(file_bytes), + ) + asset.metadata = metadata + asset.save() + return asset diff --git a/llmstack/server/settings.py b/llmstack/server/settings.py index 46520c031b9..9be1bc058f9 100644 --- a/llmstack/server/settings.py +++ b/llmstack/server/settings.py @@ -208,6 +208,9 @@ "base_url": GENERATEDFILES_URL, }, }, + "useruploads": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, } # Default primary key field type From c4d490dd81ce005c5c5074ef6af27028424d1c6e Mon Sep 17 00:00:00 2001 From: Vignesh Aigal Date: Sat, 16 Mar 2024 03:50:42 -0700 Subject: [PATCH 2/3] Update --- .../datasources/migrations/0003_userfiles.py | 3 +-- llmstack/datasources/models.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/llmstack/datasources/migrations/0003_userfiles.py b/llmstack/datasources/migrations/0003_userfiles.py index 074630869ad..b7de3cad248 100644 --- a/llmstack/datasources/migrations/0003_userfiles.py +++ b/llmstack/datasources/migrations/0003_userfiles.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-03-15 08:33 +# Generated by Django 4.2.10 on 2024-03-16 10:49 from django.conf import settings from django.db import migrations, models @@ -18,7 +18,6 @@ class Migration(migrations.Migration): name='UserFiles', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('path', models.CharField(blank=True, help_text='Path to the asset', max_length=256, null=True)), ('file', models.FileField(blank=True, null=True, storage=llmstack.datasources.models.select_storage, upload_to=llmstack.datasources.models.upload_to)), ('metadata', models.JSONField(blank=True, default=dict, help_text='Metadata for the asset', null=True)), ('created_at', models.DateTimeField(auto_now_add=True)), diff --git a/llmstack/datasources/models.py b/llmstack/datasources/models.py index f3411fae754..df38a596156 100644 --- a/llmstack/datasources/models.py +++ b/llmstack/datasources/models.py @@ -1,3 +1,4 @@ +import base64 import logging import uuid @@ -6,6 +7,7 @@ from django.utils.timezone import now from llmstack.base.models import Profile +from llmstack.common.utils.utils import validate_parse_data_uri logger = logging.getLogger(__name__) @@ -196,7 +198,7 @@ def upload_to(instance, filename): class UserFiles(models.Model): user = models.OneToOneField(User, on_delete=models.DO_NOTHING, help_text="User this asset belongs to") - path = models.CharField(max_length=256, help_text="Path to the asset", null=True, blank=True) + path = "" file = models.FileField( storage=select_storage, upload_to=upload_to, @@ -211,6 +213,10 @@ class UserFiles(models.Model): ) created_at = models.DateTimeField(auto_now_add=True) + def __init__(self, *args, path="", **kwargs) -> None: + super(UserFiles, self).__init__(*args, **kwargs) + self.path = path + @property def profile_uuid(self): return Profile.objects.get(user=self.user).uuid @@ -224,6 +230,13 @@ def create_from_bytes(user, file_bytes, filename, metadata=None): filename, ContentFile(file_bytes), ) - asset.metadata = metadata + bytes_size = len(file_bytes) + asset.metadata = {**metadata, "file_size": bytes_size} asset.save() return asset + + +def create_from_data_uri(user, data_uri, metadata={}): + mime_type, file_name, file_data = validate_parse_data_uri(data_uri) + file_bytes = base64.b64decode(file_data) + return create_from_bytes(user, file_bytes, file_name, {**metadata, "mime_type": mime_type, "file_name": file_name}) From e44de8a3573641ae6f4d186fa6d6945eaa55d8e7 Mon Sep 17 00:00:00 2001 From: Vignesh Aigal Date: Sat, 16 Mar 2024 03:54:25 -0700 Subject: [PATCH 3/3] . --- llmstack/datasources/migrations/0003_userfiles.py | 4 +++- llmstack/datasources/models.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/llmstack/datasources/migrations/0003_userfiles.py b/llmstack/datasources/migrations/0003_userfiles.py index b7de3cad248..1bda2cc091d 100644 --- a/llmstack/datasources/migrations/0003_userfiles.py +++ b/llmstack/datasources/migrations/0003_userfiles.py @@ -1,9 +1,10 @@ -# Generated by Django 4.2.10 on 2024-03-16 10:49 +# Generated by Django 4.2.10 on 2024-03-16 10:53 from django.conf import settings from django.db import migrations, models import django.db.models.deletion import llmstack.datasources.models +import uuid class Migration(migrations.Migration): @@ -18,6 +19,7 @@ class Migration(migrations.Migration): name='UserFiles', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, help_text='UUID of the asset')), ('file', models.FileField(blank=True, null=True, storage=llmstack.datasources.models.select_storage, upload_to=llmstack.datasources.models.upload_to)), ('metadata', models.JSONField(blank=True, default=dict, help_text='Metadata for the asset', null=True)), ('created_at', models.DateTimeField(auto_now_add=True)), diff --git a/llmstack/datasources/models.py b/llmstack/datasources/models.py index df38a596156..506ec8aeff0 100644 --- a/llmstack/datasources/models.py +++ b/llmstack/datasources/models.py @@ -197,6 +197,7 @@ def upload_to(instance, filename): class UserFiles(models.Model): + uuid = models.UUIDField(default=uuid.uuid4, editable=False, help_text="UUID of the asset") user = models.OneToOneField(User, on_delete=models.DO_NOTHING, help_text="User this asset belongs to") path = "" file = models.FileField(