Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

APIEndpointAsset model #165

Merged
merged 10 commits into from
Aug 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 8 additions & 0 deletions chirps/asset/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from polymorphic.admin import PolymorphicChildModelAdmin, PolymorphicParentModelAdmin

from .models import BaseAsset
from .providers.api_endpoint import APIEndpointAsset
from .providers.mantium import MantiumAsset
from .providers.pinecone import PineconeAsset
from .providers.redis import RedisAsset
Expand Down Expand Up @@ -33,6 +34,13 @@ class RedisAssetAdmin(PolymorphicChildModelAdmin):
base_model = RedisAsset


class APIEndpointAssetAdmin(PolymorphicChildModelAdmin):
"""Admin class for the APIEndpoint model."""

base_model = APIEndpointAsset


admin.site.register(RedisAsset)
admin.site.register(MantiumAsset)
admin.site.register(PineconeAsset)
admin.site.register(APIEndpointAsset)
31 changes: 31 additions & 0 deletions chirps/asset/forms.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
"""Forms for rendering and validating the asset models."""
from asset.models import BaseAsset
from asset.widgets import KeyValueWidget
from django import forms
from django.forms import ModelForm
from django.urls import reverse_lazy
from embedding.models import Embedding

from .providers.api_endpoint import APIEndpointAsset
from .providers.mantium import MantiumAsset
from .providers.pinecone import PineconeAsset
from .providers.redis import RedisAsset
Expand Down Expand Up @@ -164,10 +166,39 @@ def __init__(self, *args, **kwargs):
)


class APIEndpointAssetForm(ModelForm):
"""Form for the APIEndpointAsset model."""

class Meta:
"""Django Meta options for the APIEndpointAssetForm."""

model = APIEndpointAsset
fields = [
'name',
'url',
'authentication_method',
'api_key',
'headers',
'body',
]

widgets = {
'name': forms.TextInput(attrs={'class': 'form-control', 'placeholder': 'Enter a name for the asset'}),
'url': forms.URLInput(attrs={'class': 'form-control', 'placeholder': 'URL'}),
'authentication_method': forms.Select(
choices=[('Basic', 'Basic'), ('Bearer', 'Bearer')], attrs={'class': 'form-control'}
),
'api_key': forms.TextInput(attrs={'class': 'form-control', 'placeholder': 'API Key'}),
'headers': KeyValueWidget(attrs={'class': 'form-control'}),
'body': KeyValueWidget(attrs={'class': 'form-control'}),
}


assets = [
{'form': RedisAssetForm, 'model': RedisAsset},
{'form': MantiumAssetForm, 'model': MantiumAsset},
{'form': PineconeAssetForm, 'model': PineconeAsset},
{'form': APIEndpointAssetForm, 'model': APIEndpointAsset},
]


Expand Down
50 changes: 50 additions & 0 deletions chirps/asset/migrations/0003_apiendpointasset.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# Generated by Django 4.2.3 on 2023-08-21 13:22

import django.db.models.deletion
import fernet_fields.fields
from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
('asset', '0002_alter_pineconeasset_api_key'),
]

operations = [
migrations.CreateModel(
name='APIEndpointAsset',
fields=[
(
'baseasset_ptr',
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to='asset.baseasset',
),
),
('url', models.URLField(max_length=2048)),
(
'authentication_method',
models.CharField(
choices=[('Basic', 'Basic'), ('Bearer', 'Bearer')],
default='Bearer',
max_length=10,
),
),
('api_key', fernet_fields.fields.EncryptedCharField(max_length=256)),
('headers', models.JSONField(blank=True, null=True)),
(
'body',
models.JSONField(blank=True, default={'data': '%query%'}, null=True),
),
],
options={
'abstract': False,
'base_manager_name': 'objects',
},
bases=('asset.baseasset',),
),
]
54 changes: 54 additions & 0 deletions chirps/asset/providers/api_endpoint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
"""Logic for interfacing with an API Endpoint Asset."""
from logging import getLogger

from asset.models import BaseAsset, PingResult, SearchResult
from django.db import models
from fernet_fields import EncryptedCharField

logger = getLogger(__name__)


class APIEndpointAsset(BaseAsset):
"""Implementation of an API Endpoint asset."""

url = models.URLField(max_length=2048, blank=False, null=False)
authentication_method = models.CharField(
max_length=10, choices=[('Basic', 'Basic'), ('Bearer', 'Bearer')], default='Bearer'
)
api_key = EncryptedCharField(max_length=256, editable=True)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since this asset will support multiple auth types in the future, it's probably worth making this an opaque "secret" and adding an enum of the supported types (bearer is the only one to start). I'm guessing oauth is probably going to be needed sooner rather than later.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

spoke with @zimventures about this and I'm going to leave it as is. We don't expect that changing it later, with clearer direction, will take more effort than changing it now.

headers = models.JSONField(blank=True, null=True)
body = models.JSONField(blank=True, null=True, default={'data': '%query%'})

# Name of the file in the ./asset/static/ directory to use as a logo
html_logo = 'asset/api-endpoint-logo.svg'
html_name = 'API Endpoint'
html_description = 'Generic API Endpoint Asset'

HAS_PING = True

@property
def decrypted_api_key(self):
"""Return the decrypted API key."""
if self.api_key is not None:
try:
decrypted_value = self.api_key
masked_value = decrypted_value[:4] + '*' * (len(decrypted_value) - 4)
return masked_value
except UnicodeDecodeError:
return 'Error: Decryption failed'
return None

def search(self, query: str, max_results: int) -> list[SearchResult]:
"""Search the API Endpoint asset with the specified query."""
raise NotImplementedError('The search method is not implemented for this asset.')
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So then how do we scan this asset? 🤔

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I want to break this out into a separate PR to keep them both a reasonable size.


def test_connection(self) -> PingResult:
"""Ensure that the API Endpoint asset can be connected to."""
raise NotImplementedError('The test_connection method is not implemented for this asset.')

def displayable_attributes(self):
"""Display a subset of the model's attributes"""
return [
{'label': 'URL', 'value': self.url},
{'label': 'Authentication Method', 'value': self.authentication_method},
]
4 changes: 4 additions & 0 deletions chirps/asset/providers/mantium.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,7 @@ def search(self, query: str, max_results: int) -> list[SearchResult]:
documents = [SearchResult(data=doc['content']) for doc in results['documents']]
logger.debug('Mantium asset search complete', extra={'id': self.id})
return documents

def displayable_attributes(self):
"""Display a subset of the model's attributes"""
return [{'label': 'Application ID', 'value': self.app_id}]
11 changes: 10 additions & 1 deletion chirps/asset/providers/pinecone.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class PineconeAsset(BaseAsset):
embedding_model_service = models.CharField(max_length=256, default='OpenAI')

# Name of the file in the ./asset/static/ directory to use as a logo
html_logo = 'asset/pinecone-logo.png'
html_logo = 'asset/pinecone-logo.svg'
html_name = 'Pinecone'
html_description = 'Pinecone Vector Database'

Expand Down Expand Up @@ -67,3 +67,12 @@ def test_connection(self) -> PingResult:
except pinecone_lib.core.client.exceptions.UnauthorizedException as err:
logger.error('Pinecone connection test failed', extra={'error': err.body})
return PingResult(success=False, error=err.body)

def displayable_attributes(self):
"""Display a subset of the model's attributes"""
return [
{'label': 'API Key', 'value': self.decrypted_api_key},
{'label': 'Environment', 'value': self.environment},
{'label': 'Index Name', 'value': self.index_name},
{'label': 'Project Name', 'value': self.project_name},
]
11 changes: 10 additions & 1 deletion chirps/asset/providers/redis.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class RedisAsset(BaseAsset):
embedding_model_service = models.CharField(max_length=256, default='OpenAI')

# Name of the file in the ./asset/static/ directory to use as a logo
html_logo = 'asset/redis-logo.png'
html_logo = 'asset/redis-logo.svg'
html_name = 'Redis'
html_description = 'Redis Vector Database'

Expand Down Expand Up @@ -106,3 +106,12 @@ def test_connection(self) -> PingResult:
except exceptions.ConnectionError as err:
client.close()
return PingResult(success=False, error=str(err))

def displayable_attributes(self):
"""Display a subset of the model's attributes"""
return [
{'label': 'Host', 'value': self.host},
{'label': 'Port', 'value': self.port},
{'label': 'Database Name', 'value': self.database_name},
{'label': 'Username', 'value': self.username},
]
13 changes: 13 additions & 0 deletions chirps/asset/static/asset/api-endpoint-logo.svg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file removed chirps/asset/static/asset/mantiumai-logo.jpg
Binary file not shown.
Binary file removed chirps/asset/static/asset/pinecone-logo.png
Binary file not shown.
28 changes: 28 additions & 0 deletions chirps/asset/static/asset/pinecone-logo.svg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file removed chirps/asset/static/asset/redis-logo.png
Binary file not shown.
25 changes: 25 additions & 0 deletions chirps/asset/static/asset/redis-logo.svg
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.