diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 53befc8..82934f0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,7 @@ v0.8 (Upcoming) * sqlite bug workaround: python ./manage.py shell -c "import django;django.db.connection.cursor().execute('SELECT InitSpatialMetaData(1);')"; * Enhancements: Postgres backend option * Scripts for deleting user data from database +* Vega visualization framework v0.7.1 (14-09-2023) ------------------- diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst index 819f426..4c441e3 100644 --- a/CODE_OF_CONDUCT.rst +++ b/CODE_OF_CONDUCT.rst @@ -2,7 +2,7 @@ Code of Conduct =============== -This code of conduct applies to all online spaces managed by Open Risk. This includes Github, mailing lists, the issue tracker, the Open Risk Commons and/or any other forums created by the Open Risk team towards community usage. +This code of conduct applies to all online spaces managed by Open Risk. This includes Github, mailing lists, issue trackers, the Open Risk Commons and/or any other forums created by the Open Risk team towards community usage. .. warning:: Violations of this code also outside the above mentioned spaces may affect a person's ability to participate within them. @@ -23,4 +23,4 @@ This code of conduct applies to all online spaces managed by Open Risk. This inc * Repeated harassment of others. In general, if someone asks you to stop, then stop. -Original text courtesy of the Django Project \ No newline at end of file +Original text of this Code of Conduct is courtesy of the Django Project \ No newline at end of file diff --git a/equinox/settings.py b/equinox/settings.py index 9762784..3a1b5f2 100644 --- a/equinox/settings.py +++ b/equinox/settings.py @@ -73,6 +73,7 @@ 'policy', 'risk', 'reporting', + 'visualization', 'debug_toolbar', 'behave_django' ] diff --git a/portfolio/views.py b/portfolio/views.py index 1634257..3a29127 100644 --- a/portfolio/views.py +++ b/portfolio/views.py @@ -17,7 +17,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - +from django.contrib.auth.decorators import login_required from django.views.generic import ListView from django.views.generic.base import TemplateView @@ -36,8 +36,8 @@ def get_context_data(self, **kwargs): context = super(ListView, self).get_context_data(**kwargs) return context - class AssetMapView(TemplateView): """Asset Markers Map view.""" template_name = "asset_map.html" + diff --git a/reporting/CHANGELOG.rst b/reporting/CHANGELOG.rst index 573110f..4e82244 100644 --- a/reporting/CHANGELOG.rst +++ b/reporting/CHANGELOG.rst @@ -1,6 +1,10 @@ ChangeLog =========================== +v0.7.0 (21-06-2024) +------------------- +* Vega visualization framework + v0.6.0 (03-06-2023) ------------------- * Model testing framework diff --git a/reporting/models.py b/reporting/models.py index ee1d9cd..6b1ed75 100644 --- a/reporting/models.py +++ b/reporting/models.py @@ -158,6 +158,10 @@ class Meta: verbose_name_plural = "Results" +OBJECTIVE_CHOICE = [(0, 'General Information'), (1, 'Concentration Risk'), (2, 'Origination'), + (3, 'Risk Appetite'), (4, 'Risk Capital'), (5, 'Other')] + + class Visualization(models.Model): """ The Visualization Data object holds the structural Vega / Vega-Lite specification of a visualization @@ -165,15 +169,6 @@ class Visualization(models.Model): Includes reference to user creating the Visualization """ - VISUALIZATION_DATA_CHOICES = [(0, 'Load portfolio data from local JSON files'), - (1, 'Fetch portfolio data via REST API'), - (2, 'Create new portfolio from local JSON configuration'), - (3, 'Fetch portfolio configuration via REST API'), - (4, 'Attached portfolio data in JSON format')] - - OBJECTIVE_CHOICE = [(0, 'Portfolio Information'), (1, 'Concentration Risk'), (2, 'Origination'), - (3, 'Risk Appetite'), (4, 'Risk Capital'), (5, 'Other')] - name = models.CharField(max_length=200, help_text="Assigned name to help manage Visualization collections") user_id = models.ForeignKey(User, on_delete=models.CASCADE, default=1, help_text="The creator of the Visualization") creation_date = models.DateTimeField(auto_now_add=True) @@ -185,9 +180,6 @@ class Visualization(models.Model): description = models.TextField(null=True, blank=True, help_text="A description of the main purpose and " "characteristics of the Visualization") - visualization_data_mode = models.IntegerField(default=1, null=True, blank=True, choices=VISUALIZATION_DATA_CHOICES, - help_text="Select the mode for portfolio data inputs") - visualization_data = models.JSONField(null=True, blank=True, help_text="Container for visualization data") visualization_data_url = models.URLField(null=True, blank=True, help_text="URL for visualization data") results_url = models.CharField(max_length=200, null=True, blank=True, help_text="Where to store the results") diff --git a/reporting/templates/reporting/vega_viz.html b/reporting/templates/reporting/vega_viz.html new file mode 100644 index 0000000..f7f4046 --- /dev/null +++ b/reporting/templates/reporting/vega_viz.html @@ -0,0 +1,71 @@ +{% extends "start/generic.html" %} + +{% load static %} +{% load custom_tags %} +{% load humanize %} + +{% block title %} + {{ title }} Standard Visualization +{% endblock %} + +{% block extrahead %} + {{ block.super }} + + + +{% endblock %} + +{% block extrastyle %} + {{ block.super }} + +{% endblock %} + +{% block messages %} +
+

Declarative Visualization: {{ object.name }} : + {{ object.description }}

+
+{% endblock %} + + +{% block content %} + +
+ +
+
+
+
+
+ +
+ + + + +{% endblock %} + diff --git a/reporting/urls.py b/reporting/urls.py index 5624555..27db6f2 100644 --- a/reporting/urls.py +++ b/reporting/urls.py @@ -18,7 +18,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from django.urls import re_path +from django.urls import re_path, path from reporting import views @@ -52,4 +52,5 @@ re_path(r'^portfolio_aggregates$', views.portfolio_aggregates, name='portfolio_aggregates'), # re_path(r'^result_types$', views.result_types, name='result_types'), re_path(r'^results_view/(?P\d+)$', views.results_view, name='results_view'), + re_path(r'^visualization_vega$', views.visualization_vega, name='visualization_vega'), ] diff --git a/reporting/views.py b/reporting/views.py index 7324e21..d419aae 100644 --- a/reporting/views.py +++ b/reporting/views.py @@ -42,7 +42,7 @@ from portfolio.models import MultiAreaSource from reference.NUTS3Data import NUTS3PointData from reporting.forms import CustomPortfolioAggregatesForm, portfolio_attributes, aggregation_choices -from reporting.models import Calculation, SummaryStatistics, AggregatedStatistics +from reporting.models import Calculation, SummaryStatistics, AggregatedStatistics, Visualization """ @@ -788,3 +788,23 @@ def visualization_sector(request): context.update({'img_list': img_list}) context.update({'dataset': top_level}) return HttpResponse(t.template.render(context)) + + +@login_required(login_url='/login/') +def visualization_vega(request, pk): + """ + + """ + + # get the Visualization object + visualization = Visualization.objects.get(pk=pk) + context = RequestContext(request, {}) + + t = loader.get_template('vega_viz.html') + spec = json.dumps(visualization.vega_specification) + data = json.dumps(visualization.visualization_data) + context.update({'object': visualization}) + context.update({'visualization_data': data}) + context.update({'vega_specification': spec}) + + return HttpResponse(t.template.render(context)) diff --git a/requirements.txt b/requirements.txt index ac19689..9386c52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ django-import-export django-prettyjson django-countries django-extensions +django-jsoneditor django-htmx django-debug-toolbar~=4.3.0 psycopg2 @@ -26,6 +27,6 @@ behave-django pytest~=8.0.1 pytest-bdd coverage -requests~=2.32.0 +requests~=2.31.0 behave~=1.2.6 setuptools~=69.5.1 \ No newline at end of file diff --git a/visualization/admin.py b/visualization/admin.py new file mode 100644 index 0000000..f5f7f5f --- /dev/null +++ b/visualization/admin.py @@ -0,0 +1,33 @@ +from django.contrib import admin +from django.db.models import JSONField +from jsoneditor.forms import JSONEditor + +from visualization.models import VegaSpecification, VegaLiteSpecification + + +class VegaSpecificationAdmin(admin.ModelAdmin): + formfield_overrides = { + JSONField: {'widget': JSONEditor}, + } + # + # Searchable fields + # + search_fields = ['description'] + list_display = ('title', 'width', 'height', 'description',) + save_as = True + view_on_site = False + + +class VegaLiteSpecificationAdmin(admin.ModelAdmin): + formfield_overrides = { + JSONField: {'widget': JSONEditor}, + } + + search_fields = ['description'] + list_display = ('title', 'description',) + save_as = True + view_on_site = False + + +admin.site.register(VegaSpecification, VegaSpecificationAdmin) +admin.site.register(VegaLiteSpecification, VegaLiteSpecificationAdmin) diff --git a/visualization/models.py b/visualization/models.py new file mode 100644 index 0000000..1676a26 --- /dev/null +++ b/visualization/models.py @@ -0,0 +1,256 @@ +from django.contrib.auth.models import User +from django.db.models import JSONField +from django.urls import reverse +from django.db import models + + +def padding_default(): + return {"left": 5, "top": 5, "right": 5, "bottom": 5} + + +CATEGORY_CHOICES = ( + (0, 'line plot'), + (1, 'timeseries'), + (2, 'bar plot'), + (3, 'histogram'), + (4, 'x-y plot'), + (5, 'radial chart'), + (6, 'box plot'), + (7, 'distribution'), + (8, 'contingency chart'), + (9, 'tree diagram'), + (10, 'network diagram'), + (11, 'geographical map'), + (12, 'ternary'), + (13, 'specialty'), + (14, 'customized')) + +MARK_CHOICES = ( + (0, "area"), + (1, "bar"), + (2, "circle"), + (3, "line"), + (4, "point"), + (5, "rect"), + (6, "rule"), + (7, "square"), + (8, "text"), + (9, "tick"), + (10, "geoshape") +) + + +class VegaSpecification(models.Model): + """ + Data object holds normalized vega specification + + The specification allows multiple possible types for each property (e.g. both a Number, an Object and a Signal) We select the most common / simple use case. Alternative use cases must override the model defaults at some point during the View or Page rendering phase + + The basic data model used by Vega is tabular data, similar to a spreadsheet or database table. Individual data sets are assumed to contain a collection of records (or “rows”), which may contain any number of named data attributes (fields, or “columns”). Records are modeled using standard JavaScript objects. + + Principle 1 is that datasets are from a finite list of distinct reference formats that are already implemented as Django models. This does not fix the format into a dataframe because there could be JSONFields. Datasets could have a REST API (alternative channel) + Examples: + * 1 array (timeseries data) / dataseries like (also SDMX) + * 2 tabular (portfolio data) / dataframe like + * 3 graph like (node - links), stored in multiple models + * tree like + + Principle 2 is that the view that renders the visualization compiles a full Vega specification including the data needed Hence the client side javascript is minimal (loading the full specification, bindings etc.), always the same, easy to maintain Changes to the visualization are done on the path to full spec creation: + * changing the dataset source endpoints from the eligible classes -> choices from admin or via user forms + * changing the specification from the vega spec collection -> manual admin exercise or user forms + * structural changes -> require python view modifications + + Principle 3 is that a visualization is a type of post-processing and data transformation that is applied to a primary dataset. Additional datasets might be required. Such datasets might be generic (mashups with other dataseries), or specific (geographical data) Hence the core Vega visualization specification is a view that renders a given model with all eligible visualizations + + Principle 4 is that the overall data and specification flow architecture should make sense and ber re-usable for non-vega visualizations + * Internal (matplotlib, pygal) + * Partially custom js (d3) + * Fully custom js (vol gauge etc) + + Principle 5 is that the URL patterns follow a well-structured REST API + * The pattern has intuitive placeholders for object selection (viz & data) + + Principle 6 is that the rendered graphic is a well-behaved component + * The Rendered object (svg) is self-documenting as a static object + * NO DYNAMIC CHANGES of DATA or VIZ (Ajax calls) beyond what is achievable via signals + * Can be replicated on the page as multiple objects (expressing an option list or parameter range) + + """ + + schema = models.URLField(default="https://vega.github.io/schema/vega/v5.json", + help_text=" The URL for the Vega schema.") + + # category choices are preselected + # sub-category choices are free form text (for flexibility) + category = models.IntegerField(default=0, choices=CATEGORY_CHOICES, + help_text="The high level category to which the visualization belongs") + + subcategory = models.CharField(max_length=200, null=True, blank=True, + help_text="Subcategory to which the visualization belongs") + + description = models.CharField(max_length=500, default="Specification Description", + help_text="A text description of the visualization. In versions ≥ 5.10, " + "the description determines the aria-label attribute for the " + "container element of a Vega view.") + width = models.IntegerField(default=500, help_text="The width of the visualization in pixels.") + height = models.IntegerField(default=200, help_text="The height of the visualization in pixels") + padding = JSONField(default=padding_default, null=True, blank=True, + help_text="The padding in pixels to add around the visualization." + "Number and Schema Types") + + AUTOSIZE_CHOICES = ((0, 'pad'), (1, 'fit'), (2, 'fit-x'), (3, 'fit-y'), (4, 'none')) + autosize = models.IntegerField(default=0, choices=AUTOSIZE_CHOICES, + help_text="Sets how the visualization size should be determined") + + config = JSONField(default=dict, null=True, blank=True, + help_text="Configuration settings with default values for marks, axes, and legends.") + + signals = JSONField(default=dict, null=True, blank=True, + help_text="Signals are dynamic variables that parameterize a visualization.") + + scales = JSONField(default=dict, null=True, blank=True, + help_text="Scales map data values (numbers, dates, categories, etc) to visual values " + "(pixels, colors, sizes).") + + projections = JSONField(default=dict, null=True, blank=True, + help_text="Cartographic projections map (longitude, latitude) pairs to projected " + "(x, y) coordinates.") + + axes = JSONField(default=dict, null=True, blank=True, + help_text="Coordinate axes visualize spatial scale mappings.") + + legends = JSONField(default=dict, null=True, blank=True, + help_text="Legends visualize scale mappings for visual values such as color, shape and size.") + + title = models.CharField(max_length=200, default="Visualization", + help_text="Title text to describe a visualization.") + + marks = JSONField(default=dict, null=True, blank=True, + help_text="Graphical marks visually encode data using geometric primitives such as rectangles, " + "lines, and plotting symbols.") + + encode = JSONField(default=dict, null=True, blank=True, + help_text="Encoding directives for the visual properties of the top-level group mark " + "representing a chart’s data rectangle. For example, this can be used to set a " + "background fill color for the plotting area, rather than the entire view.") + + usermeta = JSONField(default=dict, null=True, blank=True, + help_text="Optional metadata that will be ignored by the Vega parser.") + + extradata = JSONField(default=dict, null=True, blank=True, + help_text="Additional data that will be added to the vega data specification.") + + transform = JSONField(default=dict, null=True, blank=True, + help_text="Transform that will be applied to the primary data object.") + + intermediatedata = JSONField(default=dict, null=True, blank=True, + help_text="Transform generating intermediate data from the primary data object.") + + def __str__(self): + # Construct a unique name on the basis of title and pk + return self.title + " (Object: " + str(self.pk) + ")" + + def get_absolute_url(self): + return reverse('visualization:vega_specification_view', kwargs={'pk': self.pk}) + + class Meta: + verbose_name = "Vega Specification" + verbose_name_plural = "Vega Specifications" + + +class VegaLiteSpecification(models.Model): + """ + Data object holds normalized vega lite specification + + """ + + # Category is for internal use + # category choices are preselected + # sub-category choices are free form text (for flexibility) + + category = models.IntegerField(default=0, choices=CATEGORY_CHOICES, + help_text="The high level category to which the visualization belongs") + + subcategory = models.CharField(max_length=200, null=True, blank=True, + help_text="Subcategory to which the visualization belongs") + + # + # TOP LEVEL SPECIFICATION + # + + schema = models.URLField(default="https://vega.github.io/schema/vega-lite/v4.json", + help_text=" The URL for the Vega Lite schema.") + + title = models.CharField(max_length=200, default="Visualization", + help_text="Title text to describe a visualization.") + + description = models.CharField(max_length=500, default="Specification Description", + help_text="A text description of the visualization.") + + width = models.IntegerField(default=500, help_text="The width of the visualization in pixels.") + height = models.IntegerField(default=200, help_text="The height of the visualization in pixels") + + mark = models.IntegerField(default=0, choices=MARK_CHOICES, help_text="Choice of mark to use in the visualization") + + encoding = JSONField(default=dict, null=True, blank=True, + help_text="Encoding directives for the visual properties of the top-level group mark " + "representing a chart’s data rectangle. For example, this can be used to set a " + "background fill color for the plotting area, rather than the entire view.") + + transform = JSONField(default=dict, null=True, blank=True, + help_text="Transform that will be applied to the primary data object.") + + # padding = JSONField(default=padding_default, null=True, blank=True, + # help_text="The padding in pixels to add around the visualization." + # "Number and Schema Types") + # + # AUTOSIZE_CHOICES = ((0, 'pad'), (1, 'fit'), (2, 'fit-x'), (3, 'fit-y'), (4, 'none')) + # autosize = models.IntegerField(default=0, choices=AUTOSIZE_CHOICES, + # help_text="Sets how the visualization size should be determined") + # + config = JSONField(default=dict, null=True, blank=True, + help_text="Configuration settings with default values for marks, axes, and legends.") + + # signals = JSONField(default=dict, null=True, blank=True, + # help_text="Signals are dynamic variables that parameterize a visualization.") + # + # scales = JSONField(default=dict, null=True, blank=True, + # help_text="Scales map data values (numbers, dates, categories, etc) to visual values " + # "(pixels, colors, sizes).") + # + # projections = JSONField(default=dict, null=True, blank=True, + # help_text="Cartographic projections map (longitude, latitude) pairs to projected " + # "(x, y) coordinates.") + # + # axes = JSONField(default=dict, null=True, blank=True, + # help_text="Coordinate axes visualize spatial scale mappings.") + # + # legends = JSONField(default=dict, null=True, blank=True, + # help_text="Legends visualize scale mappings for visual values such as color, shape and size.") + # + # + # marks = JSONField(default=dict, null=True, blank=True, + # help_text="Graphical marks visually encode data using geometric primitives such as rectangles, " + # "lines, and plotting symbols.") + # + # usermeta = JSONField(default=dict, null=True, blank=True, + # help_text="Optional metadata that will be ignored by the Vega parser.") + # + # extradata = JSONField(default=dict, null=True, blank=True, + # help_text="Additional data that will be added to the vega data specification.") + # + + # + # intermediatedata = JSONField(default=dict, null=True, blank=True, + # help_text="Transform generating intermediate data from the primary data object.") + + def __str__(self): + # Construct a unique name on the basis of title and pk + return self.title + " (Object: " + str(self.pk) + ")" + + def get_absolute_url(self): + return reverse('visualization:vega_lite_specification_view', kwargs={'pk': self.pk}) + + class Meta: + verbose_name = "VegaLite Specification" + verbose_name_plural = "VegaLite Specifications"