[Fixes #9821] Time serie selection handled by the advanced metadata p…

…anel (#9840) (#9914)

* [Fixes #9821] Time serie selection handled by the advanced metadata panel

* [Fixes #9821] add new attribute to time series

* [Fixes #9821] add new attribute to time series

* [Fixes #9821] add new attribute to time series

* [Fixes #9821] add new attribute to time series

* [Fixes #9821] add new attribute to time series

* [Fixes #9821] add new attribute to time series

* [Fixes #9821] add new attribute to time series

* [Fixes #9821] add new attribute to time series

* [Fixes #9821] add new attribute to time series

* [Fixes #9821] Time serie selection handled by the advanced metadata panel

* [Fixes #9861]Cannot open metadata detail with thesaurus installed

* [Fixes #9821] Time serie selection handled by the advanced metadata panel

* [Fixes #9821] Time serie selection handled by the advanced metadata panel

* [Fixes #9821] Fix wrong subtype in case of timeserie dataset

* [Fixes #9821] Fix circleci build

Co-authored-by: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com>

geonode/geoserver/helpers.py

@@ -1041,7 +1041,7 @@ def set_attributes_from_geoserver(layer, overwrite=False):
             tb = traceback.format_exc()
             logger.debug(tb)
             attribute_map = []
-    elif layer.subtype in {"vector", "tileStore", "remote", "wmsStore"}:
+    elif layer.subtype in {"vector", "tileStore", "remote", "wmsStore", "vector_time"}:
         typename = layer.alternate if layer.alternate else layer.typename
         dft_url_path = re.sub(r"\/wms\/?$", "/", server_url)
         dft_query = urlencode(
@@ -1548,8 +1548,13 @@ def fetch_gs_resource(instance, values, tries):
             gs_resource.abstract = values.get('abstract', '')
         else:
             values = {}
+
+        _subtype = gs_resource.store.resource_type
+        if gs_resource.metadata and gs_resource.metadata.get('time', False) and gs_resource.metadata.get('time').enabled:
+            _subtype = "vectorTimeSeries"
+
         values.update(dict(store=gs_resource.store.name,
-                           subtype=gs_resource.store.resource_type,
+                           subtype=_subtype,
                            alternate=f"{gs_resource.store.workspace.name}:{gs_resource.name}",
                            title=gs_resource.title or gs_resource.store.name,
                            abstract=gs_resource.abstract or '',

geonode/layers/forms.py

@@ -281,3 +281,40 @@ class LayerStyleUploadForm(forms.Form):
     name = forms.CharField(required=False)
     update = forms.BooleanField(required=False)
     sld = forms.FileField()
+
+
+class DatasetTimeSerieForm(forms.ModelForm):
+
+    def __init__(self, *args, **kwargs):
+        _choises = [(None, '-----')] + [(_a.pk, _a.attribute) for _a in kwargs.get('instance').attributes if _a.attribute_type in ['xsd:dateTime']]
+        self.base_fields.get('attribute').choices = _choises
+        self.base_fields.get('end_attribute').choices = _choises
+        super().__init__(*args, **kwargs)
+
+    class Meta:
+        model = Attribute
+        fields = ('attribute',)
+
+    attribute = forms.ChoiceField(
+            required=False,
+        )
+    end_attribute = forms.ChoiceField(
+            required=False,
+        )
+    presentation = forms.ChoiceField(
+        required=False,
+        choices=[
+            ('LIST', 'List of all the distinct time values'),
+            ('DISCRETE_INTERVAL', 'Intervals defined by the resolution'),
+            ('CONTINUOUS_INTERVAL', 'Continuous Intervals for data that is frequently updated, resolution describes the frequency of updates')
+        ]
+    )
+    precision_value = forms.IntegerField(required=False)
+    precision_step = forms.ChoiceField(required=False, choices=[
+        ('years',) * 2,
+        ('months',) * 2,
+        ('days',) * 2,
+        ('hours',) * 2,
+        ('minutes',) * 2,
+        ('seconds',) * 2
+    ])

geonode/layers/models.py

@@ -182,15 +182,15 @@ class Dataset(ResourceBase):
         null=True)
 
     def is_vector(self):
-        return self.subtype == 'vector'
+        return self.subtype in ['vector', 'vector_time']
 
     @property
     def is_raster(self):
         return self.subtype == 'raster'
 
     @property
     def display_type(self):
-        if self.subtype == "vector":
+        if self.subtype in ["vector", "vector_time"]:
             return "Vector Data"
         elif self.subtype == "raster":
             return "Raster Data"
@@ -264,7 +264,7 @@ def get_base_file(self):
 
         # we need to check, for shapefile, if column names are valid
         list_col = None
-        if self.subtype == 'vector':
+        if self.subtype in ['vector', 'vector_time']:
             valid_shp, wrong_column_name, list_col = check_shp_columnnames(
                 self)
             if wrong_column_name:
@@ -332,7 +332,7 @@ def maps(self):
 
     @property
     def download_url(self):
-        if self.subtype not in ['vector', 'raster']:
+        if self.subtype not in ['vector', 'raster', 'vector_time']:
             logger.error("Download URL is available only for datasets that have been harvested and copied locally")
             return None
         return build_absolute_uri(reverse('dataset_download', args=(self.alternate,)))

geonode/layers/templates/layouts/panels.html

@@ -720,6 +720,38 @@
             </div>
         </div>
     </div>
+    <div class="col-xs-12 col-lg-4" id="settings_time_series">
+      <div class="panel-group">
+          <div class="panel panel-default">
+              <div class="panel-body">
+                <div class="panel panel-default" >
+                  <div class="panel-heading">{% trans "Time series settings" %}     <button type="button" class="btn btn-link" data-toggle="modal" data-target="#exampleModal">
+                    <i class="fa fa-question-circle "></i>
+                  </button></div>
+                  <div class="panel-body">
+                    <div>
+                      <span><label for="dataset_attribute">Attribute</label></span>
+                      {{timeseries_form.attribute}}
+                    </div>
+                    <div>
+                    <span><label for="dataset_end_attribute">End attribute</label></span>
+                      {{timeseries_form.end_attribute}}
+                    </div>
+                    <span><label for="dataset_presentation">Presentation</label></span>
+                      {{timeseries_form.presentation}}
+                      <div id='precision_value'>                      
+                      <span><label for="dataset_precision_value">Precision Value</label></span><br>
+                      {{timeseries_form.precision_value}}<br>
+                      <span><label for="dataset_precision_step">Precision Step</label></span>
+                      {{timeseries_form.precision_step}}
+                    </div>
+                  </div>
+                  </div>
+                </div>
+              </div>
+            </div>
+        </div>
+    </div>
 </div>
 </div>
 
@@ -737,3 +769,34 @@
   });
 </script>
 {% endblock %}
+
+<div class="modal fade" id="exampleModal" tabindex="-1" role="dialog" aria-labelledby="exampleModalLabel" aria-hidden="true">
+  <div class="modal-dialog" role="document">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h5 class="modal-title" id="exampleModalLabel">{% trans "Additional Help" %}</h5>
+        <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+          <span aria-hidden="true">&times;</span>
+        </button>
+      </div>
+      <div class="modal-body">
+        <h4>{% trans "Enabling Time" %}</h4>
+        <p>{% blocktrans %}A dataset can support one or two time attributes. If a single
+                        attribute is used, the dataset is considered to contain data that is valid at single points in time. If two
+                        attributes are used, the second attribute represents the end of a valid period  hence the dataset is considered
+                        to contain data that is valid at certain periods in time.{% endblocktrans %}</p>
+        <h4>{% trans "Selecting an Attribute" %}</h4>
+        <p>{% trans "A time attribute can be" %}:</p>
+        <ul>
+          <li>{% trans "An existing date" %}</li>
+          <li>{% trans "Text that can be converted to a timestamp" %}</li>
+          <li>{% trans "A number representing a year" %}</li>
+        </ul>
+        <p>
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-secondary" data-dismiss="modal">Close</button>
+      </div>
+    </div>
+  </div>
+</div>

geonode/layers/tests.py

@@ -60,7 +60,7 @@
 from geonode.resource.manager import resource_manager
 from geonode.tests.utils import NotificationsTestsHelper
 from geonode.layers.models import Dataset, Style, Attribute
-from geonode.layers.forms import DatasetForm, JSONField, LayerUploadForm
+from geonode.layers.forms import DatasetForm, DatasetTimeSerieForm, JSONField, LayerUploadForm
 from geonode.layers.populate_datasets_data import create_dataset_data
 from geonode.base.models import TopicCategory, License, Region, Link
 from geonode.utils import check_ogc_backend, set_resource_default_links
@@ -1861,6 +1861,7 @@ def setUp(self) -> None:
         self.user = get_user_model().objects.get(username='admin')
         self.dataset = create_single_dataset("my_single_layer", owner=self.user)
         self.sut = DatasetForm
+        self.time_form = DatasetTimeSerieForm
 
     def test_resource_form_is_invalid_extra_metadata_not_json_format(self):
         self.client.login(username="admin", password="admin")
@@ -1915,3 +1916,48 @@ def test_resource_form_is_valid_extra_metadata(self):
             "extra_metadata": '[{"id": 1, "filter_header": "object", "field_name": "object", "field_label": "object", "field_value": "object"}]'
         })
         self.assertTrue(form.is_valid())
+
+    def test_dataset_time_form_should_work(self):
+
+        attr, _ = Attribute.objects.get_or_create(
+            dataset=self.dataset,
+            attribute="field_date",
+            attribute_type="xsd:dateTime"
+        )
+        self.dataset.attribute_set.add(attr)
+        self.dataset.save()
+        form = self.time_form(
+            instance=self.dataset,
+            data={
+                'attribute': self.dataset.attributes.first().id,
+                'end_attribute': '',
+                'presentation': 'DISCRETE_INTERVAL',
+                'precision_value': 12345,
+                'precision_step': 'seconds'
+            }
+        )
+        self.assertTrue(form.is_valid())
+        self.assertDictEqual({}, form.errors)
+
+    def test_dataset_time_form_should_raise_error_if_invalid_payload(self):
+
+        attr, _ = Attribute.objects.get_or_create(
+            dataset=self.dataset,
+            attribute="field_date",
+            attribute_type="xsd:dateTime"
+        )
+        self.dataset.attribute_set.add(attr)
+        self.dataset.save()
+        form = self.time_form(
+            instance=self.dataset,
+            data={
+                'attribute': self.dataset.attributes.first().id,
+                'end_attribute': '',
+                'presentation': 'INVALID_PRESENTATION_VALUE',
+                'precision_value': 12345,
+                'precision_step': 'seconds'
+            }
+        )
+        self.assertFalse(form.is_valid())
+        self.assertTrue('presentation' in form.errors)
+        self.assertEqual("Select a valid choice. INVALID_PRESENTATION_VALUE is not one of the available choices.", form.errors['presentation'][0])

geonode/layers/views.py

@@ -65,6 +65,7 @@
 from geonode.decorators import check_keyword_write_perms
 from geonode.layers.forms import (
     DatasetForm,
+    DatasetTimeSerieForm,
     LayerAttributeForm,
     NewLayerUploadForm)
 from geonode.layers.models import (
@@ -485,6 +486,7 @@ def dataset_metadata(
 
         thumbnail_url = layer.thumbnail_url
         dataset_form = DatasetForm(request.POST, instance=layer, prefix="resource", user=request.user)
+
         if not dataset_form.is_valid():
             logger.error(f"Dataset Metadata form is not valid: {dataset_form.errors}")
             out = {
@@ -542,6 +544,18 @@ def dataset_metadata(
                 json.dumps(out),
                 content_type='application/json',
                 status=400)
+
+        timeseries_form = DatasetTimeSerieForm(request.POST, instance=layer, prefix='timeseries')
+        if not timeseries_form.is_valid():
+            out = {
+                'success': False,
+                'errors': [f"{x}: {y[0].messages[0]}" for x, y in timeseries_form.errors.as_data().items()]
+            }
+            logger.error(f"{out.get('errors')}")
+            return HttpResponse(
+                json.dumps(out),
+                content_type='application/json',
+                status=400)
     else:
         dataset_form = DatasetForm(instance=layer, prefix="resource", user=request.user)
         dataset_form.disable_keywords_widget_for_non_superuser(request.user)
@@ -553,6 +567,37 @@ def dataset_metadata(
             prefix="category_choice_field",
             initial=topic_category.id if topic_category else None)
 
+        gs_layer = gs_catalog.get_layer(name=layer.name)
+        initial = {}
+        if gs_layer is not None and layer.has_time:
+            gs_time_info = gs_layer.resource.metadata.get("time")
+            if gs_time_info.enabled:
+                _attr = layer.attributes.filter(attribute=gs_time_info.attribute).first()
+                initial["attribute"] = _attr.pk if _attr else None
+                if gs_time_info.end_attribute is not None:
+                    end_attr = layer.attributes.filter(attribute=gs_time_info.end_attribute).first()
+                    initial["end_attribute"] = end_attr.pk if end_attr else None
+                initial["presentation"] = gs_time_info.presentation
+                lookup_value = sorted(list(gs_time_info._lookup), key=lambda x: x[1], reverse=True)
+                if gs_time_info.resolution is not None:
+                    res = gs_time_info.resolution // 1000
+                    for el in lookup_value:
+                        if res % el[1] == 0:
+                            initial["precision_value"] = res // el[1]
+                            initial["precision_step"] = el[0]
+                            break
+                else:
+                    initial["precision_value"] = gs_time_info.resolution
+                    initial["precision_step"] = "seconds"
+
+        timeseries_form = DatasetTimeSerieForm(
+            instance=layer,
+            prefix="timeseries",
+            initial=initial
+        )
+        timeseries_form.fields.get('attribute').queryset = layer.attributes.filter(attribute_type__in=['xsd:dateTime'])
+        timeseries_form.fields.get('end_attribute').queryset = layer.attributes.filter(attribute_type__in=['xsd:dateTime'])
+
         # Create THESAURUS widgets
         lang = settings.THESAURUS_DEFAULT_LANG if hasattr(settings, 'THESAURUS_DEFAULT_LANG') else 'en'
         if hasattr(settings, 'THESAURUS') and settings.THESAURUS:
@@ -587,7 +632,7 @@ def dataset_metadata(
                 tkeywords_form.fields[tid].initial = values
 
     if request.method == "POST" and dataset_form.is_valid() and attribute_form.is_valid(
-    ) and category_form.is_valid() and tkeywords_form.is_valid():
+    ) and category_form.is_valid() and tkeywords_form.is_valid() and timeseries_form.is_valid():
         new_poc = dataset_form.cleaned_data['poc']
         new_author = dataset_form.cleaned_data['metadata_author']
 
@@ -696,13 +741,35 @@ def dataset_metadata(
         if any([x in dataset_form.changed_data for x in ['is_approved', 'is_published']]):
             vals['is_approved'] = dataset_form.cleaned_data.get('is_approved', layer.is_approved)
             vals['is_published'] = dataset_form.cleaned_data.get('is_published', layer.is_published)
+
+        layer.has_time = dataset_form.cleaned_data.get('has_time', layer.has_time)
+
+        if timeseries_form.cleaned_data and ('has_time' in dataset_form.changed_data or timeseries_form.changed_data):
+            ts = timeseries_form.cleaned_data
+            end_attr = layer.attributes.get(pk=ts.get("end_attribute")).attribute if ts.get("end_attribute") else None
+            start_attr = layer.attributes.get(pk=ts.get("attribute")).attribute if ts.get("attribute") else None
+            resource_manager.exec(
+                'set_time_info',
+                None,
+                instance=layer,
+                time_info={
+                    "attribute": start_attr,
+                    "end_attribute": end_attr,
+                    "presentation": ts.get('presentation', None),
+                    "precision_value": ts.get('precision_value', None),
+                    "precision_step": ts.get('precision_step', None),
+                    "enabled": dataset_form.cleaned_data.get('has_time', False)
+                }
+            )
+
         resource_manager.update(
             layer.uuid,
             instance=layer,
             notify=True,
             vals=vals,
             extra_metadata=json.loads(dataset_form.cleaned_data['extra_metadata'])
         )
+
         return HttpResponse(json.dumps({'message': message}))
 
     if not AdvancedSecurityWorkflowManager.is_allowed_to_publish(request.user, layer):
@@ -736,6 +803,7 @@ def dataset_metadata(
         "poc_form": poc_form,
         "author_form": author_form,
         "attribute_form": attribute_form,
+        "timeseries_form": timeseries_form,
         "category_form": category_form,
         "tkeywords_form": tkeywords_form,
         "preview": getattr(settings, 'GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY', 'mapstore'),

geonode/security/models.py

@@ -351,7 +351,7 @@ def get_user_perms(self, user):
 
         PERMISSIONS_TO_FETCH = VIEW_PERMISSIONS + DOWNLOAD_PERMISSIONS + ADMIN_PERMISSIONS + SERVICE_PERMISSIONS
         # include explicit permissions appliable to "subtype == 'vector'"
-        if self.subtype == 'vector':
+        if self.subtype in ['vector', 'vector_time']:
             PERMISSIONS_TO_FETCH += DATASET_ADMIN_PERMISSIONS
         elif self.subtype == 'raster':
             PERMISSIONS_TO_FETCH += DATASET_EDIT_STYLE_PERMISSIONS