Rest subset fields

jsapp/js/components/RESTServices/RESTServicesForm.es6

@@ -51,7 +51,7 @@ export default class RESTServicesForm extends React.Component {
       ],
       authUsername: '',
       authPassword: '',
-      selectedFields: [],
+      subsetFields: [],
       customHeaders: [
         this.getEmptyHeaderRow()
       ]
@@ -68,7 +68,8 @@ export default class RESTServicesForm extends React.Component {
             name: data.name,
             endpoint: data.endpoint,
             isActive: data.active,
-            emailNotification: data.email_notification || true,
+            emailNotification: data.email_notification,
+            subsetFields: data.subset_fields,
             type: data.export_type,
             authLevel: AUTH_OPTIONS[data.auth_level] || null,
             customHeaders: this.headersObjToArr(data.settings.custom_headers)
@@ -151,7 +152,7 @@ export default class RESTServicesForm extends React.Component {
   handleAuthPasswordChange(newPassword) {this.setState({authPassword: newPassword});}
 
   handleActiveChange(isChecked) {this.setState({isActive: isChecked});}
-    
+
   handleEmailNotificationChange(isChecked) {
     this.setState({emailNotification: isChecked});
   }
@@ -186,6 +187,7 @@ export default class RESTServicesForm extends React.Component {
       name: this.state.name,
       endpoint: this.state.endpoint,
       active: this.state.isActive,
+      subset_fields: this.state.subsetFields,
       email_notification: this.state.emailNotification,
       export_type: this.state.type,
       auth_level: authLevel,
@@ -349,25 +351,25 @@ export default class RESTServicesForm extends React.Component {
    * handle fields
    */
 
-  onSelectedFieldsChange(evt) {
-    this.setState({selectedFields: evt});
+  onSubsetFieldsChange(evt) {
+    this.setState({subsetFields: evt});
   }
 
   renderFieldsSelector() {
     const inputProps = {
       placeholder: t('Add field(s)'),
-      id: 'selected-fields-input'
+      id: 'subset-fields-input'
     };
 
     return (
       <bem.FormModal__item>
-        <label htmlFor='selected-fields-input'>
-          {t('Select fields')}
+        <label htmlFor='subset-fields-input'>
+          {t('Select fields subset')}
         </label>
 
         <TagsInput
-          value={this.state.selectedFields}
-          onChange={this.onSelectedFieldsChange.bind(this)}
+          value={this.state.subsetFields}
+          onChange={this.onSubsetFieldsChange.bind(this)}
           inputProps={inputProps}
         />
       </bem.FormModal__item>

jsapp/js/dataInterface.es6

@@ -85,7 +85,7 @@ var dataInterface;
     },
     getHook(uid, hookUid) {
       return $ajax({
-        url: `${rootUrl}/assets/${uid}/hooks/${hookUid}`,
+        url: `${rootUrl}/assets/${uid}/hooks/${hookUid}/`,
         method: 'GET'
       });
     },
@@ -100,7 +100,7 @@ var dataInterface;
     },
     updateExternalService(uid, hookUid, data) {
       return $ajax({
-        url: `${rootUrl}/assets/${uid}/hooks/${hookUid}`,
+        url: `${rootUrl}/assets/${uid}/hooks/${hookUid}/`,
         method: 'PATCH',
         data: JSON.stringify(data),
         dataType: 'json',
@@ -109,7 +109,7 @@ var dataInterface;
     },
     deleteExternalService(uid, hookUid) {
       return $ajax({
-        url: `${rootUrl}/assets/${uid}/hooks/${hookUid}`,
+        url: `${rootUrl}/assets/${uid}/hooks/${hookUid}/`,
         method: 'DELETE'
       });
     },

kobo/apps/hook/migrations/0003_add_subset_fields_to_hook_model.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.contrib.postgres.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hook', '0002_add_email_notifications_field'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='hook',
+            name='subset_fields',
+            field=django.contrib.postgres.fields.ArrayField(default=[], base_field=models.CharField(max_length=500), size=None),
+        ),
+    ]

kobo/apps/hook/models/hook.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from importlib import import_module
 
+from django.contrib.postgres.fields import ArrayField
 from django.db import models
 from django.utils import timezone
 from jsonbfield.fields import JSONField as JSONBField
@@ -42,6 +43,10 @@ class Hook(models.Model):
     date_created = models.DateTimeField(default=timezone.now)
     date_modified = models.DateTimeField(default=timezone.now)
     email_notification = models.BooleanField(default=True)
+    subset_fields = ArrayField(
+        models.CharField(max_length=500, blank=False),
+        default=[],
+    )
 
     class Meta:
         ordering = ["name"]

kobo/apps/hook/models/service_definition_interface.py

@@ -30,12 +30,25 @@ def _get_data(self):
         Retrieves data from deployment backend of the asset.
         """
         try:
-            return self._hook.asset.deployment.get_submission(self._instance_id, self._hook.export_type)
+            submission = self._hook.asset.deployment.get_submission(self._instance_id, self._hook.export_type)
+            return self._parse_data(submission, self._hook.subset_fields)
         except Exception as e:
             logging.error("service_json.ServiceDefinition._get_data - Hook #{} - Data #{} - {}".format(
                 self._hook.uid, self._instance_id, str(e)), exc_info=True)
         return None
 
+    @abstractmethod
+    def _parse_data(self, submission, fields):
+        """
+        Data must be parsed to include only `self._hook.subset_fields` if there are any.
+        :param submission: json|xml
+        :param fields: list
+        :return: mixed: json|xml
+        """
+        if len(fields) > 0:
+            pass
+        return submission
+
     @abstractmethod
     def _prepare_request_kwargs(self):
         """

kobo/apps/hook/serializers/hook.py

@@ -15,7 +15,7 @@ class Meta:
         model = Hook
         fields = ("url", "logs_url", "asset", "uid", "name", "endpoint", "active", "export_type",
                   "auth_level", "success_count", "failed_count", "pending_count", "settings",
-                  "date_modified", "email_notification")
+                  "date_modified", "email_notification", "subset_fields")
 
         read_only_fields = ("asset", "uid", "date_modified", "success_count", "failed_count", "pending_count")
 

kobo/apps/hook/services/service_json.py

@@ -2,13 +2,31 @@
 from __future__ import absolute_import
 
 import json
+import re
 
 from ..models.service_definition_interface import ServiceDefinitionInterface
 
 
 class ServiceDefinition(ServiceDefinitionInterface):
     id = u"json"
 
+    def _parse_data(self, submission, fields):
+        if len(fields) > 0:
+            parsed_submission = {}
+            submission_keys = submission.keys()
+
+            for field_ in fields:
+                pattern = r"^{}$" if "/" in field_ else r"(^|/){}(/|$)"
+                for key_ in submission_keys:
+                    if re.search(pattern.format(field_), key_):
+                        parsed_submission.update({
+                            key_: submission[key_]
+                        })
+
+            return parsed_submission
+
+        return submission
+
     def _prepare_request_kwargs(self):
         return {
             "headers": {"Content-Type": "application/json"},

kobo/apps/hook/services/service_xml.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
+from lxml import etree
 import requests
 
 from ..models.service_definition_interface import ServiceDefinitionInterface
@@ -9,6 +10,100 @@
 class ServiceDefinition(ServiceDefinitionInterface):
     id = u"xml"
 
+    def _parse_data(self, submission, fields):
+        if len(fields) > 0:
+
+            # Build xml to be parsed
+            xml_doc = etree.fromstring(submission)
+            tree = etree.ElementTree(xml_doc)
+            matched_nodes_paths = []
+            root_element = tree.getroot()
+            root_path = tree.getpath(root_element)
+
+            def remove_root_path(path_):
+                return path_.replace(root_path, "")
+
+            def is_group(node_):
+                """
+                Checks whether `node_` has children that are also xml nodes with children.
+                Not text.
+                It lets us assume `node_` is a group.
+
+                :param node_: lxml.etree._Element
+                :return: bool
+                """
+                for nested_node_ in node_.iterchildren():
+                    if nested_node_.iterchildren():
+                        return True
+                return False
+
+            def process_node(node_, matched_nodes_paths_):
+                """
+                Removes node from XML tree if it's not included in subset of fields
+                :param node_: lxml.etree._Element
+                :param matched_nodes_paths_: tuple. Nodes to keep
+                """
+                # Calling process_node(xml, `matched_nodes_path`) will first loop through all children of the root element.
+                # Then for each child, it will loop through its children if any etc...
+                # When all children are processed, it checks whether the node should be removed or not.
+                # Trivial case, it's removed. Otherwise, the parent of the current node is tagged as `do_not_delete`.
+                # It let us know when the parent is processed, it has to be kept because one of its children matches
+                # one of the occurrences of `subset_fields`
+                # For example, with `subset_fields = ['question_2', 'question_3'] and this `xml`:
+                # <root>
+                #   <group>
+                #       <question_1>Value1</question1>
+                #       <question_2>Value2</question1>
+                #   </group>
+                #   <question3>Value3</question3>
+                # </root>
+                #
+                #  - `<question_1>` is processed first and removed.
+                #  - `<question_2>` is processed second and kept. `<group>` is tagged as `do_not_delete`
+                #  - `<group>` is processed, is not part of `subset_field` but it's tagged as `do_not_delete`. It's kept
+                #  - `<question_3>` is processed and kept
+                #
+                # Results:
+                # <root>
+                #   <group>
+                #       <question_2>Value2</question1>
+                #   </group>
+                #   <question3>Value3</question3>
+                # </root>
+
+                for child in node_.getchildren():
+                    process_node(child, matched_nodes_paths_)
+
+                node_path = remove_root_path(tree.getpath(node_))
+
+                # if `node_path` does not match one of the occurrences previously found,
+                # it must be removed.
+                if not node_path.startswith(matched_nodes_paths_) and node_.get("do_not_delete") != "true":
+                    node_.getparent().remove(node_)
+                elif node_path != "":
+                    node_.getparent().set("do_not_delete", "true")
+
+                if node_.attrib.get("do_not_delete"):
+                    del node_.attrib["do_not_delete"]
+
+            # Keep all paths of nodes that match the subset of fields
+            for field_ in fields:
+                for node in tree.iter(field_):
+                    matched_node_path = remove_root_path(tree.getpath(node))
+                    # To make a difference between groups with same beginning of name, we need to add a trailing slash
+                    # for later comparison in `process_node`.
+                    # e.g `subgroup1` and `subgroup11` both start with `subgroup1` but
+                    # `subgroup11/` and # `subgroup1/` don't.
+                    if is_group(node):
+                        matched_node_path += "/"
+                    matched_nodes_paths.append(matched_node_path)
+
+            process_node(root_element, tuple(matched_nodes_paths))
+
+            return etree.tostring(tree, pretty_print=True)
+
+        return submission
+
     def _prepare_request_kwargs(self):
         return {
             "headers": {"Content-Type": "application/xml"},