diff --git a/jsapp/js/components/RESTServices/RESTServicesForm.es6 b/jsapp/js/components/RESTServices/RESTServicesForm.es6 index ccd0fc2613..c1ff457f03 100644 --- a/jsapp/js/components/RESTServices/RESTServicesForm.es6 +++ b/jsapp/js/components/RESTServices/RESTServicesForm.es6 @@ -51,7 +51,7 @@ export default class RESTServicesForm extends React.Component { ], authUsername: '', authPassword: '', - selectedFields: [], + subsetFields: [], customHeaders: [ this.getEmptyHeaderRow() ] @@ -68,7 +68,8 @@ export default class RESTServicesForm extends React.Component { name: data.name, endpoint: data.endpoint, isActive: data.active, - emailNotification: data.email_notification || true, + emailNotification: data.email_notification, + subsetFields: data.subset_fields, type: data.export_type, authLevel: AUTH_OPTIONS[data.auth_level] || null, customHeaders: this.headersObjToArr(data.settings.custom_headers) @@ -151,7 +152,7 @@ export default class RESTServicesForm extends React.Component { handleAuthPasswordChange(newPassword) {this.setState({authPassword: newPassword});} handleActiveChange(isChecked) {this.setState({isActive: isChecked});} - + handleEmailNotificationChange(isChecked) { this.setState({emailNotification: isChecked}); } @@ -186,6 +187,7 @@ export default class RESTServicesForm extends React.Component { name: this.state.name, endpoint: this.state.endpoint, active: this.state.isActive, + subset_fields: this.state.subsetFields, email_notification: this.state.emailNotification, export_type: this.state.type, auth_level: authLevel, @@ -349,25 +351,25 @@ export default class RESTServicesForm extends React.Component { * handle fields */ - onSelectedFieldsChange(evt) { - this.setState({selectedFields: evt}); + onSubsetFieldsChange(evt) { + this.setState({subsetFields: evt}); } renderFieldsSelector() { const inputProps = { placeholder: t('Add field(s)'), - id: 'selected-fields-input' + id: 'subset-fields-input' }; return ( - diff --git a/jsapp/js/dataInterface.es6 b/jsapp/js/dataInterface.es6 index cb94613e4a..b1727717b9 100644 --- a/jsapp/js/dataInterface.es6 +++ b/jsapp/js/dataInterface.es6 @@ -85,7 +85,7 @@ var dataInterface; }, getHook(uid, hookUid) { return $ajax({ - url: `${rootUrl}/assets/${uid}/hooks/${hookUid}`, + url: `${rootUrl}/assets/${uid}/hooks/${hookUid}/`, method: 'GET' }); }, @@ -100,7 +100,7 @@ var dataInterface; }, updateExternalService(uid, hookUid, data) { return $ajax({ - url: `${rootUrl}/assets/${uid}/hooks/${hookUid}`, + url: `${rootUrl}/assets/${uid}/hooks/${hookUid}/`, method: 'PATCH', data: JSON.stringify(data), dataType: 'json', @@ -109,7 +109,7 @@ var dataInterface; }, deleteExternalService(uid, hookUid) { return $ajax({ - url: `${rootUrl}/assets/${uid}/hooks/${hookUid}`, + url: `${rootUrl}/assets/${uid}/hooks/${hookUid}/`, method: 'DELETE' }); }, diff --git a/kobo/apps/hook/migrations/0003_add_subset_fields_to_hook_model.py b/kobo/apps/hook/migrations/0003_add_subset_fields_to_hook_model.py new file mode 100644 index 0000000000..a40d162aa6 --- /dev/null +++ b/kobo/apps/hook/migrations/0003_add_subset_fields_to_hook_model.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import django.contrib.postgres.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('hook', '0002_add_email_notifications_field'), + ] + + operations = [ + migrations.AddField( + model_name='hook', + name='subset_fields', + field=django.contrib.postgres.fields.ArrayField(default=[], base_field=models.CharField(max_length=500), size=None), + ), + ] diff --git a/kobo/apps/hook/models/hook.py b/kobo/apps/hook/models/hook.py index dabc44f8fa..3377d49131 100644 --- a/kobo/apps/hook/models/hook.py +++ b/kobo/apps/hook/models/hook.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from importlib import import_module +from django.contrib.postgres.fields import ArrayField from django.db import models from django.utils import timezone from jsonbfield.fields import JSONField as JSONBField @@ -42,6 +43,10 @@ class Hook(models.Model): date_created = models.DateTimeField(default=timezone.now) date_modified = models.DateTimeField(default=timezone.now) email_notification = models.BooleanField(default=True) + subset_fields = ArrayField( + models.CharField(max_length=500, blank=False), + default=[], + ) class Meta: ordering = ["name"] diff --git a/kobo/apps/hook/models/service_definition_interface.py b/kobo/apps/hook/models/service_definition_interface.py index b0d111a29b..23434a4f6f 100644 --- a/kobo/apps/hook/models/service_definition_interface.py +++ b/kobo/apps/hook/models/service_definition_interface.py @@ -30,12 +30,25 @@ def _get_data(self): Retrieves data from deployment backend of the asset. """ try: - return self._hook.asset.deployment.get_submission(self._instance_id, self._hook.export_type) + submission = self._hook.asset.deployment.get_submission(self._instance_id, self._hook.export_type) + return self._parse_data(submission, self._hook.subset_fields) except Exception as e: logging.error("service_json.ServiceDefinition._get_data - Hook #{} - Data #{} - {}".format( self._hook.uid, self._instance_id, str(e)), exc_info=True) return None + @abstractmethod + def _parse_data(self, submission, fields): + """ + Data must be parsed to include only `self._hook.subset_fields` if there are any. + :param submission: json|xml + :param fields: list + :return: mixed: json|xml + """ + if len(fields) > 0: + pass + return submission + @abstractmethod def _prepare_request_kwargs(self): """ diff --git a/kobo/apps/hook/serializers/hook.py b/kobo/apps/hook/serializers/hook.py index 8456bbf5de..e68a973902 100644 --- a/kobo/apps/hook/serializers/hook.py +++ b/kobo/apps/hook/serializers/hook.py @@ -15,7 +15,7 @@ class Meta: model = Hook fields = ("url", "logs_url", "asset", "uid", "name", "endpoint", "active", "export_type", "auth_level", "success_count", "failed_count", "pending_count", "settings", - "date_modified", "email_notification") + "date_modified", "email_notification", "subset_fields") read_only_fields = ("asset", "uid", "date_modified", "success_count", "failed_count", "pending_count") diff --git a/kobo/apps/hook/services/service_json.py b/kobo/apps/hook/services/service_json.py index c3add9756d..c873d5dd39 100644 --- a/kobo/apps/hook/services/service_json.py +++ b/kobo/apps/hook/services/service_json.py @@ -2,6 +2,7 @@ from __future__ import absolute_import import json +import re from ..models.service_definition_interface import ServiceDefinitionInterface @@ -9,6 +10,23 @@ class ServiceDefinition(ServiceDefinitionInterface): id = u"json" + def _parse_data(self, submission, fields): + if len(fields) > 0: + parsed_submission = {} + submission_keys = submission.keys() + + for field_ in fields: + pattern = r"^{}$" if "/" in field_ else r"(^|/){}(/|$)" + for key_ in submission_keys: + if re.search(pattern.format(field_), key_): + parsed_submission.update({ + key_: submission[key_] + }) + + return parsed_submission + + return submission + def _prepare_request_kwargs(self): return { "headers": {"Content-Type": "application/json"}, diff --git a/kobo/apps/hook/services/service_xml.py b/kobo/apps/hook/services/service_xml.py index e5ecc76ffd..59d34a6b16 100644 --- a/kobo/apps/hook/services/service_xml.py +++ b/kobo/apps/hook/services/service_xml.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +from lxml import etree import requests from ..models.service_definition_interface import ServiceDefinitionInterface @@ -9,6 +10,100 @@ class ServiceDefinition(ServiceDefinitionInterface): id = u"xml" + def _parse_data(self, submission, fields): + if len(fields) > 0: + + # Build xml to be parsed + xml_doc = etree.fromstring(submission) + tree = etree.ElementTree(xml_doc) + matched_nodes_paths = [] + root_element = tree.getroot() + root_path = tree.getpath(root_element) + + def remove_root_path(path_): + return path_.replace(root_path, "") + + def is_group(node_): + """ + Checks whether `node_` has children that are also xml nodes with children. + Not text. + It lets us assume `node_` is a group. + + :param node_: lxml.etree._Element + :return: bool + """ + for nested_node_ in node_.iterchildren(): + if nested_node_.iterchildren(): + return True + return False + + def process_node(node_, matched_nodes_paths_): + """ + Removes node from XML tree if it's not included in subset of fields + :param node_: lxml.etree._Element + :param matched_nodes_paths_: tuple. Nodes to keep + """ + # Calling process_node(xml, `matched_nodes_path`) will first loop through all children of the root element. + # Then for each child, it will loop through its children if any etc... + # When all children are processed, it checks whether the node should be removed or not. + # Trivial case, it's removed. Otherwise, the parent of the current node is tagged as `do_not_delete`. + # It let us know when the parent is processed, it has to be kept because one of its children matches + # one of the occurrences of `subset_fields` + # For example, with `subset_fields = ['question_2', 'question_3'] and this `xml`: + # + # + # Value1 + # Value2 + # + # Value3 + # + # + # - `` is processed first and removed. + # - `` is processed second and kept. `` is tagged as `do_not_delete` + # - `` is processed, is not part of `subset_field` but it's tagged as `do_not_delete`. It's kept + # - `` is processed and kept + # + # Results: + # + # + # Value2 + # + # Value3 + # + + for child in node_.getchildren(): + process_node(child, matched_nodes_paths_) + + node_path = remove_root_path(tree.getpath(node_)) + + # if `node_path` does not match one of the occurrences previously found, + # it must be removed. + if not node_path.startswith(matched_nodes_paths_) and node_.get("do_not_delete") != "true": + node_.getparent().remove(node_) + elif node_path != "": + node_.getparent().set("do_not_delete", "true") + + if node_.attrib.get("do_not_delete"): + del node_.attrib["do_not_delete"] + + # Keep all paths of nodes that match the subset of fields + for field_ in fields: + for node in tree.iter(field_): + matched_node_path = remove_root_path(tree.getpath(node)) + # To make a difference between groups with same beginning of name, we need to add a trailing slash + # for later comparison in `process_node`. + # e.g `subgroup1` and `subgroup11` both start with `subgroup1` but + # `subgroup11/` and # `subgroup1/` don't. + if is_group(node): + matched_node_path += "/" + matched_nodes_paths.append(matched_node_path) + + process_node(root_element, tuple(matched_nodes_paths)) + + return etree.tostring(tree, pretty_print=True) + + return submission + def _prepare_request_kwargs(self): return { "headers": {"Content-Type": "application/xml"}, diff --git a/kobo/apps/hook/tests/hook_test_case.py b/kobo/apps/hook/tests/hook_test_case.py index 454394bb34..03f18a529a 100644 --- a/kobo/apps/hook/tests/hook_test_case.py +++ b/kobo/apps/hook/tests/hook_test_case.py @@ -3,18 +3,16 @@ import json -from django.conf import settings -from django.core.urlresolvers import reverse import responses +from django.conf import settings from rest_framework import status +from django.core.urlresolvers import reverse - - - -from ..constants import HOOK_LOG_FAILED from ..models import HookLog, Hook -from kpi.constants import INSTANCE_FORMAT_TYPE_JSON +from ..constants import HOOK_LOG_FAILED +from kpi.exceptions import BadFormatException from kpi.tests.kpi_test_case import KpiTestCase +from kpi.constants import INSTANCE_FORMAT_TYPE_JSON, INSTANCE_FORMAT_TYPE_XML class HookTestCase(KpiTestCase): @@ -23,24 +21,44 @@ def setUp(self): self.client.login(username="someuser", password="someuser") self.asset = self.create_asset( "some_asset", - content=json.dumps({"survey": [{"type": "text", "name": "q1"}]}), + content=json.dumps({"survey": [ + {"type": "text", "name": "q1"}, + {"type": "begin_group", "name": "group1"}, + {"type": "text", "name": "q2"}, + {"type": "text", "name": "q3"}, + {"type": "end_group"}, + {"type": "begin_group", "name": "group2"}, + {"type": "begin_group", "name": "subgroup1"}, + {"type": "text", "name": "q4"}, + {"type": "text", "name": "q5"}, + {"type": "text", "name": "q6"}, + {"type": "end_group"}, + {"type": "end_group"}, + ]}), format="json") self.asset.deploy(backend='mock', active=True) self.asset.save() self.hook = Hook() + self._submission_pk = 1 - v_uid = self.asset.latest_deployed_version.uid - submission = { - "__version__": v_uid, - "q1": u"¿Qué tal?", - "id": 1 - } - self.asset.deployment._mock_submission(submission) - self.asset.save(create_version=False) settings.CELERY_TASK_ALWAYS_EAGER = True def _create_hook(self, return_response_only=False, **kwargs): - url = reverse("hook-list", kwargs={"parent_lookup_asset": self.asset.uid}) + + format_type = kwargs.get("format_type", INSTANCE_FORMAT_TYPE_JSON) + + if format_type == INSTANCE_FORMAT_TYPE_JSON: + self.__prepare_json_submission() + _asset = self.asset + elif format_type == INSTANCE_FORMAT_TYPE_XML: + self.__prepare_xml_submission() + _asset = self.asset_xml + else: + raise BadFormatException( + "The format {} is not supported".format(format_type) + ) + + url = reverse("hook-list", kwargs={"parent_lookup_asset": _asset.uid}) data = { "name": kwargs.get("name", "some external service with token"), "endpoint": kwargs.get("endpoint", "http://external.service.local/"), @@ -48,22 +66,26 @@ def _create_hook(self, return_response_only=False, **kwargs): "custom_headers": { "X-Token": "1234abcd" } - }) + }), + "export_type": format_type, + "active": kwargs.get("active", True), + "subset_fields": kwargs.get("subset_fields", []) } - response = self.client.post(url, data, format=INSTANCE_FORMAT_TYPE_JSON) + response = self.client.post(url, data, format='json') if return_response_only: return response else: self.assertEqual(response.status_code, status.HTTP_201_CREATED, msg=response.data) - hook = self.asset.hooks.last() + hook = _asset.hooks.last() self.assertTrue(hook.active) return hook def _send_and_fail(self): """ + The public method which calls this method needs to be decorated by + `@responses.activate` - The public method which calls this method, needs to be decorated by `@responses.activate` :return: dict """ self.hook = self._create_hook() @@ -93,7 +115,7 @@ def _send_and_fail(self): "parent_lookup_hook": self.hook.uid }) - response = self.client.get(url, format=INSTANCE_FORMAT_TYPE_JSON) + response = self.client.get(url) first_hooklog_response = response.data.get("results")[0] # Result should match first try @@ -105,3 +127,49 @@ def _send_and_fail(self): first_hooklog.change_status(HOOK_LOG_FAILED) return first_hooklog_response + + def __prepare_json_submission(self): + v_uid = self.asset.latest_deployed_version.uid + submission = { + "__version__": v_uid, + "q1": u"¿Qué tal?", + "group1/q2": u"¿Cómo está en el grupo uno la primera vez?", + "group1/q3": u"¿Cómo está en el grupo uno la segunda vez?", + "group2/subgroup1/q4": u"¿Cómo está en el subgrupo uno la primera vez?", + "group2/subgroup1/q5": u"¿Cómo está en el subgrupo uno la segunda vez?", + "group2/subgroup1/q6": u"¿Cómo está en el subgrupo uno la tercera vez?", + "group2/subgroup11/q1": u"¿Cómo está en el subgrupo once?", + "id": self._submission_pk + } + self.__inject_submission(self.asset, submission) + + def __prepare_xml_submission(self): + v_uid = self.asset_xml.latest_deployed_version.uid + submission = ("<{asset_uid}>" + " <__version__>{v_uid}" + " ¿Qué tal?" + " " + " ¿Cómo está en el grupo uno la primera vez?" + " ¿Cómo está en el grupo uno la segunda vez?" + " " + " " + " " + " ¿Cómo está en el subgrupo uno la primera vez?" + " ¿Cómo está en el subgrupo uno la segunda vez?" + " ¿Cómo está en el subgrupo uno la tercera vez?" + " " + " " + " ¿Cómo está en el subgrupo once?" + " " + " " + " {id}" + "").format( + asset_uid=self.asset_xml.uid, + v_uid=v_uid, + id=self._submission_pk + ) + self.__inject_submission(self.asset_xml, submission) + + def __inject_submission(self, asset, submission): + self._submission_pk += 1 + asset.deployment.mock_submissions([submission]) diff --git a/kobo/apps/hook/tests/test_parser.py b/kobo/apps/hook/tests/test_parser.py new file mode 100644 index 0000000000..84d0283e72 --- /dev/null +++ b/kobo/apps/hook/tests/test_parser.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +from lxml import etree +import json +import re + +from .hook_test_case import HookTestCase +from kpi.constants import INSTANCE_FORMAT_TYPE_XML + + +class ParserTestCase(HookTestCase): + + def test_json_parser(self): + hook = self._create_hook(subset_fields=["id", "subgroup1", "q3"]) + + ServiceDefinition = hook.get_service_definition() + submissions = hook.asset.deployment.get_submissions() + uuid = submissions[0].get("id") + service_definition = ServiceDefinition(hook, uuid) + expected_data = { + "group1/q3": u"¿Cómo está en el grupo uno la segunda vez?", + "group2/subgroup1/q4": u"¿Cómo está en el subgrupo uno la primera vez?", + "group2/subgroup1/q5": u"¿Cómo está en el subgrupo uno la segunda vez?", + "group2/subgroup1/q6": u"¿Cómo está en el subgrupo uno la tercera vez?", + "id": 1 + } + self.assertEquals(service_definition._get_data(), expected_data) + + def test_xml_parser(self): + self.asset_xml = self.create_asset( + "some_asset_with_xml_submissions", + content=json.dumps(self.asset.content), + format="json") + self.asset_xml.deploy(backend='mock', active=True) + self.asset_xml.save() + + hook = self._create_hook(subset_fields=["id", "subgroup1", "q3"], format_type=INSTANCE_FORMAT_TYPE_XML) + + ServiceDefinition = hook.get_service_definition() + submissions = hook.asset.deployment.get_submissions(format_type=INSTANCE_FORMAT_TYPE_XML) + xml_doc = etree.fromstring(submissions[0]) + tree = etree.ElementTree(xml_doc) + uuid = tree.find("id").text + + service_definition = ServiceDefinition(hook, uuid) + expected_etree = etree.fromstring(("<{asset_uid}>" + " " + " ¿Cómo está en el grupo uno la segunda vez?" + " " + " " + " " + " ¿Cómo está en el subgrupo uno la primera vez?" + " ¿Cómo está en el subgrupo uno la segunda vez?" + " ¿Cómo está en el subgrupo uno la tercera vez?" + " " + " " + " {id}" + "").format( + asset_uid=self.asset_xml.uid, + id=uuid) + ) + expected_xml = etree.tostring(expected_etree, pretty_print=True) + + def remove_whitespace(str_): + return re.sub(r">\s+<", "><", str_) + + self.assertEquals(remove_whitespace(service_definition._get_data()), + remove_whitespace(expected_xml)) diff --git a/kobo/apps/hook/views/hook.py b/kobo/apps/hook/views/hook.py index 2ddf1850ca..c8a41ce3f0 100644 --- a/kobo/apps/hook/views/hook.py +++ b/kobo/apps/hook/views/hook.py @@ -71,7 +71,9 @@ class HookViewSet(NestedViewSetMixin, viewsets.ModelViewSet): > "name": {string}, > "endpoint": {string}, > "active": {boolean}, + > "email_notification": {boolean}, > "export_type": {string}, + > "subset_fields": [{string}], > "auth_level": {string}, > "settings": { > "username": {string}, @@ -93,11 +95,13 @@ class HookViewSet(NestedViewSetMixin, viewsets.ModelViewSet): 1. `json` (_default_) 2. `xml` + * `email_notification` is a boolean. If true, User will be notified when request to remote server has failed. * `auth_level` must be one these values: 1. `no_auth` (_default_) 2. `basic_auth` + * `subset_fields` is the list of fields of the form definition. Only these fields should be present in data sent to remote server * `settings`.`custom_headers` is dictionary of `custom header`: `value` For example: @@ -129,11 +133,18 @@ class HookViewSet(NestedViewSetMixin, viewsets.ModelViewSet): > > curl -X DELETE https://[kpi-url]/assets/a9PkXcgVgaDXuwayVeAuY5/hooks/hfgha2nxBdoTVcwohdYNzb - #### Retries all failed attempts Not implemented yet + #### Retries all failed attempts 
     PATCH /assets/{asset_uid}/hooks/{hook_uid}/retry/
+ **This call is asynchronous. Job is sent to Celery to be run in background** + + > Example + > + > curl -X PATCH https://[kpi-url]/assets/a9PkXcgVgaDXuwayVeAuY5/hooks/hfgha2nxBdoTVcwohdYNzb/retry/ + + It returns all logs `uid`s that are being retried. ### CURRENT ENDPOINT """ diff --git a/kpi/deployment_backends/base_backend.py b/kpi/deployment_backends/base_backend.py index 0ab4dcedcc..11b6f93165 100644 --- a/kpi/deployment_backends/base_backend.py +++ b/kpi/deployment_backends/base_backend.py @@ -3,6 +3,9 @@ class BaseDeploymentBackend(object): + + # TODO. Stop using protected property `_deployment_data`. + def __init__(self, asset): self.asset = asset diff --git a/kpi/deployment_backends/kobocat_backend.py b/kpi/deployment_backends/kobocat_backend.py index 9bb89ffbf9..488c087ba7 100644 --- a/kpi/deployment_backends/kobocat_backend.py +++ b/kpi/deployment_backends/kobocat_backend.py @@ -18,10 +18,10 @@ from rest_framework.request import Request from rest_framework.authtoken.models import Token +from ..exceptions import BadFormatException from .base_backend import BaseDeploymentBackend from .kc_access.utils import instance_count, last_submission_time from .kc_access.shadow_models import _models -from ..exceptions import BadFormatException from kpi.constants import INSTANCE_FORMAT_TYPE_JSON, INSTANCE_FORMAT_TYPE_XML from kpi.utils.mongo_helper import MongoDecodingHelper from kpi.utils.log import logging @@ -489,14 +489,14 @@ def get_submissions(self, format_type=INSTANCE_FORMAT_TYPE_JSON, instances_ids=[ :return: list: mixed """ submissions = [] - getter = getattr(self, "_{}__get_submissions_in_{}".format( - self.__class__.__name__, - format_type)) - try: - submissions = getter(instances_ids) - except Exception as e: - logging.error("KobocatDeploymentBackend.get_submissions - {}".format(str(e))) - + if format_type == INSTANCE_FORMAT_TYPE_JSON: + submissions = self.__get_submissions_in_json(instances_ids) + elif format_type == INSTANCE_FORMAT_TYPE_XML: + submissions = self.__get_submissions_in_xml(instances_ids) + else: + raise BadFormatException( + "The format {} is not supported".format(format_type) + ) return submissions def get_submission(self, pk, format_type=INSTANCE_FORMAT_TYPE_JSON): @@ -553,4 +553,6 @@ def __get_submissions_in_xml(self, instances_ids=[]): if len(instances_ids) > 0: queryset = queryset.filter(id__in=instances_ids) - return (lazy_instance.xml for lazy_instance in queryset) \ No newline at end of file + queryset = queryset.order_by("id") + + return (lazy_instance.xml for lazy_instance in queryset) diff --git a/kpi/deployment_backends/mock_backend.py b/kpi/deployment_backends/mock_backend.py index 1524acb1d4..2c6e7cc72b 100644 --- a/kpi/deployment_backends/mock_backend.py +++ b/kpi/deployment_backends/mock_backend.py @@ -1,9 +1,9 @@ #!/usr/bin/python # -*- coding: utf-8 -*- +import re from base_backend import BaseDeploymentBackend from kpi.constants import INSTANCE_FORMAT_TYPE_JSON, INSTANCE_FORMAT_TYPE_XML -from kpi.exceptions import BadFormatException class MockDeploymentBackend(BaseDeploymentBackend): @@ -11,6 +11,8 @@ class MockDeploymentBackend(BaseDeploymentBackend): only used for unit testing and interface testing. defines the interface for a deployment backend. + + # TODO. Stop using protected property `_deployment_data`. ''' def connect(self, active=False): self.store_data({ @@ -52,32 +54,53 @@ def _submission_count(self): return len(submissions) def _mock_submission(self, submission): + """ + @TODO may be useless because of mock_submissions. Remove if it's not used anymore anywhere else. + :param submission: + """ submissions = self.asset._deployment_data.get('submissions', []) submissions.append(submission) self.store_data({ 'submissions': submissions, }) + def mock_submissions(self, submissions): + """ + Insert dummy submissions into `asset._deployment_data` + :param submissions: list + """ + self.store_data({"submissions": submissions}) + self.asset.save(create_version=False) + def get_submissions(self, format_type=INSTANCE_FORMAT_TYPE_JSON, instances_ids=[]): """ Returns a list of json representation of instances. - :param format: str. xml or json + :param format_type: str. xml or json :param instances_ids: list. Ids of instances to retrieve :return: list """ - if format_type == INSTANCE_FORMAT_TYPE_XML: - raise BadFormatException("XML is not supported") - else: - submissions = self.asset._deployment_data.get("submissions", []) - if len(instances_ids) > 0: - instances_ids = [instance_id for instance_id in instances_ids] + submissions = self.asset._deployment_data.get("submissions", []) + + if len(instances_ids) > 0: + if format_type == INSTANCE_FORMAT_TYPE_XML: + # ugly way to find matches, but it avoids to load each xml in memory. + pattern = "|".join(instances_ids) + submissions = [submission for submission in submissions + if re.search(r"({})<\/id>".format(pattern), submission)] + else: submissions = [submission for submission in submissions if submission.get("id") in instances_ids] - return submissions + return submissions def get_submission(self, pk, format_type=INSTANCE_FORMAT_TYPE_JSON): - return self.get_submissions(format_type=format_type, instances_ids=[pk])[0] + if pk: + submissions = list(self.get_submissions(format_type, [pk])) + if len(submissions) > 0: + return submissions[0] + return None + else: + raise ValueError("Primary key must be provided") def set_has_kpi_hooks(self): """ @@ -86,4 +109,4 @@ def set_has_kpi_hooks(self): has_active_hooks = self.asset.has_active_hooks self.store_data({ "has_kpi_hooks": has_active_hooks, - }) \ No newline at end of file + }) diff --git a/kpi/tests/test_api_assets.py b/kpi/tests/test_api_assets.py index 0d37e51a6f..f859523793 100644 --- a/kpi/tests/test_api_assets.py +++ b/kpi/tests/test_api_assets.py @@ -513,8 +513,7 @@ def setUp(self): '__version__': v_uid, 'q1': u'¿Qué tal?' } - self.asset.deployment._mock_submission(submission) - self.asset.save(create_version=False) + self.asset.deployment.mock_submissions([submission]) settings.CELERY_TASK_ALWAYS_EAGER = True def result_stored_locally(self, detail_response): diff --git a/kpi/tests/test_mock_data.py b/kpi/tests/test_mock_data.py index 6af34479b1..7c0819d613 100644 --- a/kpi/tests/test_mock_data.py +++ b/kpi/tests/test_mock_data.py @@ -95,8 +95,7 @@ def setUp(self): submission.update({ '__version__': v_uid }) - self.asset.deployment._mock_submission(submission) - self.asset.save(create_version=False) + self.asset.deployment.mock_submissions(submissions) schemas = [v.to_formpack_schema() for v in self.asset.deployed_versions] self.fp = FormPack(versions=schemas, id_string=self.asset.uid) self.vs = self.fp.versions.keys() diff --git a/kpi/tests/test_mock_data_exports.py b/kpi/tests/test_mock_data_exports.py index 9d478b6640..b18dbff01c 100644 --- a/kpi/tests/test_mock_data_exports.py +++ b/kpi/tests/test_mock_data_exports.py @@ -194,8 +194,7 @@ def setUp(self): submission.update({ '__version__': v_uid }) - self.asset.deployment._mock_submission(submission) - self.asset.save(create_version=False) + self.asset.deployment.mock_submissions(self.submissions) self.formpack, self.submission_stream = report_data.build_formpack( self.asset, submission_stream=self.asset.deployment.get_submissions()