From 7d8f5291324592aea57291ab4f52e8602b8e04e5 Mon Sep 17 00:00:00 2001 From: Giovanni Curiel dos Santos Date: Thu, 25 Oct 2018 16:55:04 -0300 Subject: [PATCH 1/8] feat: adding attribute type filter --- DeviceManager/DeviceHandler.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/DeviceManager/DeviceHandler.py b/DeviceManager/DeviceHandler.py index 7dd5727..b24ae47 100644 --- a/DeviceManager/DeviceHandler.py +++ b/DeviceManager/DeviceHandler.py @@ -251,13 +251,20 @@ def get_devices(req, sensitive_data=False): attr_filter = [] query = req.args.getlist('attr') - for attr in query: - parsed = re.search('^(.+){1}=(.+){1}$', attr) - attr = [] - attr.append("attrs.label = '{}'".format(parsed.group(1))) + for attr_type in query: + parsed = re.search('^(.+){1}=(.+){1}$', attr_type) + attr_type = [] + attr_type.append("attrs.label = '{}'".format(parsed.group(1))) # static value must be the override, if any - attr.append("coalesce(overrides.static_value, attrs.static_value) = '{}'".format(parsed.group(2))) - attr_filter.append(and_(*attr)) + attr_type.append("coalesce(overrides.static_value, attrs.static_value) = '{}'".format(parsed.group(2))) + attr_filter.append(and_(*attr_type)) + + query = req.args.getlist('attr_type') + for attr_type_item in query: + LOGGER.debug("attr_type: " + attr_type_item) + attr_type = [] + attr_type.append("attrs.value_type = '{}'".format(attr_type_item)) + attr_filter.append(and_(*attr_type)) label_filter = [] target_label = req.args.get('label', None) From e260710a825d3b6a32eca7984f7b153d3c22cead Mon Sep 17 00:00:00 2001 From: Giovanni Curiel dos Santos Date: Fri, 26 Oct 2018 08:36:07 -0300 Subject: [PATCH 2/8] feat: adding attr_type filter to template listing --- DeviceManager/TemplateHandler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DeviceManager/TemplateHandler.py b/DeviceManager/TemplateHandler.py index 049471e..7775795 100644 --- a/DeviceManager/TemplateHandler.py +++ b/DeviceManager/TemplateHandler.py @@ -75,6 +75,10 @@ def get_templates(req): parsed_query.append(text("attrs.label = '{}'".format(parsed.group(1)))) parsed_query.append(text("attrs.static_value = '{}'".format(parsed.group(2)))) + query = req.args.getlist('attr_type') + for attr_type_item in query: + parsed_query.append(text("attrs.value_type = '{}'".format(attr_type_item))) + target_label = req.args.get('label', None) if target_label: parsed_query.append(text("templates.label like '%{}%'".format(target_label))) From a75906d26ef1dfc5868d54313f74551b0ecb726c Mon Sep 17 00:00:00 2001 From: Giovanni Curiel dos Santos Date: Fri, 26 Oct 2018 08:36:17 -0300 Subject: [PATCH 3/8] docs: updating docs --- docs/apiary.apib | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/apiary.apib b/docs/apiary.apib index 49437ed..11bff39 100644 --- a/docs/apiary.apib +++ b/docs/apiary.apib @@ -111,7 +111,7 @@ Register a new template } -### Get the current list of templates [GET /template{?page_size,page_num,attr_format,attr,label,sortBy}] +### Get the current list of templates [GET /template{?page_size,page_num,attr_format,attr,attr_type,label,sortBy}] Get the full list of templates with all their associated attributes. + Parameters @@ -124,6 +124,7 @@ Get the full list of templates with all their associated attributes. template attributes that are listed by both 'data_attrs' and 'config_attrs'. + attr: foo=bar (string, optional) - Return only templates that posess a given attribute's value + + attr_type: geopoint (string, optional) - Return only templates with attributes of a particular type. + label: dummy (string, optional) - Return only templates that are named accordingly (prefix or suffix match) + sortBy: label (string, optional) - Return entries sorted by given field. Currently only `label` is supported. @@ -570,7 +571,7 @@ Retrieves all information from a specific device } -### Get the current list of devices [GET /device{?page_size,page_num,idsOnly,label,attr,sortBy}] +### Get the current list of devices [GET /device{?page_size,page_num,idsOnly,label,attr,attr_type,sortBy}] Get the full list of devices with all their associated attributes. Each attribute from *attrs* is the template ID from where the attributes came from. In this example, there is only one template (ID 1). @@ -579,6 +580,7 @@ In this example, there is only one template (ID 1). + page_num: 1 (integer, optional) + idsOnly: false (booelan, optional) - Return only the ids of all existing devices. Ignores any `page_size` and `page_num` configurations + attr: foo=bar (string, optional) - Return only devices that posess a given attribute's value + + attr_type: geopoint (string, optional) - Return only devices with attributes of a particular type. + label: dummy (string, optional) - Return only devices that are named accordingly (prefix or suffix match) + sortBy: label (string, optional) - Return entries sorted by given field. Currently only `label` is supported. From 82bdaca24ccee224763f472674c7785035038475 Mon Sep 17 00:00:00 2001 From: Giovanni Curiel dos Santos Date: Fri, 26 Oct 2018 08:36:27 -0300 Subject: [PATCH 4/8] fix: removing logs --- DeviceManager/DeviceHandler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/DeviceManager/DeviceHandler.py b/DeviceManager/DeviceHandler.py index b24ae47..58dfeda 100644 --- a/DeviceManager/DeviceHandler.py +++ b/DeviceManager/DeviceHandler.py @@ -261,7 +261,6 @@ def get_devices(req, sensitive_data=False): query = req.args.getlist('attr_type') for attr_type_item in query: - LOGGER.debug("attr_type: " + attr_type_item) attr_type = [] attr_type.append("attrs.value_type = '{}'".format(attr_type_item)) attr_filter.append(and_(*attr_type)) From 3a58c13da24604ab702d13fdbc9c604081e2143b Mon Sep 17 00:00:00 2001 From: Giovanni Curiel dos Santos Date: Tue, 20 Nov 2018 19:03:23 -0200 Subject: [PATCH 5/8] fixing tests: adding geopoint to the mix --- docs/apiary.apib | 32 +++++++++++++++++++++++++++++ tests/dredd-hooks/operation_hook.py | 5 +++++ 2 files changed, 37 insertions(+) diff --git a/docs/apiary.apib b/docs/apiary.apib index 11bff39..2950ca7 100644 --- a/docs/apiary.apib +++ b/docs/apiary.apib @@ -156,6 +156,14 @@ Get the full list of templates with all their associated attributes. "type": "static", "id": 2, "template_id": "1" + }, + { + "created": "2018-02-27T16:10:23.178557+00:00", + "label": "position", + "value_type": "geopoint", + "type": "dynamic", + "id": 3, + "template_id": "1" } ], "label": "SensorModel", @@ -177,6 +185,14 @@ Get the full list of templates with all their associated attributes. "value_type": "float", "type": "dynamic", "id": 1 + }, + { + "created": "2018-02-27T16:10:23.178557+00:00", + "label": "position", + "value_type": "geopoint", + "type": "dynamic", + "id": 3, + "template_id": "1" } ], "id": 1 @@ -403,6 +419,14 @@ Removes a template. If any device is based on the template being removed, then a { "removed": { "attrs": [ + { + "created": "2018-11-20T20:58:28.462464+00:00", + "id": 182, + "label": "position", + "template_id": "4865", + "type": "dynamic", + "value_type": "geopoint" + }, { "created": "2017-12-27T15:22:13.331950+00:00", "id": 9, @@ -674,6 +698,14 @@ Get the full list of devices that belong to a given template.. { "attrs": { "4865": [ + { + "created": "2018-11-20T20:58:29.564672+00:00", + "id": 198, + "label": "position", + "template_id": "668", + "type": "dynamic", + "value_type": "geopoint" + }, { "label": "temperature", "type": "dynamic", diff --git a/tests/dredd-hooks/operation_hook.py b/tests/dredd-hooks/operation_hook.py index 65f4dd0..cb83f9f 100644 --- a/tests/dredd-hooks/operation_hook.py +++ b/tests/dredd-hooks/operation_hook.py @@ -30,6 +30,11 @@ def create_sample_template(): "type": "dynamic", "value_type": "float" }, + { + "label": "position", + "type": "dynamic", + "value_type": "geopoint" + }, { "label": "model-id", "type": "static", From 5a525292a9ec056ee366cca81d03cce648461ce1 Mon Sep 17 00:00:00 2001 From: Giovanni Curiel dos Santos Date: Tue, 5 Feb 2019 15:48:42 -0200 Subject: [PATCH 6/8] fix: changing variable name in DeviceHandler attribute type lookup --- DeviceManager/DeviceHandler.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/DeviceManager/DeviceHandler.py b/DeviceManager/DeviceHandler.py index 58dfeda..c916352 100644 --- a/DeviceManager/DeviceHandler.py +++ b/DeviceManager/DeviceHandler.py @@ -251,13 +251,14 @@ def get_devices(req, sensitive_data=False): attr_filter = [] query = req.args.getlist('attr') - for attr_type in query: - parsed = re.search('^(.+){1}=(.+){1}$', attr_type) - attr_type = [] - attr_type.append("attrs.label = '{}'".format(parsed.group(1))) + for attr_label_item in query: + parsed = re.search('^(.+){1}=(.+){1}$', attr_label_item) + attr_label = [] + attr_label.append("attrs.label = '{}'".format(parsed.group(1))) # static value must be the override, if any - attr_type.append("coalesce(overrides.static_value, attrs.static_value) = '{}'".format(parsed.group(2))) - attr_filter.append(and_(*attr_type)) + attr_label.append("coalesce(overrides.static_value, attrs.static_value) = '{}'".format(parsed.group(2))) + LOGGER.debug(f"Adding filter ${attr_label} to query") + attr_filter.append(and_(*attr_label)) query = req.args.getlist('attr_type') for attr_type_item in query: From d9e17c06fa1ce6cda8c71c914d082b72a03db410 Mon Sep 17 00:00:00 2001 From: Giovanni Curiel dos Santos Date: Tue, 19 Mar 2019 16:31:53 -0300 Subject: [PATCH 7/8] fix: fixing sort_by call when using extra parameters This commit fixes a problem when the user sets a sort_by parameter and postgres indicates that the DeviceTemplate.id should be used as sort_by instead (as this attribute is used in a "distinct" clause) --- DeviceManager/TemplateHandler.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/DeviceManager/TemplateHandler.py b/DeviceManager/TemplateHandler.py index 693772c..dea4792 100644 --- a/DeviceManager/TemplateHandler.py +++ b/DeviceManager/TemplateHandler.py @@ -99,17 +99,22 @@ def get_templates(req): SORT_CRITERION = { 'label': DeviceTemplate.label, - None: DeviceTemplate.id + None: None } sortBy = SORT_CRITERION.get(req.args.get('sortBy', None), DeviceTemplate.id) LOGGER.debug(f"Sortby filter is {sortBy}") if parsed_query: LOGGER.debug(f" Filtering template by {parsed_query}") + + # Always sort by DeviceTemplate.id page = db.session.query(DeviceTemplate) \ .join(DeviceAttr, isouter=True) \ .filter(*parsed_query) \ - .order_by(sortBy) \ - .distinct(DeviceTemplate.id) + .order_by(DeviceTemplate.id) + if sortBy: + page = page.order_by(sortBy) + page = page.distinct(DeviceTemplate.id) + LOGGER.debug(f"Current query: {type(page)}") page = BaseQuery(page.subquery(), db.session()).paginate(**pagination) else: From 76f455c2823e6422ed577eb21acac9dc3a4a7fd4 Mon Sep 17 00:00:00 2001 From: Giovanni Curiel dos Santos Date: Tue, 19 Mar 2019 16:42:18 -0300 Subject: [PATCH 8/8] fix: fixing query This commit fixes a bug which removes all `joins` when mixing query structures from SQLAlchemy and Flask-SQLAlchemy. This is fully detailed [here](https://stackoverflow.com/questions/15727155/how-to-paginate-in-flask-sqlalchemy-for-db-session-joined-queries) --- DeviceManager/TemplateHandler.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/DeviceManager/TemplateHandler.py b/DeviceManager/TemplateHandler.py index dea4792..27c78f0 100644 --- a/DeviceManager/TemplateHandler.py +++ b/DeviceManager/TemplateHandler.py @@ -1,7 +1,7 @@ import logging import re from flask import Blueprint, request, jsonify, make_response -from flask_sqlalchemy import BaseQuery +from flask_sqlalchemy import BaseQuery, Pagination from sqlalchemy.exc import IntegrityError from sqlalchemy.sql import text, collate, func @@ -50,6 +50,19 @@ def remove(d,k): return result +def paginate(query, page, per_page=20, error_out=False): + if error_out and page < 1: + return None + items = query.limit(per_page).offset((page - 1) * per_page).all() + if not items and page != 1 and error_out: + return None + + if page == 1 and len(items) < per_page: + total = len(items) + else: + total = query.count() + + return Pagination(query, page, per_page, total, items) class TemplateHandler: @@ -101,7 +114,7 @@ def get_templates(req): 'label': DeviceTemplate.label, None: None } - sortBy = SORT_CRITERION.get(req.args.get('sortBy', None), DeviceTemplate.id) + sortBy = SORT_CRITERION.get(req.args.get('sortBy', None), None) LOGGER.debug(f"Sortby filter is {sortBy}") if parsed_query: LOGGER.debug(f" Filtering template by {parsed_query}") @@ -113,10 +126,11 @@ def get_templates(req): .order_by(DeviceTemplate.id) if sortBy: page = page.order_by(sortBy) + page = page.distinct(DeviceTemplate.id) LOGGER.debug(f"Current query: {type(page)}") - page = BaseQuery(page.subquery(), db.session()).paginate(**pagination) + page = paginate(page, **pagination) else: LOGGER.debug(f" Querying templates sorted by {sortBy}") page = db.session.query(DeviceTemplate).order_by(sortBy).paginate(**pagination) @@ -139,6 +153,8 @@ def get_templates(req): 'templates': templates } + LOGGER.debug(f"Full response is {result}") + return result @staticmethod