From b14c7c6f67330613c51ae4b0604e9719dacc92f1 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Wed, 18 Feb 2015 19:25:12 -0200 Subject: [PATCH 01/25] Add bytes transformation Humanize bytes on that transformation. --- datagrid_gtk3/tests/test_datagrid-gtk3.py | 21 +++++++++++++++ datagrid_gtk3/ui/grid.py | 33 ++++++++++++++++++++--- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/datagrid_gtk3/tests/test_datagrid-gtk3.py b/datagrid_gtk3/tests/test_datagrid-gtk3.py index 2403c3f..cd9d0e4 100644 --- a/datagrid_gtk3/tests/test_datagrid-gtk3.py +++ b/datagrid_gtk3/tests/test_datagrid-gtk3.py @@ -227,6 +227,27 @@ def test_datetime_transform_microseconds_time(self): self.datagrid_model._datetime_transform(315532801000000), '1980-01-01T00:00:01') + def test_bytes_transform(self): + """Test bytes humanization.""" + self.assertEqual( + self.datagrid_model._bytes_transform(1), + '1 byte') + self.assertEqual( + self.datagrid_model._bytes_transform(50), + '50.0 bytes') + self.assertEqual( + self.datagrid_model._bytes_transform(2348), + '2.3 kB') + self.assertEqual( + self.datagrid_model._bytes_transform(1420000), + '1.4 MB') + self.assertEqual( + self.datagrid_model._bytes_transform(1420000328), + '1.3 GB') + self.assertEqual( + self.datagrid_model._bytes_transform(24200003283214), + '22.0 TB') + @mock.patch('datagrid_gtk3.ui.grid.NO_IMAGE_PIXBUF.scale_simple') def test_image_transform_no_value(self, scale_simple): """Return an invisible image when no value is provided.""" diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index b9b5298..cc1ef84 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -1391,7 +1391,6 @@ def get_formatted_value(self, value, column_index, visible=True): value = ' '.join(value) else: value = '' - elif col_dict['transform'] == 'boolean': if col_dict['name'] != '__selected': value = self._boolean_transform(value) @@ -1401,15 +1400,18 @@ def get_formatted_value(self, value, column_index, visible=True): # 0 or null return False - elif col_dict['transform'] == 'image': value = self._image_transform(value, visible=visible) - elif col_dict['transform'] == 'datetime': if value: value = self._datetime_transform(value) else: return '' + elif col_dict['transform'] == 'bytes': + if value: + value = self._bytes_transform(value) + else: + return '' else: # If no transformation is required, at least convert the value to # str as required by CellRendererText @@ -1492,6 +1494,31 @@ def _image_transform(self, value, visible=True): return self._get_pixbuf(filename) + def _bytes_transform(self, value): + """Transform bytes into a human-readable value. + + :param int value: bytes to be humanized + :returns: the humanized bytes + :rtype: str + """ + if value == 1: + return '1 byte' + + for suffix, factor in [ + ('PB', 1 << 50), + ('TB', 1 << 40), + ('GB', 1 << 30), + ('MB', 1 << 20), + ('kB', 1 << 10), + ('bytes', 1)]: + if value >= factor: + value = '%.*f %s' % (1, float(value) / factor, suffix) + break + else: + raise ValueError('Unexpected value: %s' % (value, )) + + return value + def _datetime_transform(self, value): """Transform timestamps to ISO 8601 date format. From 2c3f5c777b273760d43c76bee19e7962d0bb33ef Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Thu, 19 Feb 2015 17:24:57 -0200 Subject: [PATCH 02/25] The database id is not necessarily an integer It also doesn't make sense since record_id gets casted to str() just after this to be used. --- datagrid_gtk3/db/sqlite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 54d1c5c..92f4b9b 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -223,7 +223,6 @@ class default table :return: row of data :rtype: tuple """ - record_id = int(record_id) if table is None: table = self.table sql_statement = 'SELECT * FROM %s WHERE %s = ?' % ( From 530e4819ed57b134ad0c81e1434263476f135549 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Fri, 20 Feb 2015 18:28:35 -0200 Subject: [PATCH 03/25] Allow datagrid to show hierarchical data One can now specify a parent column to SQLiteSource, which will make the data be displayed in an hierarchical way on the datagrid. --- datagrid_gtk3/db/sqlite.py | 86 ++++++++++++++++++++--- datagrid_gtk3/ui/grid.py | 138 ++++++++++++++++++++++++++++++------- 2 files changed, 190 insertions(+), 34 deletions(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 92f4b9b..69989b0 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -18,6 +18,21 @@ logger = logging.getLogger(__name__) +class Node(list): + + """A list that can hold data. + + Just like a simple list, but one can set/get its data + from :obj:`.data`. + + :param object data: the data that will be stored in this node + """ + + def __init__(self, data=None): + super(Node, self).__init__() + self.data = data + + class SQLiteDataSource(DataSource): """SQLite data source especially for use with a `Gtk.TreeModel`. @@ -57,6 +72,7 @@ class SQLiteDataSource(DataSource): 'BLOB': str } ID_COLUMN = 'rowid' + PARENT_ID_COLUMN = None def __init__(self, db_file, table=None, update_table=None, config=None, ensure_selected_column=True, display_all=False, query=None): @@ -75,6 +91,8 @@ def __init__(self, db_file, table=None, update_table=None, config=None, self.update_table = table self.config = config self.rows = None + self._id_column_idx = None + self._parent_column_idx = None self.columns = self.get_columns() column_names = ['"%s"' % col['name'] for col in self.columns] self.column_name_str = ', '.join(column_names) @@ -119,6 +137,10 @@ def load(self, params=None): bindings = [] where_sql = '' order_sql = '' + # FIXME: We probably should return rows in this function + # instead of setting it to self.rows and having datagrid to + # access it after. + self.rows = Node() if params: # construct WHERE clause if 'where' in params: @@ -139,17 +161,46 @@ def load(self, params=None): if offset >= self.total_recs: # at end of total records, return no records # for paging - results = [] last_page = True if not last_page: - sql = 'SELECT %s FROM %s %s %s LIMIT %d OFFSET %d' % ( - self.column_name_str, self.table, where_sql, - order_sql, self.MAX_RECS, offset - ) - logger.debug('SQL: %s, %s', sql, bindings) - cursor.execute(sql, bindings) - results = cursor.fetchall() - self.rows = results + # FIXME: How to properly do lazy loading in this case? + if self.PARENT_ID_COLUMN: + def get_results(parent): + operator = 'is' if parent is None else '=' + if where_sql: + parent_where = '%s AND %s %s ?' % ( + where_sql, operator, self.PARENT_ID_COLUMN) + else: + parent_where = ' WHERE %s %s ? ' % ( + self.PARENT_ID_COLUMN, operator) + sql = 'SELECT %s FROM %s %s %s' % ( + self.column_name_str, self.table, + parent_where, order_sql) + + bindings_ = bindings + [parent] + logger.debug('SQL: %s, %s', sql, bindings_) + # FIXME: If would be better to do: + # for row in cursor.execute(sql, bindings_): + # yield row + # But for that we would need different cursors + cursor.execute(sql, bindings_) + return cursor.fetchall() + + def build_tree(parent, parent_id): + for row in get_results(parent_id): + node = Node(data=row) + parent.append(node) + build_tree(node, row[self._id_column_idx]) + + build_tree(self.rows, None) + else: + sql = 'SELECT %s FROM %s %s %s LIMIT %d OFFSET %d' % ( + self.column_name_str, self.table, where_sql, + order_sql, self.MAX_RECS, offset) + logger.debug('SQL: %s, %s', sql, bindings) + for row in cursor.execute(sql, bindings): + self.rows.append(Node(data=row)) + if first_access: # set the total record count the only the first time the # record set is requested @@ -422,7 +473,7 @@ def get_columns(self): rows = cursor.fetchall() has_selected = False counter = 0 - for row in rows: + for i, row in enumerate(rows): col_defined = False col_name = row[1] if self.config is not None: @@ -441,6 +492,12 @@ def get_columns(self): 'type': data_type, 'transform': transform } + + if col_name == self.ID_COLUMN: + self._id_column_idx = i + if col_name == self.PARENT_ID_COLUMN: + self._parent_column_idx = i + if row[1] == '__selected': col_dict['transform'] = 'boolean' cols.insert(0, col_dict) @@ -459,6 +516,15 @@ def get_columns(self): 'transform': 'boolean' } cols.insert(0, col_dict) + has_selected = True + + # If __selected column is present, it was inserted on position + # 0, so we need to increase the id/parent columns by 1 + if has_selected and self._id_column_idx is not None: + self._id_column_idx += 1 + if has_selected and self._parent_column_idx is not None: + self._parent_column_idx += 1 + return cols diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index cc1ef84..7e8387c 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -4,6 +4,7 @@ import base64 import os from datetime import datetime +import itertools from gi.repository import ( GObject, @@ -893,16 +894,19 @@ def _setup_columns(self): self.check_btn_toggle_all = check_btn self.append_column(col) - samples = self.model.rows[:self.SAMPLE_SIZE] + # FIXME: We should find a better way for hiding this columns. + # A way to specify the visibility on the columns config would be nice. + dont_display = set([ + self.model.data_source.ID_COLUMN, + self.model.data_source.PARENT_ID_COLUMN, + '__selected']) + + samples = itertools.islice(self.model.iter_rows(), self.SAMPLE_SIZE) for column_index, column in enumerate(self.model.columns): item = column['name'] display = (self.model.display_columns is None or item in self.model.display_columns) - if not self.model.data_source.display_all: - # First column is "_selected" checkbox, - # second is invisible primary key ID - display = display and column_index > 1 - if display: + if display and column['name'] not in dont_display: item_display = column['display'] if column['transform'] in ['boolean', 'image']: renderer = Gtk.CellRendererPixbuf() @@ -1324,6 +1328,8 @@ def add_rows(self): """ self.active_params['page'] = self.active_params.get('page', 0) + 1 + # FIXME: Is it right to call row_inserted with the same path/itr + # for all the new rows? path = (len(self.rows) - 1,) itr = self.get_iter(path) self.data_source.load(self.active_params) @@ -1435,14 +1441,57 @@ def set_value(self, itr, column, value, emit_event=True): :param bool emit_event: if we should call :meth:`.row_changed`. Be sure to know what you are doing before passind `False` here """ - path = self.get_path(itr)[0] - self.rows[path][column] = value + path = self.get_path(itr) + # path and iter are the same in this model + row = self._get_row_by_iter(path) + row.data[column] = value id_ = self.get_value(itr, 1) self.update_data_source( self.columns[column]['name'], value, [int(id_)]) if emit_event: self.row_changed(path, itr) + def iter_rows(self): + """Iterate over the rows of the model. + + This will iterate using a depth-first algorithm. That means that, + on a hierarchy like this:: + + A + B + E + C + F + G + H + D + + This would generate an iteration like:: + + [ A, B, E, C, F, G, H, D ] + + """ + def _iter_children_aux(parent): + for row in parent: + yield row + for inner_row in _iter_children_aux(row): + yield inner_row + + for row in _iter_children_aux(self.rows): + yield row.data + + ### + # Private + ### + + def _get_row_by_iter(self, iter_): + def get_row_by_iter_aux(iter_aux, rows): + if len(iter_aux) == 1: + return rows[iter_aux[0]] + return get_row_by_iter_aux(iter_aux[1:], rows[iter_aux[0]]) + + return get_row_by_iter_aux(iter_, self.rows) + ### # Transforms ### @@ -1610,7 +1659,7 @@ def _get_pixbuf(self, path=None): def on_get_flags(self): """Return the GtkTreeModelFlags for this particular type of model.""" - return Gtk.TreeModelFlags.LIST_ONLY + return Gtk.TreeModelFlags.ITERS_PERSIST def on_get_n_columns(self): """Return the number of columns in the model.""" @@ -1631,52 +1680,93 @@ def on_get_column_type(self, index): def on_get_path(self, rowref): """Return the tree path (a tuple of indices) for a particular node.""" - return (rowref,) + return tuple(rowref) def on_get_iter(self, path): """Return the node corresponding to the given path (node is path).""" - if path[0] < len(self.rows): - return path[0] - - return None + try: + # row and path are the same in this model. We just need + # to make sure that the iter is valid + self._get_row_by_iter(path) + except IndexError: + return None + else: + return tuple(path) def on_get_value(self, rowref, column): """Return the value stored in a particular column for the node.""" if self.visible_range: - visible = self.visible_range[0][0] <= rowref <= self.visible_range[1][0] + start = tuple(self.visible_range[0]) + end = tuple(self.visible_range[1]) + visible = start <= rowref <= end else: visible = True - raw = self.rows[rowref][column] + row = self._get_row_by_iter(rowref) + raw = row.data[column] val = self.get_formatted_value(raw, column, visible=visible) return val def on_iter_next(self, rowref): """Return the next node at this level of the tree.""" - if rowref + 1 < len(self.rows): - return rowref + 1 + if rowref is None: + return None - return None + # root node + if len(rowref) == 1: + rows = self.rows + next_value = (rowref[0] + 1, ) + else: + parentref = rowref[:-1] + rows = self._get_row_by_iter(parentref) + next_value = parentref + (rowref[-1] + 1, ) + + if not next_value[-1] < len(rows): + return None + + return next_value def on_iter_children(self, rowref): """Return the first child of this node.""" - return 0 + if rowref is None: + return (0, ) + + parent_row = self._get_row_by_iter(rowref) + if not len(parent_row): + return None + + return rowref + (0, ) def on_iter_has_child(self, rowref): """Return true if this node has children.""" - return False + return bool(self._get_row_by_iter(rowref)) def on_iter_n_children(self, rowref): """Return the number of children of this node.""" - return len(self.rows) + if rowref is None: + return len(self.rows) + + return len(self._get_row_by_iter(rowref)) def on_iter_nth_child(self, parent, n): """Return the nth child of this node.""" - return n + if parent is None: + parent = () + rows = self.rows + else: + rows = self._get_row_by_iter(parent) + + if not 0 <= n < len(rows): + return None + + return parent + (n, ) def on_iter_parent(self, child): """Return the parent of this node.""" - return None + if len(child) == 1: + return None + + return child[:-1] ### # END Required implementations for GenericTreeModel From 1f9ac796bb054dc3e11930419c4823cbd24bfbf4 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Mon, 23 Feb 2015 13:38:13 -0300 Subject: [PATCH 04/25] Return rows from load instead of storing them on data source Instead of setting rows on the data source, which will be overwritten after load is called again, return it on the method. --- datagrid_gtk3/db/sqlite.py | 12 +++++------- datagrid_gtk3/ui/grid.py | 10 +++++----- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 69989b0..089f15a 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -90,7 +90,6 @@ def __init__(self, db_file, table=None, update_table=None, config=None, else: self.update_table = table self.config = config - self.rows = None self._id_column_idx = None self._parent_column_idx = None self.columns = self.get_columns() @@ -137,10 +136,7 @@ def load(self, params=None): bindings = [] where_sql = '' order_sql = '' - # FIXME: We probably should return rows in this function - # instead of setting it to self.rows and having datagrid to - # access it after. - self.rows = Node() + rows = Node() if params: # construct WHERE clause if 'where' in params: @@ -192,14 +188,14 @@ def build_tree(parent, parent_id): parent.append(node) build_tree(node, row[self._id_column_idx]) - build_tree(self.rows, None) + build_tree(rows, None) else: sql = 'SELECT %s FROM %s %s %s LIMIT %d OFFSET %d' % ( self.column_name_str, self.table, where_sql, order_sql, self.MAX_RECS, offset) logger.debug('SQL: %s, %s', sql, bindings) for row in cursor.execute(sql, bindings): - self.rows.append(Node(data=row)) + rows.append(Node(data=row)) if first_access: # set the total record count the only the first time the @@ -209,6 +205,8 @@ def build_tree(parent, parent_id): cursor.execute(sql, bindings) self.total_recs = int(cursor.fetchone()[0]) + return rows + def update(self, params, ids=None): """Update the recordset with a SQL ``UPDATE`` statement. diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 7e8387c..4f0d897 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -1315,8 +1315,7 @@ def refresh(self): if 'page' in self.active_params: del self.active_params['page'] - self.data_source.load(self.active_params) - self.rows = self.data_source.rows + self.rows = self.data_source.load(self.active_params) self.total_recs = self.data_source.total_recs self.emit('data-loaded', self.total_recs) @@ -1332,11 +1331,12 @@ def add_rows(self): # for all the new rows? path = (len(self.rows) - 1,) itr = self.get_iter(path) - self.data_source.load(self.active_params) - if not self.data_source.rows: + + rows = self.data_source.load(self.active_params) + if not len(rows): return False - for row in self.data_source.rows: + for row in rows: self.rows.append(row) self.row_inserted(path, itr) return True From 5983723f8f4fc810dbac68c8ea338807b871a69e Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Mon, 23 Feb 2015 14:43:45 -0300 Subject: [PATCH 05/25] Don't format id and parent columns when getting the row We may want that value to compare to something and in some cases, if the id is a string, it cannot be ellipsized. --- datagrid_gtk3/db/sqlite.py | 18 +++++++++--------- datagrid_gtk3/ui/grid.py | 23 ++++++++++++++++------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 089f15a..149e5d5 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -90,8 +90,8 @@ def __init__(self, db_file, table=None, update_table=None, config=None, else: self.update_table = table self.config = config - self._id_column_idx = None - self._parent_column_idx = None + self.id_column_idx = None + self.parent_column_idx = None self.columns = self.get_columns() column_names = ['"%s"' % col['name'] for col in self.columns] self.column_name_str = ', '.join(column_names) @@ -186,7 +186,7 @@ def build_tree(parent, parent_id): for row in get_results(parent_id): node = Node(data=row) parent.append(node) - build_tree(node, row[self._id_column_idx]) + build_tree(node, row[self.id_column_idx]) build_tree(rows, None) else: @@ -492,9 +492,9 @@ def get_columns(self): } if col_name == self.ID_COLUMN: - self._id_column_idx = i + self.id_column_idx = i if col_name == self.PARENT_ID_COLUMN: - self._parent_column_idx = i + self.parent_column_idx = i if row[1] == '__selected': col_dict['transform'] = 'boolean' @@ -518,10 +518,10 @@ def get_columns(self): # If __selected column is present, it was inserted on position # 0, so we need to increase the id/parent columns by 1 - if has_selected and self._id_column_idx is not None: - self._id_column_idx += 1 - if has_selected and self._parent_column_idx is not None: - self._parent_column_idx += 1 + if has_selected and self.id_column_idx is not None: + self.id_column_idx += 1 + if has_selected and self.parent_column_idx is not None: + self.parent_column_idx += 1 return cols diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 4f0d897..8a0eab2 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -583,7 +583,7 @@ def on_treeview_cursor_changed(self, view): model, row_iterator = selection.get_selected() if row_iterator and self.selected_record_callback: record = self.model.data_source.get_single_record( - model[row_iterator][1]) + model[row_iterator][self.model.id_column_idx]) self.selected_record_callback(record) elif self.selected_record_callback: self.selected_record_callback(None) @@ -599,7 +599,7 @@ def on_iconview_selection_changed(self, view): if row_iterator and self.selected_record_callback: model = view.get_model() record = self.model.data_source.get_single_record( - model[row_iterator][1]) + model[row_iterator][self.model.id_column_idx]) self.selected_record_callback(record) elif self.selected_record_callback: self.selected_record_callback(None) @@ -616,7 +616,7 @@ def on_iconview_item_activated(self, view, path): row_iterator = view.model.get_iter(path) record = self.model.data_source.get_single_record( - self.model[row_iterator][1]) + self.model[row_iterator][self.model.id_column_idx]) # Why is the pixbuf column on view.pixbuf_column -1 position in this rec? self.activated_icon_callback(record, view.pixbuf_column - 1) @@ -1307,6 +1307,8 @@ def __init__(self, data_source, get_media_callback, decode_fallback, self.encoding_hint = encoding_hint self.selected_cells = list() + self.id_column_idx = None + self.parent_column_idx = None self.rows = None self.total_recs = None @@ -1316,6 +1318,8 @@ def refresh(self): del self.active_params['page'] self.rows = self.data_source.load(self.active_params) + self.id_column_idx = self.data_source.id_column_idx + self.parent_column_idx = self.data_source.parent_column_idx self.total_recs = self.data_source.total_recs self.emit('data-loaded', self.total_recs) @@ -1442,10 +1446,10 @@ def set_value(self, itr, column, value, emit_event=True): Be sure to know what you are doing before passind `False` here """ path = self.get_path(itr) - # path and iter are the same in this model + # path and iter are the same in this model. row = self._get_row_by_iter(path) row.data[column] = value - id_ = self.get_value(itr, 1) + id_ = self.get_value(itr, self.id_column_idx) self.update_data_source( self.columns[column]['name'], value, [int(id_)]) if emit_event: @@ -1704,8 +1708,13 @@ def on_get_value(self, rowref, column): row = self._get_row_by_iter(rowref) raw = row.data[column] - val = self.get_formatted_value(raw, column, visible=visible) - return val + # Don't format value for id and parent columns. They are not displayed + # on the grid and we may need their full values to get their records + # (e.g. when the id is a string column) + if column in [self.id_column_idx, self.parent_column_idx]: + return raw + else: + return self.get_formatted_value(raw, column, visible=visible) def on_iter_next(self, rowref): """Return the next node at this level of the tree.""" From f236648e0d7dd0828fa6fabc4f110970daf8db6a Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Mon, 23 Feb 2015 15:33:34 -0300 Subject: [PATCH 06/25] Avoid loading more results on an hierarchical data --- datagrid_gtk3/db/sqlite.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 149e5d5..5b3afbb 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -151,12 +151,17 @@ def load(self, params=None): order_sql = order_sql + ' DESC' # determine OFFSET value for paging if 'page' in params: + # FIXME: If we have a PARENT_ID_COLUMN, all results + # were loaded on first load. How to handle this better? + if self.PARENT_ID_COLUMN: + return [] first_access = False if params['page']: offset = params['page'] * self.MAX_RECS if offset >= self.total_recs: # at end of total records, return no records # for paging + return [] last_page = True if not last_page: # FIXME: How to properly do lazy loading in this case? From d1d0464a4331a38fdd935d5a488865ec1b2e0916 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Mon, 23 Feb 2015 16:06:39 -0300 Subject: [PATCH 07/25] Add an optional callback for row activated on treeview --- datagrid_gtk3/ui/grid.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 8a0eab2..f8e1be1 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -370,8 +370,8 @@ class DataGridController(object): MAX_TIMESTAMP = 2147485547 # 2038 def __init__(self, container, data_source, selected_record_callback=None, - activated_icon_callback=None, has_checkboxes=True, - decode_fallback=None, get_full_path=None): + activated_icon_callback=None, activated_row_callback=None, + has_checkboxes=True, decode_fallback=None, get_full_path=None): """Setup UI controls and load initial data view.""" if decode_fallback is None: decode_fallback = default_decode_fallback @@ -384,6 +384,7 @@ def __init__(self, container, data_source, selected_record_callback=None, self.get_full_path = get_full_path self.selected_record_callback = selected_record_callback self.activated_icon_callback = activated_icon_callback + self.activated_row_callback = activated_row_callback self.vscroll = container.grid_scrolledwindow.get_vadjustment() self.vscroll.connect('value-changed', self.on_scrolled) @@ -393,6 +394,8 @@ def __init__(self, container, data_source, selected_record_callback=None, self.tree_view.connect('cursor-changed', self.on_treeview_cursor_changed) + self.tree_view.connect('row-activated', + self.on_treeview_row_activated) self.icon_view.connect('selection-changed', self.on_iconview_selection_changed) self.icon_view.connect('item-activated', @@ -620,6 +623,28 @@ def on_iconview_item_activated(self, view, path): # Why is the pixbuf column on view.pixbuf_column -1 position in this rec? self.activated_icon_callback(record, view.pixbuf_column - 1) + def on_treeview_row_activated(self, view, path, column): + """Handle row-activated signal on the treeview. + + Run the optional :obj:`.activated_row_callback` when + a row gets activated. + + :param view: The treeview containing the row + :type view: :class:`Gtk.TreeView` + :param path: the activated path + :type path: :class:`Gtk.TreePath` + :param column: the column that was activated on the row + :type column: class:`Gtk.TreeViewColumn` + + """ + if self.activated_row_callback is None: + return + + row = self.model[self.model.get_iter(path)] + selected_id = row[self.model.id_column_idx] + record = self.model.data_source.get_single_record(selected_id) + self.activated_row_callback(record) + def on_data_loaded(self, model, total_recs): """Update the total records label. From 02d883cc9820fb43fc8b167e9d380962a9f246d7 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 24 Feb 2015 15:16:49 -0300 Subject: [PATCH 08/25] Format byte as B --- datagrid_gtk3/tests/test_datagrid-gtk3.py | 4 ++-- datagrid_gtk3/ui/grid.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/datagrid_gtk3/tests/test_datagrid-gtk3.py b/datagrid_gtk3/tests/test_datagrid-gtk3.py index cd9d0e4..b1d11ec 100644 --- a/datagrid_gtk3/tests/test_datagrid-gtk3.py +++ b/datagrid_gtk3/tests/test_datagrid-gtk3.py @@ -231,10 +231,10 @@ def test_bytes_transform(self): """Test bytes humanization.""" self.assertEqual( self.datagrid_model._bytes_transform(1), - '1 byte') + '1 B') self.assertEqual( self.datagrid_model._bytes_transform(50), - '50.0 bytes') + '50.0 B') self.assertEqual( self.datagrid_model._bytes_transform(2348), '2.3 kB') diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index f8e1be1..3389931 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -1579,16 +1579,13 @@ def _bytes_transform(self, value): :returns: the humanized bytes :rtype: str """ - if value == 1: - return '1 byte' - for suffix, factor in [ ('PB', 1 << 50), ('TB', 1 << 40), ('GB', 1 << 30), ('MB', 1 << 20), ('kB', 1 << 10), - ('bytes', 1)]: + ('B', 1)]: if value >= factor: value = '%.*f %s' % (1, float(value) / factor, suffix) break From 5cd758490f8e849f86d530fd08eb7d7e7d7b3499 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 24 Feb 2015 15:47:04 -0300 Subject: [PATCH 09/25] Keep track of expanded rows If the view is refreshed (because its sorting changed for example), be sure that the expanded rows are still expanded. --- datagrid_gtk3/ui/grid.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 3389931..cb80ed6 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -776,6 +776,8 @@ def __init__(self, model, **kwargs): super(DataGridView, self).__init__(**kwargs) self.connect_after('notify::model', self.after_notify_model) + self.connect('row-expanded', self.on_row_expanded) + self.connect('row-collapsed', self.on_row_collapsed) # FIXME: Ideally, we should pass model directly to treeview and get # it from self.get_model instead of here. We would need to refresh @@ -786,6 +788,7 @@ def __init__(self, model, **kwargs): self.set_rules_hint(True) self.active_sort_column = None self.active_sort_column_order = None + self._expanded_ids = set() ### # Public @@ -802,6 +805,13 @@ def refresh(self): self._setup_columns() + if self._expanded_ids: + def _maybe_expand_row(model, path, iter_): + row_id = model.get_value(iter_, model.id_column_idx) + if row_id in self._expanded_ids: + self.expand_row(path, False) + self.model.foreach(_maybe_expand_row) + ### # Callbacks ### @@ -821,6 +831,36 @@ def after_notify_model(self, treeview, p_spec): model.connect('row-changed', self.on_model_row_changed) + def on_row_expanded(self, treeview, iter_, path): + """Handle row-expanded events. + + Keep track of which rows are currently expanded + + :param treeview: the treeview that had one of its rows expanded + :type treeview: :class:`Gtk.TreeView` + :param iter_: the iter pointing to the expanded row + :type iter_: class:`Gtk.TreeIter` + :param path: the path pointing to the expanded row + :type path: :class:`Gtk.TreePath` + """ + self._expanded_ids.add( + self.model.get_value(iter_, self.model.id_column_idx)) + + def on_row_collapsed(self, treeview, iter_, path): + """Handle row-collapsed events. + + Keep track of which rows are currently expanded + + :param treeview: the treeview that had one of its rows collapsed + :type treeview: :class:`Gtk.TreeView` + :param iter_: the iter pointing to the collapsed row + :type iter_: class:`Gtk.TreeIter` + :param path: the path pointing to the collapsed row + :type path: :class:`Gtk.TreePath` + """ + self._expanded_ids.discard( + self.model.get_value(iter_, self.model.id_column_idx)) + def on_model_row_changed(self, model, path, iter_): """Track row changes on model. From ffce8b23f9b48ed8332749e3ad0957bd066e37d9 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Wed, 25 Feb 2015 17:47:10 -0300 Subject: [PATCH 10/25] Add "expand all"/"collapse all" buttons --- datagrid_gtk3/ui/glade/datagrid.glade | 54 +++++++++++---- datagrid_gtk3/ui/grid.py | 95 ++++++++++++++++++++++++--- 2 files changed, 127 insertions(+), 22 deletions(-) diff --git a/datagrid_gtk3/ui/glade/datagrid.glade b/datagrid_gtk3/ui/glade/datagrid.glade index 83eeee7..638ca95 100644 --- a/datagrid_gtk3/ui/glade/datagrid.glade +++ b/datagrid_gtk3/ui/glade/datagrid.glade @@ -41,6 +41,32 @@ 0 + + + ▶ + True + True + Expand all rows + + + False + True + 1 + + + + + ▼ + True + True + Collapse all rows + + + False + True + 2 + + True @@ -51,9 +77,12 @@ False True 3 - 1 + 3 + + + True @@ -64,7 +93,7 @@ False True 3 - 2 + 6 @@ -76,7 +105,7 @@ True True 3 - 3 + 7 @@ -89,7 +118,7 @@ False False 2 - 4 + 8 @@ -107,7 +136,7 @@ False True - 5 + 9 @@ -120,7 +149,7 @@ False True 2 - 6 + 10 @@ -133,7 +162,7 @@ False False 2 - 7 + 11 @@ -151,7 +180,7 @@ False True - 8 + 12 @@ -164,7 +193,7 @@ False True 3 - 9 + 13 @@ -178,7 +207,7 @@ True True - 10 + 14 @@ -191,12 +220,9 @@ True True - 11 + 15 - - - False diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index cb80ed6..547e285 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -396,11 +396,18 @@ def __init__(self, container, data_source, selected_record_callback=None, self.on_treeview_cursor_changed) self.tree_view.connect('row-activated', self.on_treeview_row_activated) + self.tree_view.connect('all-expanded', + self.on_treeview_all_expanded) self.icon_view.connect('selection-changed', self.on_iconview_selection_changed) self.icon_view.connect('item-activated', self.on_iconview_item_activated) + self.container.expand_all_btn.connect( + 'clicked', self.on_expand_all_btn_clicked) + self.container.collapse_all_btn.connect( + 'clicked', self.on_collapse_all_btn_clicked) + # The treview will be the default view self.view = self.tree_view self.container.grid_scrolledwindow.add(self.view) @@ -500,7 +507,7 @@ def bind_datasource(self, data_source): for widget in widgets: widget.show() - self.view.refresh() + self._refresh_view() ### # Callbacks @@ -568,7 +575,7 @@ def on_popup_view_changed(self, popup, new_view): self.container.grid_scrolledwindow.add(self.view) self.view.show_all() - self.view.refresh() + self._refresh_view() # FIXME: Is there a way to keep the selection after the view was # refreshed? The actual selected paths are not guaranteed to be the # same, so how can we get them again? @@ -645,6 +652,35 @@ def on_treeview_row_activated(self, view, path, column): record = self.model.data_source.get_single_record(selected_id) self.activated_row_callback(record) + def on_treeview_all_expanded(self, view, all_expanded): + """Handle all-expanded signal on the treeview. + + Set visibility for "expand all" and "collapse all" buttons + based on the all_expanded value. + + :param view: The treeview that received the signal + :type view: :class:`Gtk.TreeView` + :param bool all_expanded: if all rows are expanded or not + """ + self.container.expand_all_btn.set_visible(not all_expanded) + self.container.collapse_all_btn.set_visible(all_expanded) + + def on_expand_all_btn_clicked(self, btn): + """Expand all rows on the treeview. + + :param btn: the button that received the clicked event + :type btn: :class:`Gtk.Button` + """ + self.tree_view.expand_all() + + def on_collapse_all_btn_clicked(self, btn): + """Collapse all rows on the treeview. + + :param btn: the button that received the clicked event + :type btn: :class:`Gtk.Button` + """ + self.tree_view.collapse_all() + def on_data_loaded(self, model, total_recs): """Update the total records label. @@ -754,6 +790,17 @@ def _refresh_view(self, update_dict=None, remove_keys=None): self.view.refresh() + # If any of the root rows has children, we should show the + # expand/collapse buttons + if (self.view is self.tree_view and + any(len(row) > 0 for row in self.model.rows)): + self.container.expand_all_btn.set_visible(True) + self.container.collapse_all_btn.set_visible(False) + else: + for widget in [self.container.expand_all_btn, + self.container.collapse_all_btn]: + widget.set_visible(False) + class DataGridView(Gtk.TreeView): @@ -765,6 +812,10 @@ class DataGridView(Gtk.TreeView): """ + __gsignals__ = { + 'all-expanded': (GObject.SignalFlags.RUN_FIRST, None, (bool, )) + } + has_checkboxes = GObject.property(type=bool, default=True) # Column widths @@ -788,7 +839,11 @@ def __init__(self, model, **kwargs): self.set_rules_hint(True) self.active_sort_column = None self.active_sort_column_order = None + + self._expandable_ids = set() self._expanded_ids = set() + self._all_expanded = False + self._block_all_expanded = False ### # Public @@ -805,12 +860,24 @@ def refresh(self): self._setup_columns() - if self._expanded_ids: - def _maybe_expand_row(model, path, iter_): - row_id = model.get_value(iter_, model.id_column_idx) - if row_id in self._expanded_ids: - self.expand_row(path, False) - self.model.foreach(_maybe_expand_row) + self._expandable_ids.clear() + # After refreshing the model, some rows may not be present anymore. + # Let self._expanded_ids be constructed again by the events bellow + expanded_ids = self._expanded_ids.copy() + self._expanded_ids.clear() + + def _find_exapandable_rows(model, path, iter_): + row_id = model.get_value(iter_, model.id_column_idx) + if model.iter_has_child(iter_): + self._expandable_ids.add(row_id) + # If the row was expanded before, expand it again + if row_id in expanded_ids: + self.expand_row(path, False) + + self._block_all_expanded = True + self.model.foreach(_find_exapandable_rows) + self._block_all_expanded = False + self._check_all_expanded() ### # Callbacks @@ -845,6 +912,7 @@ def on_row_expanded(self, treeview, iter_, path): """ self._expanded_ids.add( self.model.get_value(iter_, self.model.id_column_idx)) + self._check_all_expanded() def on_row_collapsed(self, treeview, iter_, path): """Handle row-collapsed events. @@ -860,6 +928,7 @@ def on_row_collapsed(self, treeview, iter_, path): """ self._expanded_ids.discard( self.model.get_value(iter_, self.model.id_column_idx)) + self._check_all_expanded() def on_model_row_changed(self, model, path, iter_): """Track row changes on model. @@ -934,6 +1003,16 @@ def on_select_all_column_clicked(self, check_btn): # Private ### + def _check_all_expanded(self): + """Check expanded rows and maybe emit all-expanded event""" + if self._block_all_expanded: + return + + old_all_expanded = self._all_expanded + self._all_expanded = self._expandable_ids == self._expanded_ids + if self._all_expanded != old_all_expanded: + self.emit('all-expanded', self._all_expanded) + def _setup_columns(self): """Configure the column widgets in the view.""" if self.has_checkboxes: From 2d043942d20af23e6d3d996f724e0a855e266aa2 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 3 Mar 2015 16:39:08 -0300 Subject: [PATCH 11/25] Avoid conversion to str when getting single record This would fail for unicode ids and also is totally unnecessary. --- datagrid_gtk3/db/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 5b3afbb..aa5c131 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -286,7 +286,7 @@ class default table conn.row_factory = sqlite3.Row # Access columns by name with closing(conn.cursor()) as cursor: self._ensure_temp_view(cursor) - cursor.execute(sql_statement, (str(record_id),)) + cursor.execute(sql_statement, (record_id, )) data = cursor.fetchone() # TODO log error if more than one return data From 71838b89ff63d7ef1212f687636608a0a4aed303 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Wed, 4 Mar 2015 17:16:07 -0300 Subject: [PATCH 12/25] Implement lazy loading for hierarchical data This will load the data as soon as it is required for the treeview or for the user requesting it for some reason. --- datagrid_gtk3/db/sqlite.py | 80 +++++++++++------- datagrid_gtk3/ui/grid.py | 167 +++++++++++++++++++++++++++++-------- 2 files changed, 184 insertions(+), 63 deletions(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index aa5c131..66f3c33 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -26,11 +26,30 @@ class Node(list): from :obj:`.data`. :param object data: the data that will be stored in this node + :param int children_len: the number of the children that will + be loaded lazely at some point """ - def __init__(self, data=None): + def __init__(self, data=None, children_len=0): super(Node, self).__init__() + self.data = data + self.children_len = children_len + self.path = None + + def is_children_loaded(self, recursive=False): + """Check if this node's children is loaded + + :param bool recursive: wheather to ask each child if their + children is loaded (and their child too and so on) too. + :returns: `True` if children is loaded, otherwise `False` + :rtype: bool + """ + loaded = len(self) == self.children_len + if recursive: + loaded = (loaded and + all(c.is_children_loaded(recursive=True) for c in self)) + return loaded class SQLiteDataSource(DataSource): @@ -164,36 +183,36 @@ def load(self, params=None): return [] last_page = True if not last_page: - # FIXME: How to properly do lazy loading in this case? if self.PARENT_ID_COLUMN: - def get_results(parent): - operator = 'is' if parent is None else '=' - if where_sql: - parent_where = '%s AND %s %s ?' % ( - where_sql, operator, self.PARENT_ID_COLUMN) - else: - parent_where = ' WHERE %s %s ? ' % ( - self.PARENT_ID_COLUMN, operator) - sql = 'SELECT %s FROM %s %s %s' % ( - self.column_name_str, self.table, - parent_where, order_sql) - - bindings_ = bindings + [parent] - logger.debug('SQL: %s, %s', sql, bindings_) - # FIXME: If would be better to do: - # for row in cursor.execute(sql, bindings_): - # yield row - # But for that we would need different cursors - cursor.execute(sql, bindings_) - return cursor.fetchall() - - def build_tree(parent, parent_id): - for row in get_results(parent_id): - node = Node(data=row) - parent.append(node) - build_tree(node, row[self.id_column_idx]) - - build_tree(rows, None) + # FIXME: We should use sqlalchemy to construct the + # queries in the whole module + parent_id = params.get('parent_id') + operator = 'is' if parent_id is None else '=' + if where_sql: + parent_where = '%s AND %s %s ?' % ( + where_sql, operator, self.PARENT_ID_COLUMN) + else: + parent_where = ' WHERE %s %s ? ' % ( + self.PARENT_ID_COLUMN, operator) + + # Check if the row has any children + count_sql = ( + '(SELECT COUNT(1) FROM %s AS __count ' + ' WHERE __count.%s = __real_table.%s)' % ( + self.table, + self.PARENT_ID_COLUMN, self.ID_COLUMN)) + + columns = ', '.join([self.column_name_str, count_sql]) + sql = 'SELECT %s FROM %s AS __real_table %s %s' % ( + columns, self.table, parent_where, order_sql) + + bindings_ = bindings + [parent_id] + logger.debug('SQL: %s, %s', sql, bindings_) + + for row in cursor.execute(sql, bindings_): + children_len = row.pop(-1) + rows.append( + Node(data=row, children_len=children_len)) else: sql = 'SELECT %s FROM %s %s %s LIMIT %d OFFSET %d' % ( self.column_name_str, self.table, where_sql, @@ -210,6 +229,7 @@ def build_tree(parent, parent_id): cursor.execute(sql, bindings) self.total_recs = int(cursor.fetchone()[0]) + rows.children_len = len(rows) return rows def update(self, params, ids=None): diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 547e285..661ebbb 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -866,16 +866,34 @@ def refresh(self): expanded_ids = self._expanded_ids.copy() self._expanded_ids.clear() - def _find_exapandable_rows(model, path, iter_): - row_id = model.get_value(iter_, model.id_column_idx) - if model.iter_has_child(iter_): - self._expandable_ids.add(row_id) - # If the row was expanded before, expand it again - if row_id in expanded_ids: - self.expand_row(path, False) + self._block_all_expanded = True + # FIXME: This is very optimized, but on some situations (e.g. all paths + # are expanded and the user changed the sort column), this would make + # everything be loaded at the same time. Is there anything we can do + # regarding this issue? + for row_id in expanded_ids: + row = self.model.get_row_by_id(row_id, load_rows=True) + if row is None: + continue + self.expand_to_path(Gtk.TreePath(row.path)) + self._block_all_expanded = False + self._check_all_expanded() + def expand_all(self): + """Expand all expandable rows on the view.""" + # A little optimization to avoid calling _check_all_expanded + # for all rows that will be expanded self._block_all_expanded = True - self.model.foreach(_find_exapandable_rows) + super(DataGridView, self).expand_all() + self._block_all_expanded = False + self._check_all_expanded() + + def collapse_all(self): + """Collapse all expandable rows on the view.""" + # A little optimization to avoid calling _check_all_expanded + # for all rows that will be collapsed + self._block_all_expanded = True + super(DataGridView, self).collapse_all() self._block_all_expanded = False self._check_all_expanded() @@ -910,8 +928,8 @@ def on_row_expanded(self, treeview, iter_, path): :param path: the path pointing to the expanded row :type path: :class:`Gtk.TreePath` """ - self._expanded_ids.add( - self.model.get_value(iter_, self.model.id_column_idx)) + row_id = self.model.get_value(iter_, self.model.id_column_idx) + self._expanded_ids.add(row_id) self._check_all_expanded() def on_row_collapsed(self, treeview, iter_, path): @@ -1003,13 +1021,29 @@ def on_select_all_column_clicked(self, check_btn): # Private ### + def _get_expandable_ids(self): + """Get the ids of the expandable rows.""" + if not self._expandable_ids: + if not self.model.rows.is_children_loaded(recursive=True): + # If there's still anything to load, for sure all + # are not expanded. Return None to avoid confusing it with + # an empty set + return None + + for row in self.model.iter_rows(): + if len(row): + row_id = row.data[self.model.id_column_idx] + self._expandable_ids.add(row_id) + + return self._expandable_ids + def _check_all_expanded(self): """Check expanded rows and maybe emit all-expanded event""" if self._block_all_expanded: return old_all_expanded = self._all_expanded - self._all_expanded = self._expandable_ids == self._expanded_ids + self._all_expanded = self._expanded_ids == self._get_expandable_ids() if self._all_expanded != old_all_expanded: self.emit('all-expanded', self._all_expanded) @@ -1045,7 +1079,8 @@ def _setup_columns(self): self.model.data_source.PARENT_ID_COLUMN, '__selected']) - samples = itertools.islice(self.model.iter_rows(), self.SAMPLE_SIZE) + samples = itertools.islice( + (r.data for r in self.model.iter_rows()), self.SAMPLE_SIZE) for column_index, column in enumerate(self.model.columns): item = column['name'] display = (self.model.display_columns is None @@ -1451,6 +1486,7 @@ def __init__(self, data_source, get_media_callback, decode_fallback, self.encoding_hint = encoding_hint self.selected_cells = list() + self.row_id_mapper = {} self.id_column_idx = None self.parent_column_idx = None self.rows = None @@ -1460,33 +1496,55 @@ def refresh(self): """Refresh the model from the data source.""" if 'page' in self.active_params: del self.active_params['page'] + if 'parent_id' in self.active_params: + del self.active_params['parent_id'] + self.row_id_mapper.clear() self.rows = self.data_source.load(self.active_params) + self.rows.path = () + self.id_column_idx = self.data_source.id_column_idx self.parent_column_idx = self.data_source.parent_column_idx self.total_recs = self.data_source.total_recs + + for i, row in enumerate(self.rows): + row.path = (i, ) + self.row_id_mapper[row.data[self.id_column_idx]] = row + self.emit('data-loaded', self.total_recs) - def add_rows(self): + def add_rows(self, parent_node=None): """Add rows to the model from a new page of data and update the view. :return: True if update took place, False if not :rtype: bool """ - self.active_params['page'] = self.active_params.get('page', 0) + 1 + # When the data is hierarchical, all the root data was already loaded + if parent_node is None and self.parent_column_idx is not None: + return False - # FIXME: Is it right to call row_inserted with the same path/itr - # for all the new rows? - path = (len(self.rows) - 1,) - itr = self.get_iter(path) + if parent_node is not None: + parent_id = parent_node.data[self.id_column_idx] + parent_row = parent_node + else: + parent_id = None + parent_row = self.rows + + self.active_params['parent_id'] = parent_id + # We are not using pages for hierarchical data + if self.parent_column_idx is None: + self.active_params['page'] = self.active_params.get('page', 0) + 1 rows = self.data_source.load(self.active_params) if not len(rows): return False - for row in rows: - self.rows.append(row) - self.row_inserted(path, itr) + for i, row in enumerate(rows): + # FIXME: How to properly call self.row_inserted here? + parent_row.append(row) + row.path = parent_row.path + (i, ) + self.row_id_mapper[row.data[self.id_column_idx]] = row + return True def update_data_source(self, column, value, ids): @@ -1591,7 +1649,7 @@ def set_value(self, itr, column, value, emit_event=True): """ path = self.get_path(itr) # path and iter are the same in this model. - row = self._get_row_by_iter(path) + row = self._get_row_by_path(path) row.data[column] = value id_ = self.get_value(itr, self.id_column_idx) self.update_data_source( @@ -1599,7 +1657,7 @@ def set_value(self, itr, column, value, emit_event=True): if emit_event: self.row_changed(path, itr) - def iter_rows(self): + def iter_rows(self, load_rows=False): """Iterate over the rows of the model. This will iterate using a depth-first algorithm. That means that, @@ -1618,24 +1676,67 @@ def iter_rows(self): [ A, B, E, C, F, G, H, D ] + :param bool load_rows: if we should load rows from the + datasource during the iteration. When using this, be sure + to use lazy iteration and stop when you found what you needed. + :returns: an iterator for the rows + :rtype: generator """ def _iter_children_aux(parent): for row in parent: + if load_rows: + self._ensure_children_is_loaded(row) yield row for inner_row in _iter_children_aux(row): yield inner_row for row in _iter_children_aux(self.rows): - yield row.data + yield row + + if load_rows: + rows_len = len(self.rows) + while self.add_rows(): + for row in _iter_children_aux(self.rows[rows_len:]): + yield row + rows_len = len(self.rows) + + def get_row_by_id(self, row_id, load_rows=False): + """Get a row given its id + + Note that this will load the data from the source until the + row is found, meaning that everything will be loaded on the + worst case (i.e. the row is not present) + + :param object row_id: the id of the row + :returns: the row or ``None`` if it wasn't found + :rtype: :class:`datagrid_gtk3.db.sqlite.Node` + """ + if row_id in self.row_id_mapper: + return self.row_id_mapper[row_id] + + for row in self.iter_rows(load_rows=load_rows): + # Although we could check row, trying self.row_id_mapper has a + # chance of needing less iterations (and thus, less loading from + # sqlite) since after loading all children of A, we can find them + # on self.row_id_mapper without having to load their children too + if row_id in self.row_id_mapper: + return self.row_id_mapper[row_id] ### # Private ### - def _get_row_by_iter(self, iter_): + def _ensure_children_is_loaded(self, row): + if not row.is_children_loaded(): + self.add_rows(row) + + def _get_row_by_path(self, iter_): def get_row_by_iter_aux(iter_aux, rows): + self._ensure_children_is_loaded(rows) if len(iter_aux) == 1: - return rows[iter_aux[0]] + row = rows[iter_aux[0]] + self._ensure_children_is_loaded(row) + return row return get_row_by_iter_aux(iter_aux[1:], rows[iter_aux[0]]) return get_row_by_iter_aux(iter_, self.rows) @@ -1832,7 +1933,7 @@ def on_get_iter(self, path): try: # row and path are the same in this model. We just need # to make sure that the iter is valid - self._get_row_by_iter(path) + self._get_row_by_path(path) except IndexError: return None else: @@ -1847,7 +1948,7 @@ def on_get_value(self, rowref, column): else: visible = True - row = self._get_row_by_iter(rowref) + row = self._get_row_by_path(rowref) raw = row.data[column] # Don't format value for id and parent columns. They are not displayed # on the grid and we may need their full values to get their records @@ -1868,7 +1969,7 @@ def on_iter_next(self, rowref): next_value = (rowref[0] + 1, ) else: parentref = rowref[:-1] - rows = self._get_row_by_iter(parentref) + rows = self._get_row_by_path(parentref) next_value = parentref + (rowref[-1] + 1, ) if not next_value[-1] < len(rows): @@ -1881,7 +1982,7 @@ def on_iter_children(self, rowref): if rowref is None: return (0, ) - parent_row = self._get_row_by_iter(rowref) + parent_row = self._get_row_by_path(rowref) if not len(parent_row): return None @@ -1889,14 +1990,14 @@ def on_iter_children(self, rowref): def on_iter_has_child(self, rowref): """Return true if this node has children.""" - return bool(self._get_row_by_iter(rowref)) + return bool(self._get_row_by_path(rowref)) def on_iter_n_children(self, rowref): """Return the number of children of this node.""" if rowref is None: return len(self.rows) - return len(self._get_row_by_iter(rowref)) + return len(self._get_row_by_path(rowref)) def on_iter_nth_child(self, parent, n): """Return the nth child of this node.""" @@ -1904,7 +2005,7 @@ def on_iter_nth_child(self, parent, n): parent = () rows = self.rows else: - rows = self._get_row_by_iter(parent) + rows = self._get_row_by_path(parent) if not 0 <= n < len(rows): return None From 8aca8f2f949e4073c47e0d435224fa350bc790d4 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Wed, 4 Mar 2015 17:32:32 -0300 Subject: [PATCH 13/25] Fix operator order on existing where clauses --- datagrid_gtk3/db/sqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 66f3c33..bf78eca 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -190,7 +190,7 @@ def load(self, params=None): operator = 'is' if parent_id is None else '=' if where_sql: parent_where = '%s AND %s %s ?' % ( - where_sql, operator, self.PARENT_ID_COLUMN) + where_sql, self.PARENT_ID_COLUMN, operator) else: parent_where = ' WHERE %s %s ? ' % ( self.PARENT_ID_COLUMN, operator) From dbc921cd96fbaa5c3ea3d4c84bab401e2fde14ca Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Thu, 5 Mar 2015 16:24:33 -0300 Subject: [PATCH 14/25] Allow to add optional attribute filters --- datagrid_gtk3/db/sqlite.py | 113 +++++++++++++++++++------- datagrid_gtk3/ui/glade/datagrid.glade | 15 +++- datagrid_gtk3/ui/grid.py | 82 ++++++++++++++++++- 3 files changed, 178 insertions(+), 32 deletions(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index bf78eca..2281436 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -184,35 +184,10 @@ def load(self, params=None): last_page = True if not last_page: if self.PARENT_ID_COLUMN: - # FIXME: We should use sqlalchemy to construct the - # queries in the whole module - parent_id = params.get('parent_id') - operator = 'is' if parent_id is None else '=' - if where_sql: - parent_where = '%s AND %s %s ?' % ( - where_sql, self.PARENT_ID_COLUMN, operator) - else: - parent_where = ' WHERE %s %s ? ' % ( - self.PARENT_ID_COLUMN, operator) - - # Check if the row has any children - count_sql = ( - '(SELECT COUNT(1) FROM %s AS __count ' - ' WHERE __count.%s = __real_table.%s)' % ( - self.table, - self.PARENT_ID_COLUMN, self.ID_COLUMN)) - - columns = ', '.join([self.column_name_str, count_sql]) - sql = 'SELECT %s FROM %s AS __real_table %s %s' % ( - columns, self.table, parent_where, order_sql) - - bindings_ = bindings + [parent_id] - logger.debug('SQL: %s, %s', sql, bindings_) - - for row in cursor.execute(sql, bindings_): - children_len = row.pop(-1) - rows.append( - Node(data=row, children_len=children_len)) + rows.extend( + self._load_tree_rows( + cursor, where_sql, bindings, order_sql, + params.get('parent_id'))) else: sql = 'SELECT %s FROM %s %s %s LIMIT %d OFFSET %d' % ( self.column_name_str, self.table, where_sql, @@ -550,6 +525,86 @@ def get_columns(self): return cols + def _load_tree_rows(self, cursor, where_sql, bindings, order_sql, parent_id): + """Load rows as a tree.""" + # FIXME: We should use sqlalchemy to construct the queries here, + # but for that _get_where_clause needs to be adapted. + def get_rows(columns, where, bindings_): + sql = 'SELECT %s FROM %s AS __real_table %s %s' % ( + columns, self.table, where, order_sql) + + logger.debug('SQL: %s, %s', sql, bindings_) + + for row in cursor.execute(sql, bindings_): + yield row + + if where_sql: + # FIXME: If we have a where clause, we cant load the results lazily + # because, we don't know if a row's children/grandchildren/etc will + # match. If this optimization (loading the leafs and the necessary + # parents until the root) good enough? + children = {} + node_mapper = {} + + def load_rows(where, bindings_): + columns = self.column_name_str + for row in get_rows(columns, where, bindings_): + row_id = row[self.id_column_idx] + if row_id in node_mapper: + continue + + c_list = children.setdefault( + row[self.parent_column_idx], []) + node = Node(data=row) + c_list.append(node) + node_mapper[row_id] = node + + load_rows(where_sql, bindings) + if not children: + return + + # Load parents incrementally until we are left with the root + while children.keys() != [None]: + parents_to_load = [] + for parent, c_list in children.items(): + if parent is None: + continue + + node = node_mapper.get(parent, None) + if node is None: + parents_to_load.append(parent) + continue + + node.extend(c_list) + node.children_len = len(node) + del children[parent] + + if parents_to_load: + # FIXME: Shouldn't cursor.execute handle this? + marker = ','.join('?' for p in parents_to_load) + where = 'WHERE %s IN (%s)' % (self.ID_COLUMN, marker) + load_rows(where, parents_to_load) + + for node in children[None]: + yield node + else: + # If there's no where clause, we can load the results lazily + operator = 'is' if parent_id is None else '=' + parent_where = ' WHERE %s %s ? ' % ( + self.PARENT_ID_COLUMN, operator) + + # Check if the row has any children + count_sql = ( + '(SELECT COUNT(1) FROM %s AS __count ' + ' WHERE __count.%s = __real_table.%s)' % ( + self.table, self.PARENT_ID_COLUMN, self.ID_COLUMN)) + columns = ', '.join([self.column_name_str, count_sql]) + + bindings_ = bindings + [parent_id] + for row in get_rows(columns, parent_where, bindings_): + children_len = row.pop(-1) + yield Node(data=row, children_len=children_len) + def rank(matchinfo): """Rank full-text search results. diff --git a/datagrid_gtk3/ui/glade/datagrid.glade b/datagrid_gtk3/ui/glade/datagrid.glade index 638ca95..5a53aff 100644 --- a/datagrid_gtk3/ui/glade/datagrid.glade +++ b/datagrid_gtk3/ui/glade/datagrid.glade @@ -81,7 +81,18 @@ - + + False + 5 + + + + + + False + True + 4 + @@ -184,7 +195,7 @@ - + True False vertical diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 661ebbb..1365bbd 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -40,6 +40,10 @@ # repeat the lastest value read in a row for that column NO_IMAGE_PIXBUF = _no_image_loader.get_pixbuf() +# Used to represent "no option selected" on filters. We use this instead of +# None as it can be a valid value for filtering. +NO_FILTER_OPTION = object() + class OptionsPopup(Gtk.Window): @@ -378,6 +382,7 @@ def __init__(self, container, data_source, selected_record_callback=None, if get_full_path is None: get_full_path = default_get_full_path + self.extra_filter_widgets = {} self.container = container self.decode_fallback = decode_fallback @@ -449,6 +454,10 @@ def __init__(self, container, data_source, selected_record_callback=None, self.bind_datasource(data_source) + ### + # Public + ### + def bind_datasource(self, data_source): """Binds a data source to the datagrid. @@ -498,7 +507,7 @@ def bind_datasource(self, data_source): self.container.label_date_to, self.container.image_end_date, self.container.vbox_end_date, - self.container.vseparator2 + self.container.filters_separator, ) if len(liststore_date_cols) == 0: for widget in widgets: @@ -509,6 +518,53 @@ def bind_datasource(self, data_source): self._refresh_view() + def add_options_filter(self, attr, options, add_empty_option=True): + """Add optional options filter for attr. + + :param str attr: the attr that will be filtered + :param iterable options: the options that will be displayed + on the filter as a tuple with (label, option). + The label will be displayed as on the combo and the + option will be used to generate the WHERE clause + :param bool add_empty_option: if we should add an empty + option as the first option in the combo. Its label will + be the label of the column in question and selecting + it will be the same as not filtering by that attr. + :returns: the newly created combobox + :rtype: :class:`Gtk.ComboBox` + """ + for col_dict in self.model.columns: + if col_dict['name'] == attr: + label = col_dict['display'] + break + else: + raise ValueError + + model = Gtk.ListStore(str, object) + if add_empty_option: + model.append((label, NO_FILTER_OPTION)) + for option in options: + model.append(option) + + combo = Gtk.ComboBox() + combo.set_model(model) + renderer = Gtk.CellRendererText() + combo.pack_start(renderer, True) + combo.add_attribute(renderer, 'text', 0) + combo.set_active(0) + + combo.connect('changed', self.on_filter_changed, attr) + + self.extra_filter_widgets[attr] = combo + self.container.extra_filters.pack_start( + combo, expand=False, fill=False, padding=0) + self.container.extra_filters.show_all() + # Make sure this separator is visible. It may have been hidden + # if we don't have any datetime columns. + self.container.filters_separator.show() + + return combo + ### # Callbacks ### @@ -681,6 +737,30 @@ def on_collapse_all_btn_clicked(self, btn): """ self.tree_view.collapse_all() + def on_filter_changed(self, combo, attr): + """Handle selection changed on filter comboboxes. + + :param combo: the combo that received the signal + :type combo: :class:`Gtk.ComboBox` + :param str attr: the name of the attr to filter + """ + model = combo.get_model() + value = model[combo.get_active()][1] + + if value is NO_FILTER_OPTION: + remove_keys = [attr] + update_dict = None + else: + remove_keys = None + update_dict = { + attr: { + 'operator': 'is' if value is None else '=', + 'param': value, + } + } + + self._refresh_view(update_dict=update_dict, remove_keys=remove_keys) + def on_data_loaded(self, model, total_recs): """Update the total records label. From b4dbc0019edef2980b970fb5ae92d2f5059eea77 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Thu, 5 Mar 2015 18:13:31 -0300 Subject: [PATCH 15/25] Use sqlalchemy to construct queries --- datagrid_gtk3/db/sqlite.py | 362 +++++++++++++++++++------------------ datagrid_gtk3/ui/grid.py | 30 +-- 2 files changed, 201 insertions(+), 191 deletions(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 2281436..e580346 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -1,5 +1,7 @@ """SQLite database backend.""" + import logging +import operator import sqlite3 import struct from contextlib import closing @@ -12,10 +14,32 @@ inspect, ) from sqlalchemy.exc import DatabaseError +from sqlalchemy.sql import ( + alias, + and_, + column, + desc, + func, + select, + table as table_, +) from datagrid_gtk3.db import DataSource logger = logging.getLogger(__name__) +_compile = lambda q: q.compile(compile_kwargs={"literal_binds": True}).string + +_OPERATOR_MAPPER = { + 'is': operator.eq, + '=': operator.eq, + '!=': operator.ne, + '<': operator.lt, + '<=': operator.le, + '<': operator.lt, + '<=': operator.le, + '>': operator.gt, + '>=': operator.ge, +} class Node(list): @@ -98,12 +122,13 @@ def __init__(self, db_file, table=None, update_table=None, config=None, """Process database column info.""" assert table or query # either table or query must be given self.db_file = db_file - self.table = table if table else "__CustomQueryTempView" + self.table = table_(table if table else "__CustomQueryTempView") self.query = query if query: logger.debug("Custom SQL: %s", query) self._ensure_selected_column = ensure_selected_column self.display_all = display_all + # FIXME: Use sqlalchemy for queries using update_table if update_table is not None: self.update_table = update_table else: @@ -111,11 +136,15 @@ def __init__(self, db_file, table=None, update_table=None, config=None, self.config = config self.id_column_idx = None self.parent_column_idx = None - self.columns = self.get_columns() - column_names = ['"%s"' % col['name'] for col in self.columns] - self.column_name_str = ', '.join(column_names) + self.columns = self._get_columns() + for col in self.columns: + self.table.append_column(column(col['name'])) self.total_recs = None + ### + # Public + ### + def load(self, params=None): """Execute SQL ``SELECT`` and populate ``rows`` attribute. @@ -141,9 +170,33 @@ def load(self, params=None): :param dict params: dict of various parameters from which to construct additional SQL clauses eg. ``WHERE``, ``ORDER BY``, etc. """ - first_access = True - last_page = False - offset = 0 + rows = Node() + # FIXME: Maybe we should use kwargs instead of params? + params = params or {} + + # WHERE + where = params.get('where', None) + if where is not None: + where = self._get_where_clause(where) + + # ORDER BY + order_by = params.get('order_by', None) + order_by = order_by and self.table.columns[order_by] + if order_by is not None and params.get('desc', False): + order_by = desc(order_by) + + # OFFSET + page = params.get('page', 0) + # FIXME: If we have a PARENT_ID_COLUMN, all results + # were loaded on first load. How to handle this better? + if page > 0 and self.PARENT_ID_COLUMN is not None: + return rows + offset = page * self.MAX_RECS + # A little optimization to avoid doing more queries when we + # already loaded everything + if self.total_recs is not None and offset >= self.total_recs: + return rows + with closing(sqlite3.connect(self.db_file)) as conn: conn.row_factory = lambda cursor, row: list(row) # ^^ make result lists mutable so we can change values in @@ -152,60 +205,27 @@ def load(self, params=None): # TODO: ^^ only if search term in params with closing(conn.cursor()) as cursor: self._ensure_temp_view(cursor) - bindings = [] - where_sql = '' - order_sql = '' - rows = Node() - if params: - # construct WHERE clause - if 'where' in params: - where_sql, bindings = self._get_where_clause( - self.table, params['where']) - # construct ORDER BY clause - if 'order_by' in params: - order_sql = order_sql + ' ORDER BY "%s"' % \ - params['order_by'] - if 'desc' in params: - if params['desc'] is True: - order_sql = order_sql + ' DESC' - # determine OFFSET value for paging - if 'page' in params: - # FIXME: If we have a PARENT_ID_COLUMN, all results - # were loaded on first load. How to handle this better? - if self.PARENT_ID_COLUMN: - return [] - first_access = False - if params['page']: - offset = params['page'] * self.MAX_RECS - if offset >= self.total_recs: - # at end of total records, return no records - # for paging - return [] - last_page = True - if not last_page: - if self.PARENT_ID_COLUMN: - rows.extend( - self._load_tree_rows( - cursor, where_sql, bindings, order_sql, - params.get('parent_id'))) - else: - sql = 'SELECT %s FROM %s %s %s LIMIT %d OFFSET %d' % ( - self.column_name_str, self.table, where_sql, - order_sql, self.MAX_RECS, offset) - logger.debug('SQL: %s, %s', sql, bindings) - for row in cursor.execute(sql, bindings): - rows.append(Node(data=row)) - - if first_access: - # set the total record count the only the first time the - # record set is requested - sql = 'SELECT COUNT(*) FROM %s %s' % ( - self.table, where_sql) - cursor.execute(sql, bindings) - self.total_recs = int(cursor.fetchone()[0]) - - rows.children_len = len(rows) - return rows + + if page == 0: + # set the total record count the only the first time the + # record set is requested + res = self.select( + conn, self.table, [func.count(1)], where=where) + self.total_recs = int(list(res)[0][0]) + + if self.PARENT_ID_COLUMN: + rows.extend( + self._load_tree_rows( + conn, where, order_by, params.get('parent_id', None))) + else: + query = self.select( + conn, self.table, self.table.columns, where=where, + limit=self.MAX_RECS, offset=offset, order_by=order_by) + for row in query: + rows.append(Node(data=row)) + + rows.children_len = len(rows) + return rows def update(self, params, ids=None): """Update the recordset with a SQL ``UPDATE`` statement. @@ -218,6 +238,7 @@ def update(self, params, ids=None): :param dict params: keys corresponding to DB columns + values to update :param list ids: database primary keys to use for updating """ + # FIXME: Use sqlalchemy to construct the queries here with closing(sqlite3.connect(self.db_file)) as conn: with closing(conn.cursor()) as cursor: update_sql_list = [] @@ -246,45 +267,33 @@ def get_all_record_ids(self, params=None): :return: primary key ids :rtype: list """ - bindings = [] - where_sql = '' - # construct WHERE clause - if params is not None: - if 'where' in params: - where_sql, bindings = self._get_where_clause( - self.table, params['where']) - sql = 'SELECT %s FROM %s %s' % (self.ID_COLUMN, self.table, where_sql) with closing(sqlite3.connect(self.db_file)) as conn: conn.create_function('rank', 1, rank) # TODO: ^^ create this function only if search term in params - with closing(conn.cursor()) as cursor: - self._ensure_temp_view(cursor) - cursor.execute(sql, bindings) - results = [row[0] for row in cursor.fetchall()] - return results + where = params and params.get('where', None) + if where is not None: + where = self._get_where_clause(where) + res = self.select( + conn, self.table, + [self.table.columns[self.ID_COLUMN]], where=where) - def get_single_record(self, record_id, table=None): + return [row[0] for row in res] + + def get_single_record(self, record_id): """Get single record from database for display in preview pane. :param int record_id: required record number to be retrieved - :param str table: optional string table name to retrieve from if not - class default table :return: row of data :rtype: tuple """ - if table is None: - table = self.table - sql_statement = 'SELECT * FROM %s WHERE %s = ?' % ( - table, self.ID_COLUMN - ) with closing(sqlite3.connect(self.db_file)) as conn: conn.row_factory = sqlite3.Row # Access columns by name - with closing(conn.cursor()) as cursor: - self._ensure_temp_view(cursor) - cursor.execute(sql_statement, (record_id, )) - data = cursor.fetchone() + res = list(self.select( + conn, self.table, self.table.columns, + where=self.table.columns[self.ID_COLUMN] == record_id)) + # TODO log error if more than one - return data + return res[0] def get_selected_columns(self): """Get selected columns info from DB. @@ -292,14 +301,18 @@ def get_selected_columns(self): :returns: list of column names :rtype: list or None """ - result = self.select( - self.db_file, - '_selected_columns', - None, - {'tablename': {'param': self.table, 'operator': '='}} - ) - if not result: - return None + table = '_selected_columns' + where = self._get_where_clause( + {'tablename': {'param': self.table, 'operator': '='}}) + + with closing(sqlite3.connect(self.db_file)) as conn: + conn.row_factory = sqlite3.Row # Access columns by name + try: + result = list(self.select(conn, table, where=where)) + except sqlite3.OperationalError as err: + # FIXME: When will this happen? + logger.warn(str(err)) + return return result[0][1].split(',') # ^^ 2nd column of returned row; first column is table name @@ -312,6 +325,7 @@ def update_selected_columns(self, columns): :param list columns: list of column names to display """ + # FIXME: Use sqlalchemy to construct the queries here with closing(sqlite3.connect(self.db_file)) as conn: with closing(conn.cursor()) as cursor: create_sql = ( @@ -345,8 +359,8 @@ def update_selected_columns(self, columns): cursor.execute(update_sql, params) conn.commit() - @classmethod - def select(cls, db_file, table, columns=None, where=None): + def select(self, conn, table, columns, where=None, + order_by=None, limit=None, offset=None): """Select records from given db and table given columns and criteria. :param str db_file: path to SQLite database file @@ -354,32 +368,28 @@ def select(cls, db_file, table, columns=None, where=None): :param list columns: list of columns to SELECT from :param dict where: dict of parameters to build ``WHERE`` clause """ - # TODO: make this an instance method to avoid having to pass db_file? - where_sql = '' - where_params = [] - if columns: - columns = ', '.join(columns) - else: - columns = '*' - if where is not None: - where_sql, where_params = \ - cls._get_where_clause(table, where) - sql = 'SELECT %s FROM %s %s' % (columns, table, where_sql) - logger.debug(sql) - with closing(sqlite3.connect(db_file)) as conn: - conn.row_factory = sqlite3.Row # Access columns by name - with closing(conn.cursor()) as cursor: - try: - cursor.execute(sql, where_params) - except sqlite3.OperationalError as err: - logger.warn(str(err)) - data = [] - else: - data = cursor.fetchall() - return data + sql = select( + columns=columns, whereclause=where, + from_obj=[table], order_by=order_by) + sql_str = _compile(sql) + + # XXX: How to make sqlalchemy use limit/offset right? It is not + # replacing the values on _compile + if limit is not None: + sql_str += '\nLIMIT %s' % (limit, ) + if offset is not None: + sql_str += '\nOFFSET %s' % (offset, ) + + logger.debug('SQL:\n%s', sql_str) + with closing(conn.cursor()) as cursor: + for row in cursor.execute(sql_str): + yield row - @classmethod - def _get_where_clause(cls, table, where_params): + ### + # Private + ### + + def _get_where_clause(self, where_params): """Construct a SQL ``WHERE`` clause. A typical ``where_params`` dict might look like this:: @@ -393,43 +403,38 @@ def _get_where_clause(cls, table, where_params): :rtype: tuple """ sql_clauses = [] - params = [] for key, value in where_params.iteritems(): dic = value if key == 'search': # full-text search # TODO: make this generic, not specific to vE implementation if dic['param']: + table = self.table.name + '_search' + # XXX: This is to make MATCH be compiled direct here. + # We should build this query using sqlalchemy instead + match = column(table).match(value['param']) + sql = '(%s IN (%s)' % ( - cls.ID_COLUMN, + self.ID_COLUMN, 'SELECT %(id)s FROM ' '(SELECT rank(matchinfo(%(table)s)) AS r, %(id)s' - ' FROM %(table)s WHERE %(table)s MATCH ?)' + ' FROM %(table)s WHERE %(match)s)' ' WHERE r > 0 ORDER BY r DESC)' % { - "id": cls.ID_COLUMN, - "table": table + '_search' + "id": self.ID_COLUMN, + "table": table, + "match": _compile(match), } ) sql_clauses.append(sql) - params.append(dic['param']) elif dic['operator'] == 'range': - sql = '(%(col)s >= ? AND %(col)s <= ?)' % {'col': key} - sql_clauses.append(sql) - params.append(dic['param'][0]) - params.append(dic['param'][1]) + sql_clauses.append( + self.table.columns[key].between(*value['param'])) else: - sql = '(%s %s ?)' % (key, dic['operator']) - sql_clauses.append(sql) - params.append(dic['param']) + clause = _OPERATOR_MAPPER[value['operator']]( + self.table.columns[key], value['param']) + sql_clauses.append(clause) - if not sql_clauses: - return ('', []) - - if len(sql_clauses) > 1: - sql = 'WHERE %s' % (' AND '.join(sql_clauses)) - else: - sql = 'WHERE %s' % sql_clauses[0] - return (sql, params) + return and_(*sql_clauses) def _ensure_temp_view(self, cursor): """If a custom query is defined, temporary view using that query @@ -444,7 +449,7 @@ def _ensure_temp_view(self, cursor): self.table, self.query )) - def get_columns(self): + def _get_columns(self): """Return a list of column information dicts. Queries either the database ``PRAGMA`` for column information or @@ -466,7 +471,7 @@ def get_columns(self): with closing(sqlite3.connect(self.db_file)) as conn: with closing(conn.cursor()) as cursor: self._ensure_temp_view(cursor) - table_info_query = 'PRAGMA table_info(%s)' % self.table + table_info_query = 'PRAGMA table_info(%s)' % self.table.name cursor.execute(table_info_query) rows = cursor.fetchall() has_selected = False @@ -525,20 +530,9 @@ def get_columns(self): return cols - def _load_tree_rows(self, cursor, where_sql, bindings, order_sql, parent_id): + def _load_tree_rows(self, conn, where, order_by, parent_id): """Load rows as a tree.""" - # FIXME: We should use sqlalchemy to construct the queries here, - # but for that _get_where_clause needs to be adapted. - def get_rows(columns, where, bindings_): - sql = 'SELECT %s FROM %s AS __real_table %s %s' % ( - columns, self.table, where, order_sql) - - logger.debug('SQL: %s, %s', sql, bindings_) - - for row in cursor.execute(sql, bindings_): - yield row - - if where_sql: + if where is not None: # FIXME: If we have a where clause, we cant load the results lazily # because, we don't know if a row's children/grandchildren/etc will # match. If this optimization (loading the leafs and the necessary @@ -546,9 +540,11 @@ def get_rows(columns, where, bindings_): children = {} node_mapper = {} - def load_rows(where, bindings_): - columns = self.column_name_str - for row in get_rows(columns, where, bindings_): + def load_rows(where_): + query = self.select( + conn, self.table, columns=self.table.columns, + where=where_, order_by=order_by) + for row in query: row_id = row[self.id_column_idx] if row_id in node_mapper: continue @@ -559,7 +555,7 @@ def load_rows(where, bindings_): c_list.append(node) node_mapper[row_id] = node - load_rows(where_sql, bindings) + load_rows(where) if not children: return @@ -580,28 +576,36 @@ def load_rows(where, bindings_): del children[parent] if parents_to_load: - # FIXME: Shouldn't cursor.execute handle this? - marker = ','.join('?' for p in parents_to_load) - where = 'WHERE %s IN (%s)' % (self.ID_COLUMN, marker) - load_rows(where, parents_to_load) + where = self.table.columns[self.ID_COLUMN].in_( + parents_to_load) + load_rows(where) for node in children[None]: yield node else: # If there's no where clause, we can load the results lazily - operator = 'is' if parent_id is None else '=' - parent_where = ' WHERE %s %s ? ' % ( - self.PARENT_ID_COLUMN, operator) - - # Check if the row has any children - count_sql = ( - '(SELECT COUNT(1) FROM %s AS __count ' - ' WHERE __count.%s = __real_table.%s)' % ( - self.table, self.PARENT_ID_COLUMN, self.ID_COLUMN)) - columns = ', '.join([self.column_name_str, count_sql]) - - bindings_ = bindings + [parent_id] - for row in get_rows(columns, parent_where, bindings_): + where = self.table.columns[self.PARENT_ID_COLUMN] == parent_id + + count_table = alias(self.table, '__count') + # We could use the comparison between the columns, but that would + # make sqlalchemy add self.table in the FROM clause, which + # would produce wrong results. + count_where = '%s.%s = %s.%s' % ( + count_table.name, self.PARENT_ID_COLUMN, + self.table.name, self.ID_COLUMN) + count_select = select( + [func.count(1)], + whereclause=count_where, from_obj=[count_table]) + + columns = self.table.columns.values() + # We have to compile this here or else sqlalchemy would put + # this inside the FROM part. + columns.append('(%s)' % (_compile(count_select), )) + query = self.select( + conn, self.table, columns=columns, + where=where, order_by=order_by) + + for row in query: children_len = row.pop(-1) yield Node(data=row, children_len=children_len) diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 1365bbd..0b7a512 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -1579,6 +1579,7 @@ def refresh(self): if 'parent_id' in self.active_params: del self.active_params['parent_id'] + self.data_source.total_recs = None self.row_id_mapper.clear() self.rows = self.data_source.load(self.active_params) self.rows.path = () @@ -1603,27 +1604,33 @@ def add_rows(self, parent_node=None): if parent_node is None and self.parent_column_idx is not None: return False - if parent_node is not None: - parent_id = parent_node.data[self.id_column_idx] - parent_row = parent_node - else: + if parent_node is None: parent_id = None parent_row = self.rows - - self.active_params['parent_id'] = parent_id - # We are not using pages for hierarchical data - if self.parent_column_idx is None: + path_offset = self.rows[-1].path[-1] + 1 + # We are not using pages for hierarchical data self.active_params['page'] = self.active_params.get('page', 0) + 1 + else: + parent_id = parent_node.data[self.id_column_idx] + parent_row = parent_node + path_offset = 0 + self.active_params['parent_id'] = parent_id rows = self.data_source.load(self.active_params) if not len(rows): return False for i, row in enumerate(rows): - # FIXME: How to properly call self.row_inserted here? - parent_row.append(row) - row.path = parent_row.path + (i, ) + row.path = parent_row.path + (path_offset + i, ) self.row_id_mapper[row.data[self.id_column_idx]] = row + parent_row.append(row) + + # FIXME: Non-hierarchical data need this to display the new row, + # but hierarchical ones not only will work without this, but will + # produce warnings if we try to call this for them. + if self.parent_column_idx is None: + path = Gtk.TreePath(row.path) + self.row_inserted(path, self.get_iter(path)) return True @@ -1812,7 +1819,6 @@ def _ensure_children_is_loaded(self, row): def _get_row_by_path(self, iter_): def get_row_by_iter_aux(iter_aux, rows): - self._ensure_children_is_loaded(rows) if len(iter_aux) == 1: row = rows[iter_aux[0]] self._ensure_children_is_loaded(row) From 29cc3495a742185fc747e383efc86d04d62e4d70 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Fri, 6 Mar 2015 18:00:02 -0300 Subject: [PATCH 16/25] Add flat view --- datagrid_gtk3/db/sqlite.py | 14 +++++++- datagrid_gtk3/ui/grid.py | 70 +++++++++++++++++++++++++++----------- 2 files changed, 63 insertions(+), 21 deletions(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index e580346..471619e 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -116,6 +116,7 @@ class SQLiteDataSource(DataSource): } ID_COLUMN = 'rowid' PARENT_ID_COLUMN = None + FLAT_COLUMN = None def __init__(self, db_file, table=None, update_table=None, config=None, ensure_selected_column=True, display_all=False, query=None): @@ -136,6 +137,7 @@ def __init__(self, db_file, table=None, update_table=None, config=None, self.config = config self.id_column_idx = None self.parent_column_idx = None + self.flat_column_idx = None self.columns = self._get_columns() for col in self.columns: self.table.append_column(column(col['name'])) @@ -197,6 +199,12 @@ def load(self, params=None): if self.total_recs is not None and offset >= self.total_recs: return rows + # Flat + flat = params.get('flat', False) + if flat: + flat_where = operator.ne(self.table.columns[self.FLAT_COLUMN], None) + where = and_(where, flat_where) if where is not None else flat_where + with closing(sqlite3.connect(self.db_file)) as conn: conn.row_factory = lambda cursor, row: list(row) # ^^ make result lists mutable so we can change values in @@ -213,7 +221,7 @@ def load(self, params=None): conn, self.table, [func.count(1)], where=where) self.total_recs = int(list(res)[0][0]) - if self.PARENT_ID_COLUMN: + if self.PARENT_ID_COLUMN and not flat: rows.extend( self._load_tree_rows( conn, where, order_by, params.get('parent_id', None))) @@ -500,6 +508,8 @@ def _get_columns(self): self.id_column_idx = i if col_name == self.PARENT_ID_COLUMN: self.parent_column_idx = i + if col_name == self.FLAT_COLUMN: + self.flat_column_idx = i if row[1] == '__selected': col_dict['transform'] = 'boolean' @@ -527,6 +537,8 @@ def _get_columns(self): self.id_column_idx += 1 if has_selected and self.parent_column_idx is not None: self.parent_column_idx += 1 + if has_selected and self.flat_column_idx is not None: + self.flat_column_idx += 1 return cols diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 0b7a512..90e5e37 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -59,7 +59,8 @@ class OptionsPopup(Gtk.Window): MAX_HEIGHT = 500 (VIEW_TREE, - VIEW_ICON) = range(2) + VIEW_FLAT, + VIEW_ICON) = range(3) __gsignals__ = { 'column-visibility-changed': (GObject.SignalFlags.RUN_FIRST, @@ -194,20 +195,37 @@ def _get_position(self): def _get_view_options(self): """Build view options for datagrid.""" - tv_radio = Gtk.RadioButton(label='Tree View') - yield tv_radio + iters = {} + model = Gtk.ListStore(str, int) - iv_radio = Gtk.RadioButton(label='Icon View', group=tv_radio) - # We can only change to icon view if we have at least one column - # with 'image' transformation. - iv_radio.set_sensitive( - any(c['transform'] == 'image' - for c in self._controller.model.columns)) + iters[self.VIEW_TREE] = model.append(("Tree View", self.VIEW_TREE)) - is_iconview = isinstance(self._controller.view, DataGridIconView) - iv_radio.set_active(is_iconview) - tv_radio.connect('toggled', self.on_treeview_radio_toggled) - yield iv_radio + if self._controller.model.flat_column_idx is not None: + iters[self.VIEW_FLAT] = model.append(("Flat View", self.VIEW_FLAT)) + + if any(c['transform'] == 'image' + for c in self._controller.model.columns): + iters[self.VIEW_ICON] = model.append(("Icon View", self.VIEW_ICON)) + + combo = Gtk.ComboBox() + combo.set_model(model) + renderer = Gtk.CellRendererText() + combo.pack_start(renderer, True) + combo.add_attribute(renderer, 'text', 0) + + if isinstance(self._controller.view, DataGridView): + if self._controller.model.active_params.get('flat', False): + combo.set_active_iter(iters[self.VIEW_FLAT]) + else: + combo.set_active_iter(iters[self.VIEW_TREE]) + elif isinstance(self._controller.view, DataGridIconView): + combo.set_active_iter(iters[self.VIEW_ICON]) + else: + raise AssertionError("Unknown view type %r" % ( + self._controller.view, )) + + combo.connect('changed', self.on_combo_view_changed) + yield combo def _get_visibility_options(self): """Construct the switches based on the actual model columns.""" @@ -273,13 +291,17 @@ def on_button_press_event(self, window, event): if not intersection[0]: self.popdown() - def on_treeview_radio_toggled(self, widget): - """Handle changes on the views radio. + def on_combo_view_changed(self, widget): + """Handle changes on the view combo. + + Emit 'view-changed' for the given view. - Emit 'view-changed' when the radio selection changes + :param widget: the combobox that received the event + :type widget: :class:`Gtk.ComboBox` """ - self.emit('view-changed', - self.VIEW_TREE if widget.get_active() else self.VIEW_ICON) + model = widget.get_model() + value = model[widget.get_active()][1] + self.emit('view-changed', value) self.popdown() def on_toggle_button_toggled(self, widget): @@ -619,13 +641,17 @@ def on_popup_view_changed(self, popup, new_view): self.view = self.icon_view self.model.image_max_size = 100.0 self.model.image_draw_border = True - elif new_view == OptionsPopup.VIEW_TREE: + elif new_view in [OptionsPopup.VIEW_TREE, OptionsPopup.VIEW_FLAT]: self.view = self.tree_view self.model.image_max_size = 24.0 self.model.image_draw_border = False else: raise AssertionError("Unrecognized option %r" % (new_view, )) + # We want flat for both flat view and icon + self.model.active_params['flat'] = new_view in [ + OptionsPopup.VIEW_FLAT, OptionsPopup.VIEW_ICON] + child = self.container.grid_scrolledwindow.get_child() self.container.grid_scrolledwindow.remove(child) self.container.grid_scrolledwindow.add(self.view) @@ -1158,6 +1184,8 @@ def _setup_columns(self): self.model.data_source.ID_COLUMN, self.model.data_source.PARENT_ID_COLUMN, '__selected']) + if not self.model.active_params.get('flat', False): + dont_display.add(self.model.data_source.FLAT_COLUMN) samples = itertools.islice( (r.data for r in self.model.iter_rows()), self.SAMPLE_SIZE) @@ -1551,7 +1579,7 @@ def __init__(self, data_source, get_media_callback, decode_fallback, self._invisible_images = {} self._fallback_images = {} self.visible_range = None - self.active_params = {} + self.active_params = {'flat': False} self.data_source = data_source self.get_media_callback = get_media_callback self.decode_fallback = decode_fallback @@ -1569,6 +1597,7 @@ def __init__(self, data_source, get_media_callback, decode_fallback, self.row_id_mapper = {} self.id_column_idx = None self.parent_column_idx = None + self.flat_column_idx = None self.rows = None self.total_recs = None @@ -1586,6 +1615,7 @@ def refresh(self): self.id_column_idx = self.data_source.id_column_idx self.parent_column_idx = self.data_source.parent_column_idx + self.flat_column_idx = self.data_source.flat_column_idx self.total_recs = self.data_source.total_recs for i, row in enumerate(self.rows): From cae8fd06bb31a362315eb51b9e61d19c7c090ef0 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Mon, 9 Mar 2015 14:29:36 -0300 Subject: [PATCH 17/25] Make the examples database work with the new code --- datagrid_gtk3/db/__init__.py | 47 +++++++++++++++++++++++--- datagrid_gtk3/db/sqlite.py | 65 +++++++++++------------------------- datagrid_gtk3/ui/grid.py | 13 ++++---- 3 files changed, 69 insertions(+), 56 deletions(-) diff --git a/datagrid_gtk3/db/__init__.py b/datagrid_gtk3/db/__init__.py index c5de75e..cfb675a 100644 --- a/datagrid_gtk3/db/__init__.py +++ b/datagrid_gtk3/db/__init__.py @@ -6,21 +6,60 @@ """ +class Node(list): + + """A list that can hold data. + + Just like a simple list, but one can set/get its data + from :obj:`.data`. + + :param object data: the data that will be stored in this node + :param int children_len: the number of the children that will + be loaded lazely at some point + """ + + def __init__(self, data=None, children_len=0): + super(Node, self).__init__() + + self.data = data + self.children_len = children_len + self.path = None + + def is_children_loaded(self, recursive=False): + """Check if this node's children is loaded + + :param bool recursive: wheather to ask each child if their + children is loaded (and their child too and so on) too. + :returns: `True` if children is loaded, otherwise `False` + :rtype: bool + """ + loaded = len(self) == self.children_len + if recursive: + loaded = (loaded and + all(c.is_children_loaded(recursive=True) for c in self)) + return loaded + + class DataSource(object): """Base class for data sources.""" + ID_COLUMN = 'rowid' + PARENT_ID_COLUMN = None + FLAT_COLUMN = None + def __init__(self): - self.rows = [] self.columns = [] - self.column_name_str = '' self.total_recs = 0 self.display_all = True + self.id_column_idx = None + self.parent_column_idx = None + self.flat_column_idx = None def get_selected_columns(self): - pass + return [] def load(self, params=None): - pass + return Node() def update_selected_columns(self, columns): pass diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 471619e..694f240 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -24,7 +24,7 @@ table as table_, ) -from datagrid_gtk3.db import DataSource +from datagrid_gtk3.db import DataSource, Node logger = logging.getLogger(__name__) _compile = lambda q: q.compile(compile_kwargs={"literal_binds": True}).string @@ -42,40 +42,6 @@ } -class Node(list): - - """A list that can hold data. - - Just like a simple list, but one can set/get its data - from :obj:`.data`. - - :param object data: the data that will be stored in this node - :param int children_len: the number of the children that will - be loaded lazely at some point - """ - - def __init__(self, data=None, children_len=0): - super(Node, self).__init__() - - self.data = data - self.children_len = children_len - self.path = None - - def is_children_loaded(self, recursive=False): - """Check if this node's children is loaded - - :param bool recursive: wheather to ask each child if their - children is loaded (and their child too and so on) too. - :returns: `True` if children is loaded, otherwise `False` - :rtype: bool - """ - loaded = len(self) == self.children_len - if recursive: - loaded = (loaded and - all(c.is_children_loaded(recursive=True) for c in self)) - return loaded - - class SQLiteDataSource(DataSource): """SQLite data source especially for use with a `Gtk.TreeModel`. @@ -114,13 +80,12 @@ class SQLiteDataSource(DataSource): 'REAL': float, 'BLOB': str } - ID_COLUMN = 'rowid' - PARENT_ID_COLUMN = None - FLAT_COLUMN = None def __init__(self, db_file, table=None, update_table=None, config=None, ensure_selected_column=True, display_all=False, query=None): """Process database column info.""" + super(SQLiteDataSource, self).__init__() + assert table or query # either table or query must be given self.db_file = db_file self.table = table_(table if table else "__CustomQueryTempView") @@ -135,13 +100,9 @@ def __init__(self, db_file, table=None, update_table=None, config=None, else: self.update_table = table self.config = config - self.id_column_idx = None - self.parent_column_idx = None - self.flat_column_idx = None self.columns = self._get_columns() for col in self.columns: self.table.append_column(column(col['name'])) - self.total_recs = None ### # Public @@ -196,7 +157,7 @@ def load(self, params=None): offset = page * self.MAX_RECS # A little optimization to avoid doing more queries when we # already loaded everything - if self.total_recs is not None and offset >= self.total_recs: + if page > 0 and offset >= self.total_recs: return rows # Flat @@ -501,7 +462,8 @@ def _get_columns(self): 'name': col_name, 'display': display_name, 'type': data_type, - 'transform': transform + 'transform': transform, + 'primary_key': bool(row[5]), } if col_name == self.ID_COLUMN: @@ -526,7 +488,8 @@ def _get_columns(self): 'name': '__selected', 'display': '__selected', 'type': int, - 'transform': 'boolean' + 'transform': 'boolean', + 'primary_key': False, } cols.insert(0, col_dict) has_selected = True @@ -540,6 +503,18 @@ def _get_columns(self): if has_selected and self.flat_column_idx is not None: self.flat_column_idx += 1 + # FIXME: If the idcolumn doesn't match any column, use the + # first primary key we can find. This actually happen on the + # examples database. + if self.id_column_idx is None: + for i, col_dict in enumerate(cols): + if col_dict['primary_key']: + self.ID_COLUMN = col_dict['name'] + self.id_column_idx = i + break + else: + raise ValueError("No id column found.") + return cols def _load_tree_rows(self, conn, where, order_by, parent_id): diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 90e5e37..97a9668 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -1180,12 +1180,12 @@ def _setup_columns(self): # FIXME: We should find a better way for hiding this columns. # A way to specify the visibility on the columns config would be nice. - dont_display = set([ - self.model.data_source.ID_COLUMN, - self.model.data_source.PARENT_ID_COLUMN, - '__selected']) - if not self.model.active_params.get('flat', False): - dont_display.add(self.model.data_source.FLAT_COLUMN) + dont_display = set(['__selected']) + if not self.model.data_source.display_all: + dont_display.add(self.model.data_source.ID_COLUMN) + dont_display.add(self.model.data_source.PARENT_ID_COLUMN) + if not self.model.active_params.get('flat', False): + dont_display.add(self.model.data_source.FLAT_COLUMN) samples = itertools.islice( (r.data for r in self.model.iter_rows()), self.SAMPLE_SIZE) @@ -1608,7 +1608,6 @@ def refresh(self): if 'parent_id' in self.active_params: del self.active_params['parent_id'] - self.data_source.total_recs = None self.row_id_mapper.clear() self.rows = self.data_source.load(self.active_params) self.rows.path = () From cac18d722b831faa4cf8df745c8c393ce0c58588 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 10 Mar 2015 12:26:51 -0300 Subject: [PATCH 18/25] Access table name by self.table.name --- datagrid_gtk3/db/sqlite.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 694f240..6151dc4 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -306,25 +306,25 @@ def update_selected_columns(self, columns): update_sql = ( 'DELETE FROM _selected_columns WHERE tablename=?' ) - params = (self.table,) + params = (self.table.name,) else: select_sql = ( 'SELECT * FROM _selected_columns WHERE tablename=?' ) - cursor.execute(select_sql, (self.table,)) + cursor.execute(select_sql, (self.table.name, )) row = cursor.fetchone() if not row: update_sql = ( 'INSERT INTO _selected_columns ' '(tablename, columns) VALUES (?, ?)' ) - params = (self.table, ','.join(columns)) + params = (self.table.name, ','.join(columns)) else: update_sql = ( 'UPDATE _selected_columns ' 'SET columns=? WHERE tablename=?' ) - params = (','.join(columns), self.table) + params = (','.join(columns), self.table.name) cursor.execute(update_sql, params) conn.commit() From 9351472a0ca5db71e8f7cddb909e00ab9889b391 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 10 Mar 2015 12:30:13 -0300 Subject: [PATCH 19/25] Fix wrong test --- datagrid_gtk3/tests/test_datagrid-gtk3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datagrid_gtk3/tests/test_datagrid-gtk3.py b/datagrid_gtk3/tests/test_datagrid-gtk3.py index b1d11ec..3e4531a 100644 --- a/datagrid_gtk3/tests/test_datagrid-gtk3.py +++ b/datagrid_gtk3/tests/test_datagrid-gtk3.py @@ -231,7 +231,7 @@ def test_bytes_transform(self): """Test bytes humanization.""" self.assertEqual( self.datagrid_model._bytes_transform(1), - '1 B') + '1.0 B') self.assertEqual( self.datagrid_model._bytes_transform(50), '50.0 B') From 0441b0a33c1b3b49f5a6de07df7496797f6eac5a Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 10 Mar 2015 12:30:26 -0300 Subject: [PATCH 20/25] We want flat for flat view only --- datagrid_gtk3/ui/grid.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 97a9668..eae60d3 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -648,9 +648,8 @@ def on_popup_view_changed(self, popup, new_view): else: raise AssertionError("Unrecognized option %r" % (new_view, )) - # We want flat for both flat view and icon - self.model.active_params['flat'] = new_view in [ - OptionsPopup.VIEW_FLAT, OptionsPopup.VIEW_ICON] + # We want flat for flat view only + self.model.active_params['flat'] = new_view == OptionsPopup.VIEW_FLAT child = self.container.grid_scrolledwindow.get_child() self.container.grid_scrolledwindow.remove(child) From 8a9088514e0847959932046370d1aec6f4cc5d0c Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 10 Mar 2015 12:50:02 -0300 Subject: [PATCH 21/25] Samples needs to be a list If it is an iterator, it will be exhausted on the first column and won't be able to generate the 'best column width' for the next ones. --- datagrid_gtk3/ui/grid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index eae60d3..8999718 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -1186,8 +1186,8 @@ def _setup_columns(self): if not self.model.active_params.get('flat', False): dont_display.add(self.model.data_source.FLAT_COLUMN) - samples = itertools.islice( - (r.data for r in self.model.iter_rows()), self.SAMPLE_SIZE) + samples = list(itertools.islice( + (r.data for r in self.model.iter_rows()), self.SAMPLE_SIZE)) for column_index, column in enumerate(self.model.columns): item = column['name'] display = (self.model.display_columns is None From b1c0e548559144f9ca37abeaf0a659c7929fb7fe Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 10 Mar 2015 12:51:17 -0300 Subject: [PATCH 22/25] Allow to set columns to expand --- datagrid_gtk3/db/sqlite.py | 25 +++++++++++++++++-------- datagrid_gtk3/ui/grid.py | 1 + 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 6151dc4..b1ea82e 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -446,24 +446,32 @@ def _get_columns(self): has_selected = False counter = 0 for i, row in enumerate(rows): - col_defined = False col_name = row[1] - if self.config is not None: - if col_name not in [self.ID_COLUMN, '__selected']: - display_name, (data_type, transform) = \ - self.config[counter] - col_defined = True - counter += 1 - if not col_defined: + if (self.config is not None and + col_name not in [self.ID_COLUMN, '__selected']): + display_name, params = self.config[counter] + data_type = params[0] + transform = params[1] + try: + expand = params[2] + except IndexError: + # FIXME: Remove this except when all callsites + # are migrated to pass expand on params + expand = False + counter += 1 + else: display_name = row[1] data_type = self.SQLITE_PY_TYPES.get(row[2].upper(), str) transform = None # TODO: eg. buffer + expand = False + col_dict = { 'name': col_name, 'display': display_name, 'type': data_type, 'transform': transform, 'primary_key': bool(row[5]), + 'expand': expand, } if col_name == self.ID_COLUMN: @@ -479,6 +487,7 @@ def _get_columns(self): has_selected = True else: cols.append(col_dict) + if self._ensure_selected_column and not has_selected: alter_sql = 'ALTER TABLE %s ADD __selected INTEGER' % ( self.update_table) diff --git a/datagrid_gtk3/ui/grid.py b/datagrid_gtk3/ui/grid.py index 8999718..3343f05 100644 --- a/datagrid_gtk3/ui/grid.py +++ b/datagrid_gtk3/ui/grid.py @@ -1213,6 +1213,7 @@ def _setup_columns(self): col.set_fixed_width( self._get_best_column_width(column_index, samples)) col.set_sizing(Gtk.TreeViewColumnSizing.FIXED) + col.set_expand(column['expand']) if item == self.active_sort_column: col.set_sort_indicator(True) col.set_sort_order(self.active_sort_column_order) From d5c37f4280a1064ce862eb16974ecfb17a6836ca Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 10 Mar 2015 14:05:01 -0300 Subject: [PATCH 23/25] Move sqlite tests from vE and fix related code --- datagrid_gtk3/db/sqlite.py | 44 ++++++------ datagrid_gtk3/main.py | 1 - datagrid_gtk3/tests/data.py | 2 +- datagrid_gtk3/tests/test_sqlite.py | 104 +++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 22 deletions(-) create mode 100644 datagrid_gtk3/tests/test_sqlite.py diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index b1ea82e..3539032 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -104,6 +104,10 @@ def __init__(self, db_file, table=None, update_table=None, config=None, for col in self.columns: self.table.append_column(column(col['name'])) + self.selected_table = table_('_selected_columns') + for col in ['tablename', 'columns']: + self.selected_table.append_column(column(col)) + ### # Public ### @@ -270,20 +274,21 @@ def get_selected_columns(self): :returns: list of column names :rtype: list or None """ - table = '_selected_columns' - where = self._get_where_clause( - {'tablename': {'param': self.table, 'operator': '='}}) + where = self.selected_table.columns['tablename'] == self.table.name + columns = [self.selected_table.columns['columns']] with closing(sqlite3.connect(self.db_file)) as conn: conn.row_factory = sqlite3.Row # Access columns by name try: - result = list(self.select(conn, table, where=where)) + result = list( + self.select(conn, self.selected_table, + columns, where=where)) except sqlite3.OperationalError as err: # FIXME: When will this happen? logger.warn(str(err)) return - return result[0][1].split(',') + return result[0][0].split(',') # ^^ 2nd column of returned row; first column is table name def update_selected_columns(self, columns): @@ -328,7 +333,7 @@ def update_selected_columns(self, columns): cursor.execute(update_sql, params) conn.commit() - def select(self, conn, table, columns, where=None, + def select(self, conn, table, columns=None, where=None, order_by=None, limit=None, offset=None): """Select records from given db and table given columns and criteria. @@ -337,6 +342,7 @@ def select(self, conn, table, columns, where=None, :param list columns: list of columns to SELECT from :param dict where: dict of parameters to build ``WHERE`` clause """ + columns = columns or table.columns sql = select( columns=columns, whereclause=where, from_obj=[table], order_by=order_by) @@ -443,6 +449,18 @@ def _get_columns(self): table_info_query = 'PRAGMA table_info(%s)' % self.table.name cursor.execute(table_info_query) rows = cursor.fetchall() + + # FIXME: If the idcolumn doesn't match any column, use the + # first primary key we can find. This actually happen on the + # examples database. + if not any(row[1] == self.ID_COLUMN for row in rows): + for row in rows: + if row[5]: # primary key + self.ID_COLUMN = row[1] + break + else: + raise ValueError("No id column found.") + has_selected = False counter = 0 for i, row in enumerate(rows): @@ -470,7 +488,6 @@ def _get_columns(self): 'display': display_name, 'type': data_type, 'transform': transform, - 'primary_key': bool(row[5]), 'expand': expand, } @@ -498,7 +515,6 @@ def _get_columns(self): 'display': '__selected', 'type': int, 'transform': 'boolean', - 'primary_key': False, } cols.insert(0, col_dict) has_selected = True @@ -512,18 +528,6 @@ def _get_columns(self): if has_selected and self.flat_column_idx is not None: self.flat_column_idx += 1 - # FIXME: If the idcolumn doesn't match any column, use the - # first primary key we can find. This actually happen on the - # examples database. - if self.id_column_idx is None: - for i, col_dict in enumerate(cols): - if col_dict['primary_key']: - self.ID_COLUMN = col_dict['name'] - self.id_column_idx = i - break - else: - raise ValueError("No id column found.") - return cols def _load_tree_rows(self, conn, where, order_by, parent_id): diff --git a/datagrid_gtk3/main.py b/datagrid_gtk3/main.py index cb07f1a..284b922 100644 --- a/datagrid_gtk3/main.py +++ b/datagrid_gtk3/main.py @@ -25,7 +25,6 @@ 'album': None, 'artist': None, 'employee': [ - ('EmployeeId', (long, None)), ('LastName', (str, None)), ('FirstName', (str, None)), ('Title', (str, None)), diff --git a/datagrid_gtk3/tests/data.py b/datagrid_gtk3/tests/data.py index 6fd10b4..f389ab2 100644 --- a/datagrid_gtk3/tests/data.py +++ b/datagrid_gtk3/tests/data.py @@ -27,7 +27,7 @@ def create_db(): with closing(conn.cursor()) as cursor: cursor.execute(""" CREATE TABLE IF NOT EXISTS %s - (__viaextract_id INTEGER, + (__viaextract_id INTEGER PRIMARY KEY, first_name TEXT, last_name TEXT, age INTEGER, diff --git a/datagrid_gtk3/tests/test_sqlite.py b/datagrid_gtk3/tests/test_sqlite.py new file mode 100644 index 0000000..6f42da9 --- /dev/null +++ b/datagrid_gtk3/tests/test_sqlite.py @@ -0,0 +1,104 @@ +import contextlib +import os +import sqlite3 +import unittest + +from datagrid_gtk3.tests.data import create_db +from datagrid_gtk3.db.sqlite import SQLiteDataSource + + +class SQLiteDataSourceTest(unittest.TestCase): + + """Test SQLiteDataSource.""" + + def setUp(self): + """Create test data.""" + self.db_file = create_db() + self.table = 'people' + self.datasource = SQLiteDataSource( + self.db_file, + table=self.table, + config=[ + ('First name', (str, None)), + ('Last name', (str, None)), + ('Age', (int, None)), + ('Start', (int, 'datetime')), + ('Image', (str, 'image')), + ], + ) + self.datasource.MAX_RECS = 2 + + def tearDown(self): + """Remove test data file.""" + os.unlink(self.db_file) + + def test_load(self): + """Load first page of records and get total.""" + rows = self.datasource.load() + self.assertEqual(len(rows), 2) + self.assertEqual(self.datasource.total_recs, 3) + + def test_load_with_params(self): + """Filter and order records.""" + param = { + 'where': { + 'age': { + 'param': 30, 'operator': '>' + } + }, + 'order_by': 'age', + 'desc': True + } + rows = self.datasource.load(param) + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0].data[3], 'Goldman') + + def test_load_paging(self): + """Load first and second pages of records.""" + self.datasource.load() # initial load is always without paging + rows = self.datasource.load({'page': 1}) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0].data[3], 'Goldman') + + def test_update(self): + """Update __selected in first record in data set.""" + self.datasource.update({'__selected': True}, [1]) + # ^^ update row with id 1 + rows = self.datasource.load() + self.assertEqual(rows[0].data[1], 1) + + def test_get_all_record_ids(self): + """Get all record ids for a particular query.""" + param = { + 'where': { + 'age': { + 'param': 30, 'operator': '>' + } + } + } + ids = self.datasource.get_all_record_ids(param) + self.assertEqual(ids, [2, 3]) + + def test_selected_columns(self): + """Set selected columns and ensure they're persisted.""" + self.datasource.update_selected_columns(['last_name']) + cols = self.datasource.get_selected_columns() + self.assertEqual(cols, ['last_name']) + + def test_get_single_record(self): + """Retrieve a single record as a tuple of values.""" + row = self.datasource.get_single_record(1) + self.assertEqual(row[1], 1) + self.assertEqual(row[2], 'Dee') + self.assertEqual(row[3], 'Timberlake') + + def test_select(self): + """Get data without class instance or paging.""" + db_file = self.datasource.db_file + with contextlib.closing(sqlite3.connect(db_file)) as conn: + results = list(self.datasource.select(conn, self.datasource.table)) + self.assertEqual(len(results), 3) + + +if __name__ == '__main__': + unittest.main() From 5ce556b70e7cc29e18813da80ea313f71cabe27e Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 10 Mar 2015 14:09:47 -0300 Subject: [PATCH 24/25] Change rowid to __id instead of __viaextract_id --- datagrid_gtk3/tests/data.py | 4 ++-- datagrid_gtk3/tests/test_datagrid-gtk3.py | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/datagrid_gtk3/tests/data.py b/datagrid_gtk3/tests/data.py index f389ab2..c54ff08 100644 --- a/datagrid_gtk3/tests/data.py +++ b/datagrid_gtk3/tests/data.py @@ -27,7 +27,7 @@ def create_db(): with closing(conn.cursor()) as cursor: cursor.execute(""" CREATE TABLE IF NOT EXISTS %s - (__viaextract_id INTEGER PRIMARY KEY, + (__id INTEGER PRIMARY KEY, first_name TEXT, last_name TEXT, age INTEGER, @@ -38,7 +38,7 @@ def create_db(): for i in TEST_DATA: insert_sql = """ INSERT INTO %s - (__viaextract_id, + (__id, first_name, last_name, age, diff --git a/datagrid_gtk3/tests/test_datagrid-gtk3.py b/datagrid_gtk3/tests/test_datagrid-gtk3.py index 3e4531a..be6baf8 100644 --- a/datagrid_gtk3/tests/test_datagrid-gtk3.py +++ b/datagrid_gtk3/tests/test_datagrid-gtk3.py @@ -15,7 +15,7 @@ from PIL import Image import mock -from datagrid_gtk3.db import sqlite +from datagrid_gtk3.db.sqlite import SQLiteDataSource from datagrid_gtk3.tests.data import create_db from datagrid_gtk3.ui.grid import ( DataGridContainer, @@ -29,10 +29,6 @@ from datagrid_gtk3.utils import imageutils -class SQLiteDataSource(sqlite.SQLiteDataSource): - ID_COLUMN = '__viaextract_id' - - class DataGridControllerTest(unittest.TestCase): """Test DataGridController.""" From 18a613124abb6086f32135a545095eb9a400b4a5 Mon Sep 17 00:00:00 2001 From: Thiago Bellini Date: Tue, 10 Mar 2015 16:01:00 -0300 Subject: [PATCH 25/25] Add a FIXME note on get_columns --- datagrid_gtk3/db/sqlite.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datagrid_gtk3/db/sqlite.py b/datagrid_gtk3/db/sqlite.py index 3539032..3f3408a 100644 --- a/datagrid_gtk3/db/sqlite.py +++ b/datagrid_gtk3/db/sqlite.py @@ -465,6 +465,8 @@ def _get_columns(self): counter = 0 for i, row in enumerate(rows): col_name = row[1] + # FIXME: config should be a dict, mapping the column name + # to the config. This is very error-prone. if (self.config is not None and col_name not in [self.ID_COLUMN, '__selected']): display_name, params = self.config[counter]