From 41d4d245449b4fd379306a1088b09adb15103897 Mon Sep 17 00:00:00 2001 From: tumb1er Date: Thu, 22 May 2014 15:03:30 +0400 Subject: [PATCH 1/5] use pandas --- django_pandas/io.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/django_pandas/io.py b/django_pandas/io.py index a9c289b..3c261f4 100644 --- a/django_pandas/io.py +++ b/django_pandas/io.py @@ -1,4 +1,6 @@ import pandas as pd +from django.db import connections +from pandas.io.sql import read_frame from .utils import update_with_verbose @@ -51,10 +53,11 @@ def read_frame(qs, fieldnames=(), index_col=None, coerce_float=False, fields = qs.model._meta.fields fieldnames = [f.name for f in fields] - recs = list(qs.values_list(*fieldnames)) - - df = pd.DataFrame.from_records(recs, columns=fieldnames, - coerce_float=coerce_float) + compiler = qs.query.get_compiler(using=qs.db) + connection = connections[qs.db] + sql, args = compiler.as_sql() + df = read_frame(sql, connection, coerce_float=coerce_float, params=args) + df.columns = qs.field_names if verbose: update_with_verbose(df, fieldnames, fields) From 75a411273f23ec15fedff40e713cccaeef9d5d7a Mon Sep 17 00:00:00 2001 From: tumb1er Date: Thu, 22 May 2014 15:14:34 +0400 Subject: [PATCH 2/5] fix read_frame redefinition #22 --- django_pandas/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_pandas/io.py b/django_pandas/io.py index 3c261f4..a4a22f1 100644 --- a/django_pandas/io.py +++ b/django_pandas/io.py @@ -1,6 +1,6 @@ import pandas as pd from django.db import connections -from pandas.io.sql import read_frame +from pandas.io import sql from .utils import update_with_verbose @@ -56,7 +56,7 @@ def read_frame(qs, fieldnames=(), index_col=None, coerce_float=False, compiler = qs.query.get_compiler(using=qs.db) connection = connections[qs.db] sql, args = compiler.as_sql() - df = read_frame(sql, connection, coerce_float=coerce_float, params=args) + df = sql.read_frame(sql, connection, coerce_float=coerce_float, params=args) df.columns = qs.field_names if verbose: From 1a1e6d504d67aa0e48142acb2bed3d3737919b8a Mon Sep 17 00:00:00 2001 From: tumb1er Date: Thu, 22 May 2014 15:21:19 +0400 Subject: [PATCH 3/5] #22 fix another redefinition Online github editor is unusable for editing python code... --- django_pandas/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_pandas/io.py b/django_pandas/io.py index a4a22f1..39dd22d 100644 --- a/django_pandas/io.py +++ b/django_pandas/io.py @@ -55,8 +55,8 @@ def read_frame(qs, fieldnames=(), index_col=None, coerce_float=False, compiler = qs.query.get_compiler(using=qs.db) connection = connections[qs.db] - sql, args = compiler.as_sql() - df = sql.read_frame(sql, connection, coerce_float=coerce_float, params=args) + query, args = compiler.as_sql() + df = sql.read_frame(query, connection, coerce_float=coerce_float, params=args) df.columns = qs.field_names if verbose: From 0687e999de2663a5bd8f5f8661383024e89d6392 Mon Sep 17 00:00:00 2001 From: tumb1er Date: Thu, 22 May 2014 15:35:42 +0400 Subject: [PATCH 4/5] Update io.py --- django_pandas/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_pandas/io.py b/django_pandas/io.py index 39dd22d..5b1ecfa 100644 --- a/django_pandas/io.py +++ b/django_pandas/io.py @@ -57,7 +57,7 @@ def read_frame(qs, fieldnames=(), index_col=None, coerce_float=False, connection = connections[qs.db] query, args = compiler.as_sql() df = sql.read_frame(query, connection, coerce_float=coerce_float, params=args) - df.columns = qs.field_names + df.columns = fieldnames if verbose: update_with_verbose(df, fieldnames, fields) From 246b328d55ba6058849c5165eb6b6ab63e0620f3 Mon Sep 17 00:00:00 2001 From: Sergey Tikhonov Date: Thu, 22 May 2014 17:14:45 +0400 Subject: [PATCH 5/5] #22 another implementation for Dj==1.6 --- django_pandas/io.py | 19 +++++++---- django_pandas/managers.py | 53 +++++++++++++++++++++++++++-- django_pandas/tests/test_manager.py | 10 ++++++ 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/django_pandas/io.py b/django_pandas/io.py index 5b1ecfa..78d60a1 100644 --- a/django_pandas/io.py +++ b/django_pandas/io.py @@ -1,6 +1,6 @@ import pandas as pd from django.db import connections -from pandas.io import sql +from pandas.io.sql import execute, _safe_fetch from .utils import update_with_verbose @@ -51,18 +51,25 @@ def read_frame(qs, fieldnames=(), index_col=None, coerce_float=False, fields = to_fields(qs, fieldnames) else: fields = qs.model._meta.fields - fieldnames = [f.name for f in fields] + fieldnames = getattr(qs, '_fields', None) or [f.name for f in fields] + qs = qs.values_list(*fieldnames) compiler = qs.query.get_compiler(using=qs.db) connection = connections[qs.db] query, args = compiler.as_sql() - df = sql.read_frame(query, connection, coerce_float=coerce_float, params=args) - df.columns = fieldnames - if verbose: - update_with_verbose(df, fieldnames, fields) + # because pandas.io.sql.read_frame always runs con.commit(), + # some code extracted from there. + cur = execute(query, connection, params=args) + rows = _safe_fetch(cur) + cur.close() + + df = pd.DataFrame.from_records(rows, columns=fieldnames, coerce_float=coerce_float) if index_col is not None: df.set_index(index_col, inplace=True) + if verbose: + update_with_verbose(df, fieldnames, fields) + return df diff --git a/django_pandas/managers.py b/django_pandas/managers.py index a74174c..088b180 100644 --- a/django_pandas/managers.py +++ b/django_pandas/managers.py @@ -1,9 +1,9 @@ -from django.db.models.query import QuerySet +from django.db.models.query import QuerySet, ValuesListQuerySet, ValuesQuerySet from model_utils.managers import PassThroughManager from .io import read_frame -class DataFrameQuerySet(QuerySet): +class DataFrameMixin(object): def to_pivot_table(self, fieldnames=(), verbose=True, values=None, rows=None, cols=None, @@ -197,6 +197,55 @@ def to_dataframe(self, fieldnames=(), verbose=True, index=None, return df +class ValuesMixin(object): + """ + Mixin for overriding return type for values() and values_list(). + """ + + def values(self, *fields): + return self._clone(klass=DataFrameValuesQuerySet, setup=True, _fields=fields) + + def values_list(self, *fields, **kwargs): + flat = kwargs.pop('flat', False) + if kwargs: + raise TypeError('Unexpected keyword arguments to values_list: %s' + % (list(kwargs),)) + if flat and len(fields) > 1: + raise TypeError("'flat' is not valid when values_list is called with more than one field.") + return self._clone(klass=DataFrameValuesListQuerySet, setup=True, flat=flat, + _fields=fields) + + +class ConstraintValuesMixin(object): + """ + Limits field list for values() and values_list() with original + field list. + """ + + def values(self, *fields): + fields = filter(lambda f: f in self._fields, fields) + return super(ConstraintValuesMixin, self).values(*fields) + + def values_list(self, *fields, **kwargs): + fields = filter(lambda f: f in self._fields, fields) + return super(ConstraintValuesMixin, self).values_list(*fields, **kwargs) + + +class DataFrameValuesQuerySet(DataFrameMixin, ConstraintValuesMixin, + ValuesMixin, ValuesQuerySet): + pass + + +class DataFrameValuesListQuerySet(DataFrameMixin, ConstraintValuesMixin, + ValuesMixin, ValuesListQuerySet): + pass + + +class DataFrameQuerySet(DataFrameMixin, ValuesMixin, QuerySet): + pass + + class DataFrameManager(PassThroughManager): + def get_query_set(self): return DataFrameQuerySet(self.model) diff --git a/django_pandas/tests/test_manager.py b/django_pandas/tests/test_manager.py index af76069..c9f8abd 100644 --- a/django_pandas/tests/test_manager.py +++ b/django_pandas/tests/test_manager.py @@ -43,6 +43,16 @@ def test_dataframe(self): n, c = df2.shape self.assertEqual((n, c), (3, 3)) + def test_values_list_qs(self): + qs = DataFrame.objects.values_list('col1', 'col2') + df = qs.to_dataframe() + self.assertNotIn('id', df.columns.values.tolist()) + + def test_values_qs(self): + qs = DataFrame.objects.values('col1', 'col2') + df = qs.to_dataframe() + self.assertNotIn('id', df.columns.values.tolist()) + class TimeSeriesTest(TestCase): def unpivot(self, frame):