From e4c704012298a561cb15e400371ecd90da34cebf Mon Sep 17 00:00:00 2001 From: Bertrand Bordage Date: Tue, 27 May 2014 01:51:23 +0200 Subject: [PATCH 1/3] Fixes the issue of handling NaN/None by using pandas Series in an optimized way. --- django_pandas/utils.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/django_pandas/utils.py b/django_pandas/utils.py index 8bd4b50..ceeb37e 100644 --- a/django_pandas/utils.py +++ b/django_pandas/utils.py @@ -1,6 +1,7 @@ # coding: utf-8 -from math import isnan +from itertools import izip + from django.core.cache import cache from django.utils.encoding import force_text @@ -31,25 +32,33 @@ def invalidate_signal_handler(sender, **kwargs): def replace_pk(model): base_cache_key = get_base_cache_key(model) - def inner(pk_list): - cache_keys = [None if isnan(pk) else base_cache_key % pk - for pk in pk_list] - out_dict = cache.get_many(frozenset(cache_keys)) - try: - return [None if k is None else out_dict[k] for k in cache_keys] - except KeyError: - out_dict = { - base_cache_key % obj.pk: force_text(obj) - for obj in model.objects.filter(pk__in={pk for pk in pk_list - if not isnan(pk)})} + def get_cache_key_from_pk(pk): + return None if pk is None else base_cache_key % pk + + def inner(pk_series): + pk_series = pk_series.where(pk_series.notnull(), None) + cache_keys = pk_series.apply( + get_cache_key_from_pk, convert_dtype=False) + unique_cache_keys = filter(None, cache_keys.unique()) + + if not unique_cache_keys: + return pk_series + + out_dict = cache.get_many(unique_cache_keys) + + if len(out_dict) < len(unique_cache_keys): + out_dict = {base_cache_key % obj.pk: force_text(obj) + for obj in model.objects.filter( + pk__in=filter(None, pk_series.unique()))} cache.set_many(out_dict) - return list(map(out_dict.get, cache_keys)) + + return list(map(out_dict.get, cache_keys)) return inner def build_update_functions(fieldnames, fields): - for fieldname, field in zip(fieldnames, fields): + for fieldname, field in izip(fieldnames, fields): if field.choices: choices = {k: force_text(v) for k, v in field.flatchoices} From 769f4b9b58d68b0faabcc68b217c662449f891f8 Mon Sep 17 00:00:00 2001 From: Bertrand Bordage Date: Tue, 27 May 2014 02:00:38 +0200 Subject: [PATCH 2/3] Removes izip for Python 3 compatibility. --- django_pandas/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/django_pandas/utils.py b/django_pandas/utils.py index ceeb37e..2ce561b 100644 --- a/django_pandas/utils.py +++ b/django_pandas/utils.py @@ -1,7 +1,5 @@ # coding: utf-8 -from itertools import izip - from django.core.cache import cache from django.utils.encoding import force_text @@ -58,7 +56,7 @@ def inner(pk_series): def build_update_functions(fieldnames, fields): - for fieldname, field in izip(fieldnames, fields): + for fieldname, field in zip(fieldnames, fields): if field.choices: choices = {k: force_text(v) for k, v in field.flatchoices} From e5039b9e7369e303c97f1bef44b50f7cd23839d4 Mon Sep 17 00:00:00 2001 From: Bertrand Bordage Date: Tue, 27 May 2014 10:06:45 +0200 Subject: [PATCH 3/3] Fixes python3 compatibility. --- django_pandas/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_pandas/utils.py b/django_pandas/utils.py index 2ce561b..0540cd8 100644 --- a/django_pandas/utils.py +++ b/django_pandas/utils.py @@ -37,7 +37,7 @@ def inner(pk_series): pk_series = pk_series.where(pk_series.notnull(), None) cache_keys = pk_series.apply( get_cache_key_from_pk, convert_dtype=False) - unique_cache_keys = filter(None, cache_keys.unique()) + unique_cache_keys = list(filter(None, cache_keys.unique())) if not unique_cache_keys: return pk_series @@ -47,7 +47,7 @@ def inner(pk_series): if len(out_dict) < len(unique_cache_keys): out_dict = {base_cache_key % obj.pk: force_text(obj) for obj in model.objects.filter( - pk__in=filter(None, pk_series.unique()))} + pk__in=list(filter(None, pk_series.unique())))} cache.set_many(out_dict) return list(map(out_dict.get, cache_keys))