Skip to content

Commit

Permalink
#7775: autotranslation respects set_index for pandas
Browse files Browse the repository at this point in the history
  • Loading branch information
@jaroslawmalekcodete
jaroslawmalekcodete committed Sep 11, 2018
1 parent 7d067d1 commit af5e2e9
Show file tree
Hide file tree
Showing 5 changed files with 593 additions and 14 deletions.
15 changes: 11 additions & 4 deletions beakerx/beakerx/runtime.py
View file
Expand Up @@ -308,14 +308,20 @@ def transformBack(obj):
if ('hasIndex' in out) and (out['hasIndex'] == "true"):
# first column becomes the index
vals = out['values']
is_standard_index = ('indexName' in out) and (len(out['indexName']) == 1) and (out['indexName'][0] == 'index')
cnames = out['columnNames'][1:]
index = []
for x in range(0,len(vals)):
for x in range(0, len(vals)):
index.append(transformBack(vals[x][0]))
v = vals[x][1:]
fixNaNsBack(v)
vals[x] = v
return pandas.DataFrame(data=vals, columns=cnames, index=index)
if len(out['indexName']) > 1:
index = pandas.MultiIndex.from_tuples(index, names=out['indexName'])
else:
index = pandas.Index(index, name=out['indexName'])
frame = pandas.DataFrame(data=vals, columns=cnames, index=index)
return frame
else:
vals = out['values']
cnames = out['columnNames']
Expand Down Expand Up @@ -371,7 +377,8 @@ def default(self, obj):
out['type'] = "TableDisplay"
out['subtype'] = "TableDisplay"
out['hasIndex'] = "true"
out['columnNames'] = ['Index'] + obj.columns.tolist()
out['columnNames'] = (['Index'] if obj.index.name is None else obj.index.names) + obj.columns.tolist()
out['indexName'] = ['index'] if (len(obj.index.names) == 1) and (obj.index.names[0] is None) else obj.index.names
vals = obj.values.tolist()
idx = obj.index.tolist()
for x in range(0,len(vals)):
Expand Down Expand Up @@ -438,7 +445,7 @@ def pandas_display_default():
@staticmethod
def pandas_display_table():
pandas.DataFrame._ipython_display_ = TableDisplayWrapper()

def set4(self, var, val, unset, sync):
args = {'name': var, 'sync':sync}
if not unset:
Expand Down
53 changes: 43 additions & 10 deletions ...el/base/src/main/java/com/twosigma/beakerx/table/serializer/TableDisplayDeSerializer.java
View file
Expand Up @@ -15,14 +15,15 @@
*/
package com.twosigma.beakerx.table.serializer;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.twosigma.beakerx.jvm.serialization.BasicObjectSerializer;
import com.twosigma.beakerx.jvm.serialization.BeakerObjectConverter;
import com.twosigma.beakerx.jvm.serialization.ObjectDeserializer;
import com.twosigma.beakerx.table.TableDisplay;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -31,9 +32,14 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static java.lang.String.join;

public class TableDisplayDeSerializer implements ObjectDeserializer {
private final static Logger logger = LoggerFactory.getLogger(TableDisplayDeSerializer.class.getName());
public static final String INDEX_NAME = "indexName";
public static final String INDEX = "index";
private final BeakerObjectConverter parent;

public TableDisplayDeSerializer(BeakerObjectConverter p) {
Expand Down Expand Up @@ -129,24 +135,51 @@ public static Pair<String, Object> getDeserializeObject(BeakerObjectConverter pa
o = getValuesAsMatrix(parent, n, mapper);
}
if (o == null) {
if (n.has("hasIndex") && mapper.readValue(n.get("hasIndex").asText(), String.class).equals("true")
&& columns != null && values != null) {
columns.remove(0);
classes.remove(0);
for (List<?> v : values) {
v.remove(0);
}
o = new TableDisplay(values, columns, classes);
if (n.has("hasIndex")
&& mapper.readValue(n.get("hasIndex").asText(), String.class).equals("true")
&& columns != null
&& values != null
&& n.has(INDEX_NAME)) {
o = handleIndex(n, mapper, values, columns, classes);
} else {
o = new TableDisplay(values, columns, classes);
}

}
} catch (Exception e) {
logger.error("exception deserializing TableDisplay ", e);
}
return new ImmutablePair<String, Object>(subtype, o);
}

@NotNull
private static Object handleIndex(JsonNode n, ObjectMapper mapper, List<List<?>> values, List<String> columns, List<String> classes) throws IOException {
List<String> indexName = (List<String>) mapper.readValue(n.get(INDEX_NAME).toString(), List.class);
boolean standardIndex = indexName.size() == 1 && indexName.get(0).equals(INDEX);
if (standardIndex) {
columns.remove(0);
classes.remove(0);
for (List<?> v : values) {
v.remove(0);
}
} else {
columns.set(0, join(", ", indexName.stream().map(TableDisplayDeSerializer::convertNullToIndexName).collect(Collectors.toList())));
}
TableDisplay td = new TableDisplay(values, columns, classes);
if (!standardIndex) {
td.setHasIndex("true");
}
return td;
}

private static String convertNullToIndexName(String x) {
if (x == null) {
return INDEX;
} else {
return x;
}
}

@SuppressWarnings("unchecked")
@Override
public Object deserialize(JsonNode n, ObjectMapper mapper) {
Expand Down
129 changes: 129 additions & 0 deletions test/ipynb/groovy/autotranslationPandas.ipynb
View file
@@ -0,0 +1,129 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%python\n",
"import pandas as pd\n",
"from beakerx import beakerx\n",
"df = pd.read_csv('../resources/data/interest-rates.csv')\n",
"df = df.set_index('time')\n",
"beakerx.rates = df"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Display pandas in python:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%python\n",
"beakerx.rates"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Display pandas in groovy:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"beakerx.rates"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%python\n",
"import pandas as pd\n",
"import numpy as np\n",
"from beakerx import beakerx\n",
"a=pd.DataFrame(np.random.rand(10,5),columns=['c'+str(i) for i in range(5)])\n",
"a = a.set_index('c0',append=True)\n",
"beakerx.rates = a"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Display pandas in python:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%python\n",
"beakerx.rates"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Display pandas in groovy:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"beakerx.rates"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Groovy",
"language": "groovy",
"name": "groovy"
},
"language_info": {
"codemirror_mode": "groovy",
"file_extension": ".groovy",
"mimetype": "",
"name": "Groovy",
"nbconverter_exporter": "",
"version": "2.4.3"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": false,
"sideBar": false,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": false,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}
96 changes: 96 additions & 0 deletions test/ipynb/python/autotranslationPandasPython.ipynb
View file
@@ -0,0 +1,96 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"import pandas as pd\n",
"from beakerx import beakerx\n",
"df = pd.read_csv('../resources/data/interest-rates.csv')\n",
"df.set_index('time')\n",
"df"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from beakerx import beakerx\n",
"df = pd.read_csv('../resources/data/interest-rates.csv')\n",
"df.set_index('time')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from beakerx import beakerx\n",
"df = pd.read_csv('../resources/data/interest-rates.csv')\n",
"df = df.set_index('time')\n",
"beakerx.rates = df\n",
"beakerx.rates"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"from beakerx import beakerx\n",
"a=pd.DataFrame(np.random.rand(10,5),columns=['c'+str(i) for i in range(5)])\n",
"a = a.set_index('c0',append=True)\n",
"beakerx.rates = a\n",
"beakerx.rates"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.6"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": false,
"sideBar": false,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": false,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}

0 comments on commit af5e2e9

Please sign in to comment.