Skip to content

Commit

Permalink
Improves the deserialization speed by reducing object initialization …
Browse files Browse the repository at this point in the history 
…overhead (#284)
  • Loading branch information
@cheqianh
cheqianh committed Sep 19, 2023
1 parent 3354034 commit 7a201c0
Show file tree
Hide file tree
Showing 8 changed files with 713 additions and 187 deletions.
16 changes: 10 additions & 6 deletions amazon/ion/equivalence.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,15 @@
from math import isnan

from amazon.ion.core import IonType, Timestamp, TimestampPrecision, MICROSECOND_PRECISION, OffsetTZInfo, Multimap
from amazon.ion.simple_types import _IonNature, IonPyList, IonPyDict, IonPyTimestamp, IonPyNull, IonPySymbol, \
from amazon.ion.simple_types import IonPyList, IonPyDict, IonPyTimestamp, IonPyNull, IonPySymbol, \
IonPyText, IonPyDecimal, IonPyFloat
from amazon.ion.symbols import SymbolToken


def obj_has_ion_type_and_annotation(obj):
return hasattr(obj, 'ion_type') and hasattr(obj, 'ion_annotations')


def ion_equals(a, b, timestamps_instants_only=False):
"""Tests two objects for equivalence under the Ion data model.
Expand Down Expand Up @@ -60,8 +64,8 @@ def _ion_equals_timestamps_data_model(a, b):
def _ion_equals(a, b, timestamp_comparison_func, recursive_comparison_func):
"""Compares a and b according to the description of the ion_equals method."""
for a, b in ((a, b), (b, a)): # Ensures that operand order does not matter.
if isinstance(a, _IonNature):
if isinstance(b, _IonNature):
if obj_has_ion_type_and_annotation(a):
if obj_has_ion_type_and_annotation(b):
# Both operands have _IonNature. Their IonTypes and annotations must be equivalent.
eq = a.ion_type is b.ion_type and _annotations_eq(a, b)
else:
Expand Down Expand Up @@ -120,8 +124,8 @@ def _sequences_eq(a, b, comparison_func):


def _structs_eq(a, b, comparison_func):
assert isinstance(a, (dict, Multimap))
if not isinstance(b, (dict, Multimap)):
assert isinstance(a, (dict, Multimap, IonPyDict))
if not isinstance(b, (dict, Multimap, IonPyDict)):
return False
dict_len = len(a)
if dict_len != len(b):
Expand All @@ -135,7 +139,7 @@ def _structs_eq(a, b, comparison_func):
break
if key not in b:
return False
if isinstance(a, Multimap) and isinstance(b, Multimap):
if isinstance(a, (IonPyDict, Multimap)) and isinstance(b, (IonPyDict, Multimap)):
values_a = a.get_all_values(key)
values_b = b.get_all_values(key)
if len(values_a) != len(values_b):
Expand Down
48 changes: 14 additions & 34 deletions amazon/ion/ioncmodule.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,27 +38,16 @@ static PyObject* _decimal_constructor;
static PyObject* _py_timestamp_constructor;
static PyObject* _simpletypes_module;
static PyObject* _ionpynull_cls;
static PyObject* _ionpynull_fromvalue;
static PyObject* _ionpybool_cls;
static PyObject* _ionpybool_fromvalue;
static PyObject* _ionpyint_cls;
static PyObject* _ionpyint_fromvalue;
static PyObject* _ionpyfloat_cls;
static PyObject* _ionpyfloat_fromvalue;
static PyObject* _ionpydecimal_cls;
static PyObject* _ionpydecimal_fromvalue;
static PyObject* _ionpytimestamp_cls;
static PyObject* _ionpytimestamp_fromvalue;
static PyObject* _ionpytext_cls;
static PyObject* _ionpytext_fromvalue;
static PyObject* _ionpysymbol_cls;
static PyObject* _ionpysymbol_fromvalue;
static PyObject* _ionpybytes_cls;
static PyObject* _ionpybytes_fromvalue;
static PyObject* _ionpylist_cls;
static PyObject* _ionpylist_fromvalue;
static PyObject* _ionpydict_cls;
static PyObject* _ionpydict_fromvalue;
static PyObject* _ion_core_module;
static PyObject* _py_ion_type;
static PyObject* py_ion_type_table[14];
Expand Down Expand Up @@ -1102,6 +1091,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
SUCCEED();
case tid_NULL_INT:
{
// TODO double check the real null type, now it's initialized to IonType.NULL by default
ION_TYPE null_type;
// Hack for ion-c issue https://github.com/amazon-ion/ion-c/issues/223
if (original_t != tid_SYMBOL_INT) {
Expand All @@ -1112,17 +1102,17 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
}

ion_type = ION_TYPE_INT(null_type);
py_value = Py_BuildValue(""); // INCREFs and returns Python None.
py_value = Py_None; // INCREFs and returns Python None.
emit_bare_values = emit_bare_values && (ion_type == tid_NULL_INT);
ion_nature_constructor = _ionpynull_fromvalue;
ion_nature_constructor = _ionpynull_cls;
break;
}
case tid_BOOL_INT:
{
BOOL bool_value;
IONCHECK(ion_reader_read_bool(hreader, &bool_value));
py_value = PyBool_FromLong(bool_value);
ion_nature_constructor = _ionpybool_fromvalue;
ion_nature_constructor = _ionpybool_cls;
break;
}
case tid_INT_INT:
Expand All @@ -1145,15 +1135,15 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
py_value = PyLong_FromString(ion_int_str, NULL, 10);
PyMem_Free(ion_int_str);

ion_nature_constructor = _ionpyint_fromvalue;
ion_nature_constructor = _ionpyint_cls;
break;
}
case tid_FLOAT_INT:
{
double double_value;
IONCHECK(ion_reader_read_double(hreader, &double_value));
py_value = Py_BuildValue("d", double_value);
ion_nature_constructor = _ionpyfloat_fromvalue;
ion_nature_constructor = _ionpyfloat_cls;
break;
}
case tid_DECIMAL_INT:
Expand Down Expand Up @@ -1208,21 +1198,21 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
}
}

ion_nature_constructor = _ionpydecimal_fromvalue;
ion_nature_constructor = _ionpydecimal_cls;
break;
}
case tid_TIMESTAMP_INT:
{
IONCHECK(ionc_read_timestamp(hreader, &py_value));
ion_nature_constructor = _ionpytimestamp_fromvalue;
ion_nature_constructor = _ionpytimestamp_cls;
break;
}
case tid_SYMBOL_INT:
{
emit_bare_values = FALSE; // Symbol values must always be emitted as IonNature because of ambiguity with string.
ION_STRING string_value;
IONCHECK(ion_reader_read_string(hreader, &string_value));
ion_nature_constructor = _ionpysymbol_fromvalue;
ion_nature_constructor = _ionpysymbol_cls;
py_value = ion_string_to_py_symboltoken(&string_value);
break;
}
Expand All @@ -1231,7 +1221,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
ION_STRING string_value;
IONCHECK(ion_reader_read_string(hreader, &string_value));
py_value = ion_build_py_string(&string_value);
ion_nature_constructor = _ionpytext_fromvalue;
ion_nature_constructor = _ionpytext_cls;
break;
}
case tid_CLOB_INT:
Expand Down Expand Up @@ -1263,12 +1253,12 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
if (length) {
PyMem_Free(buf);
}
ion_nature_constructor = _ionpybytes_fromvalue;
ion_nature_constructor = _ionpybytes_cls;
break;
}
case tid_STRUCT_INT:
{
ion_nature_constructor = _ionpydict_fromvalue;
ion_nature_constructor = _ionpydict_cls;
//Init a IonPyDict
PyObject* new_dict = PyDict_New();
py_value = PyObject_CallFunctionObjArgs(
Expand All @@ -1293,7 +1283,7 @@ iERR ionc_read_value(hREADER hreader, ION_TYPE t, PyObject* container, BOOL in_s
{
py_value = PyList_New(0);
IONCHECK(ionc_read_into_container(hreader, py_value, /*is_struct=*/FALSE, emit_bare_values));
ion_nature_constructor = _ionpylist_fromvalue;
ion_nature_constructor = _ionpylist_cls;
break;
}
case tid_DATAGRAM_INT:
Expand Down Expand Up @@ -1555,27 +1545,17 @@ PyObject* ionc_init_module(void) {
_simpletypes_module = PyImport_ImportModule("amazon.ion.simple_types");

_ionpynull_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyNull");
_ionpynull_fromvalue = PyObject_GetAttrString(_ionpynull_cls, "from_value");
_ionpybool_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyBool");
_ionpybool_fromvalue = PyObject_GetAttrString(_ionpybool_cls, "from_value");
_ionpyint_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyInt");
_ionpyint_fromvalue = PyObject_GetAttrString(_ionpyint_cls, "from_value");
_ionpyfloat_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyFloat");
_ionpyfloat_fromvalue = PyObject_GetAttrString(_ionpyfloat_cls, "from_value");
_ionpydecimal_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyDecimal");
_ionpydecimal_fromvalue = PyObject_GetAttrString(_ionpydecimal_cls, "from_value");
_ionpytimestamp_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyTimestamp");
_ionpytimestamp_fromvalue = PyObject_GetAttrString(_ionpytimestamp_cls, "from_value");
_ionpybytes_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyBytes");
_ionpybytes_fromvalue = PyObject_GetAttrString(_ionpybytes_cls, "from_value");
_ionpytext_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyText");
_ionpytext_fromvalue = PyObject_GetAttrString(_ionpytext_cls, "from_value");
_ionpysymbol_cls = PyObject_GetAttrString(_simpletypes_module, "IonPySymbol");
_ionpysymbol_fromvalue = PyObject_GetAttrString(_ionpysymbol_cls, "from_value");
_ionpylist_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyList");
_ionpylist_fromvalue = PyObject_GetAttrString(_ionpylist_cls, "from_value");
_ionpydict_cls = PyObject_GetAttrString(_simpletypes_module, "IonPyDict");
_ionpydict_fromvalue = PyObject_GetAttrString(_ionpydict_cls, "from_value");


_ion_core_module = PyImport_ImportModule("amazon.ion.core");
_py_timestamp_precision = PyObject_GetAttrString(_ion_core_module, "TimestampPrecision");
Expand Down

0 comments on commit 7a201c0

Please sign in to comment.