diff --git a/hickle/loaders/load_pandas.py b/hickle/loaders/load_pandas.py index 0b518553..b1a9edf4 100644 --- a/hickle/loaders/load_pandas.py +++ b/hickle/loaders/load_pandas.py @@ -1,4 +1,5 @@ import pandas as pd # TODO: populate with classes to load -class_register = [] \ No newline at end of file +class_register = [] +exclude_register = [] diff --git a/hickle/lookup.py b/hickle/lookup.py index 6603cd81..4eefe438 100644 --- a/hickle/lookup.py +++ b/hickle/lookup.py @@ -1,31 +1,23 @@ """ #lookup.py -<<<<<<< HEAD This file manages all the mappings between hickle/HDF5 metadata and python types. There are three dictionaries that are populated here: 1) types_dict Mapping between python types and dataset and group creation functions, e.g. -======= -This file contains all the mappings between hickle/HDF5 metadata and python types. -There are four dictionaries and one set that are populated here: - -1) types_dict -types_dict: mapping between python types and dataset creation functions, e.g. ->>>>>>> Adding setup.py optional dependencies types_dict = { - list: create_listlike_dataset, - int: create_python_dtype_dataset, - np.ndarray: create_np_array_dataset + list: (create_listlike_dataset, 'list'), + int: (create_python_dtype_dataset, 'int'), + np.ndarray: (create_np_array_dataset, 'ndarray'), } 2) hkl_types_dict -hkl_types_dict: mapping between hickle metadata and dataset loading functions, e.g. +Mapping between hickle metadata and dataset loading functions, e.g. hkl_types_dict = { - "" : load_list_dataset, - "" : load_tuple_dataset + 'list': load_list_dataset, + 'tuple': load_tuple_dataset } 3) hkl_container_dict @@ -36,33 +28,17 @@ 'dict': DictLikeContainer } -5) types_not_to_sort -type_not_to_sort is a list of hickle type attributes that may be hierarchical, -but don't require sorting by integer index. - ## Extending hickle to add support for other classes and types The process to add new load/dump capabilities is as follows: 1) Create a file called load_[newstuff].py in loaders/ -2) In the load_[newstuff].py file, define your create_dataset and load_dataset functions, - along with all required mapping dictionaries. -3) Add an import call here, and populate the lookup dictionaries with update() calls: - # Add loaders for [newstuff] - try: - from .loaders.load_[newstuff[ import types_dict as ns_types_dict - from .loaders.load_[newstuff[ import hkl_types_dict as ns_hkl_types_dict - types_dict.update(ns_types_dict) - hkl_types_dict.update(ns_hkl_types_dict) - ... (Add container_types_dict etc if required) - except ImportError: - raise +2) In the load_[newstuff].py file, define your create_dataset and load_dataset + functions, along with the 'class_register' and 'exclude_register' lists. + """ -import six -import pkg_resources -<<<<<<< HEAD # %% IMPORTS # Built-in imports import sys @@ -79,13 +55,11 @@ # hickle imports from .helpers import PyContainer,not_dumpable,nobody_is_my_name -======= ->>>>>>> Adding setup.py optional dependencies -def return_first(x): - """ Return first element of a list """ - return x[0] +# %% GLOBALS +# Define dict of all acceptable types +types_dict = {} # Define dict of all acceptable hickle types hkl_types_dict = {} @@ -96,9 +70,6 @@ def return_first(x): # Empty list (hashable) of loaded loader names loaded_loaders = set() -if six.PY2: - container_key_types_dict[b""] = unicode - container_key_types_dict[b""] = long # %% FUNCTION DEFINITIONS def load_nothing(h_node,base_type,py_obj_type): # pragma: nocover @@ -139,6 +110,7 @@ def register_class(myclass_type, hkl_str, dump_function=None, load_function=None Parameters: ----------- myclass_type type(class): type of class + hkl_str (str): String to write to HDF5 file to describe class dump_function (function def): function to write data to HDF5 load_function (function def): function to load data from HDF5 container_class (class def): proxy class to load data from HDF5 @@ -195,10 +167,12 @@ def register_class(myclass_type, hkl_str, dump_function=None, load_function=None def register_class_exclude(hkl_str_to_ignore): - """ Tell loading funciton to ignore any HDF5 dataset with attribute 'type=XYZ' + """ Tell loading funciton to ignore any HDF5 dataset with attribute + 'type=XYZ' Args: - hkl_str_to_ignore (str): attribute type=string to ignore and exclude from loading. + hkl_str_to_ignore (str): attribute type=string to ignore and exclude + from loading. """ if hkl_str_to_ignore in {b'dict_item',b'pickle'}: @@ -235,6 +209,9 @@ def load_loader(py_obj_type, type_mro = type.mro): ------- RuntimeError: in case py object is defined by hickle core machinery. + + """ + # any function or method object, any class object will be passed to pickle # ensure that in any case create_pickled_dataset is called. diff --git a/requirements.txt b/requirements.txt index e6bf7127..5dbe831f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ dill>=0.3.0 -h5py>=2.8.0 +h5py>=2.8.0,<3 numpy>=1.8 six>=1.11.0 diff --git a/requirements_test.txt b/requirements_test.txt index 14882e21..36f1b43b 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -5,4 +5,5 @@ astropy>=1.3,<4.0 scipy>=1.0.0 pandas>=0.24.0 check-manifest -twine>=1.13.0 \ No newline at end of file +twine>=1.13.0 +h5py<3 diff --git a/setup.py b/setup.py index 3dbff091..252ca0d0 100644 --- a/setup.py +++ b/setup.py @@ -1,31 +1,68 @@ # To increment version # Check you have ~/.pypirc filled in # git tag x.y.z -# git push --tags -# python setup.py sdist upload +# git push && git push --tags +# rm -rf dist; python setup.py sdist bdist_wheel +# TEST: twine upload --repository-url https://test.pypi.org/legacy/ dist/* +# twine upload dist/* + +from codecs import open +import re + from setuptools import setup, find_packages +import sys + +author = "Danny Price, Ellert van der Velden and contributors" + +with open("README.md", "r") as fh: + long_description = fh.read() + +with open("requirements.txt", 'r') as fh: + requirements = fh.read().splitlines() + +with open("requirements_test.txt", 'r') as fh: + test_requirements = fh.read().splitlines() + +# Read the __version__.py file +with open('hickle/__version__.py', 'r') as f: + vf = f.read() -version = '3.3.0' -author = 'Danny Price' +# Obtain version from read-in __version__.py file +version = re.search(r"^_*version_* = ['\"]([^'\"]*)['\"]", vf, re.M).group(1) setup(name='hickle', version=version, - description='Hickle - a HDF5 based version of pickle', + description='Hickle - an HDF5 based version of pickle', + long_description=long_description, + long_description_content_type='text/markdown', author=author, author_email='dan@thetelegraphic.com', url='http://github.com/telegraphic/hickle', - download_url='https://github.com/telegraphic/hickle/archive/%s.tar.gz' % version, + download_url=('https://github.com/telegraphic/hickle/archive/v%s.zip' + % (version)), platforms='Cross platform (Linux, Mac OSX, Windows)', + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved', + 'Natural Language :: English', + 'Operating System :: MacOS', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: Unix', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Utilities', + ], keywords=['pickle', 'hdf5', 'data storage', 'data export'], - #py_modules = ['hickle', 'hickle_legacy'], - install_requires=['numpy', 'h5py'], - extras_require={ - 'astropy': ['astropy'], - 'scipy': ['scipy'], - 'pandas': ['pandas'], - 'color': ['django'] - }, - python_requires='>=2.7', + install_requires=requirements, + tests_require=test_requirements, + python_requires='>=3.5', packages=find_packages(), zip_safe=False, )