🤝 Merge branch 'master' into v0.4.x

.moban.d/docs/source/index.rst

@@ -57,6 +57,7 @@ get_data(.., library='pyexcel-ods')
     ============= ======= ======== ======= ======== ======== ========
     `pyexcel-io`_ `xls`_  `xlsx`_  `ods`_  `ods3`_  `odsr`_  `xlsxw`_
     ============= ======= ======== ======= ======== ======== ========
+    0.4.0         0.4.0   0.4.0    0.4.0   0.4.0    0.4.0    0.4.0
     0.3.0+        0.3.0+  0.3.0    0.3.0+  0.3.0+   0.3.0    0.3.0
     0.2.2+        0.2.2+  0.2.2+   0.2.1+  0.2.1+            0.0.1
     0.2.0+        0.2.0+  0.2.0+   0.2.0   0.2.0             0.0.1

.moban.yml

@@ -2,6 +2,7 @@ configuration:
   configuration_dir: "commons/config"
   template_dir:
   - "commons/templates"
+  - "setupmobans/templates"
   - ".moban.d"
   configuration: pyexcel_io.yaml
 targets:
@@ -10,9 +11,9 @@ targets:
   - .travis.yml: travis.yml.jj2
   - requirements.txt: requirements.txt.jj2
   - "tests/requirements.txt": "tests/requirements.txt"
-  - LICENSE: LICENSE.jj2
+  - LICENSE: NEW_BSD_LICENSE.jj2
   - MANIFEST.in: MANIFEST.in.jj2
-  - test.sh: test.sh.jj2
-  - test.bat: test.sh.jj2
+  - test.sh: test.script.jj2
+  - test.bat: test.script.jj2
   - README.rst: README.rst
-  - "docs/source/index.rst": "docs/source/index.rst"
+  - "docs/source/index.rst": "docs/source/index.rst"

.travis.yml

@@ -3,17 +3,14 @@ language: python
 notifications:
   email: false
 python:
-  - pypy
+  - pypy-5.3.1
   - 3.6
   - 3.5
   - 3.4
   - 3.3
   - 2.7
   - 2.6
 before_install:
-  - cd $HOME
-  - "if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then deactivate && wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy-5.7.1-linux_x86_64-portable.tar.bz2 -O - | tar -jxf - && echo 'Setting up aliases...' && ln -s pypy-5.7.1-linux_x86_64-portable pypy2-latest && export PATH=$HOME/pypy2-latest/bin/:$PATH && virtualenv --no-site-packages --python ~/pypy2-latest/bin/pypy pypy2-env && echo 'Creating custom env...' && source pypy2-env/bin/activate && python -V; fi"
-  - cd -
   - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi
   - if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then
       mv min_requirements.txt requirements.txt ;

CHANGELOG.rst

@@ -11,7 +11,40 @@ Updated
 #. `#19 <https://github.com/pyexcel/pyexcel-io/issues/19>`_,
    use cString by default
 
-0.4.0 - unreleased
+
+0.4.3 - 29.07.2017
+--------------------------------------------------------------------------------
+
+Updated
+********************************************************************************
+
+#. `#41 <https://github.com/pyexcel/pyexcel-io/issues/41>`_, walk away gracefully
+   when mmap is not available.
+
+0.4.2 - 05.07.2017
+--------------------------------------------------------------------------------
+
+Updated
+********************************************************************************
+
+#. `#37 <https://github.com/pyexcel/pyexcel-io/issues/37>`_, permanently fix
+   the residue folder pyexcel by release all future releases in a clean clone.
+
+
+0.4.1 - 29.06.2017
+--------------------------------------------------------------------------------
+
+Updated
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+#. `#39 <https://github.com/pyexcel/pyexcel-io/issues/39>`_, raise exception
+   when bulk save in django fails. Please `bulk_save=False` if you as the
+   developer choose to save the records one by one if bulk_save cannot be
+   used. However, exception in one-by-one save case will be raised as well.
+   This change is made to raise exception in the first place so that you as
+   the developer will be suprised when it was deployed in production.
+
+0.4.0 - 19.06.2017
 --------------------------------------------------------------------------------
 
 Updated
@@ -30,6 +63,7 @@ Removed
 
 #. Removed plugin loading code and lml is used instead
 
+
 0.3.4 - 18.05.2017
 --------------------------------------------------------------------------------
 

README.rst

@@ -8,17 +8,37 @@ pyexcel-io - Let you focus on data, instead of file formats
 .. image:: https://api.travis-ci.org/pyexcel/pyexcel-io.svg?branch=master
    :target: http://travis-ci.org/pyexcel/pyexcel-io
 
-.. image:: https://codecov.io/github/pyexcel/pyexcel-io/coverage.png
-   :target: https://codecov.io/github/pyexcel/pyexcel-io
+.. image:: https://codecov.io/gh/pyexcel/pyexcel-io/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/pyexcel/pyexcel-io
+
+.. image:: https://img.shields.io/gitter/room/gitterHQ/gitter.svg
+   :target: https://gitter.im/pyexcel/Lobby
 
 .. image:: https://readthedocs.org/projects/pyexcel-io/badge/?version=latest
    :target: http://pyexcel-io.readthedocs.org/en/latest/
 
+Support the project
+================================================================================
+
+If your company has embedded pyexcel and its components into a revenue generating
+product, please `support me on patreon <https://www.patreon.com/bePatron?u=5537627>`_ to
+maintain the project and develop it further.
+
+If you are an individual, you are welcome to support me too on patreon and for however long
+you feel like to. As a patreon, you will receive
+`early access to pyexcel related contents <https://www.patreon.com/pyexcel/posts>`_.
+
+With your financial support, I will be able to invest
+a little bit more time in coding, documentation and writing interesting posts.
+
+
 Known constraints
 ==================
 
 Fonts, colors and charts are not supported.
 
+Introduction
+================================================================================
 
 **pyexcel-io** provides **one** application programming interface(API) to read
 and write the data in excel format, import the data into and export the data
@@ -44,7 +64,8 @@ sqlalchemy supported databases. Its supported file formats are extended to cover
    `pyexcel-ods3`_          ods                     `ezodf`_,       2.6, 2.7, 3.3, 3.4
                                                     lxml            3.5, 3.6
    `pyexcel-ods`_           ods                     `odfpy`_        same as above
-   `pyexcel-odsr`_          ods(read only)          lxml            same as above
+   `pyexcel-odsr`_          read only for ods, fods lxml            same as above
+   `pyexcel-htmlr`_         html(read only)         lxml,html5lib   same as above
    ======================== ======================= =============== ==================
 
 .. _pyexcel-io: https://github.com/pyexcel/pyexcel-io
@@ -54,6 +75,7 @@ sqlalchemy supported databases. Its supported file formats are extended to cover
 .. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3
 .. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr
 .. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw
+.. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr
 
 .. _xlrd: https://github.com/python-excel/xlrd
 .. _xlwt: https://github.com/python-excel/xlwt
@@ -103,21 +125,6 @@ or clone it and install it:
 
 
 
-Support the project
-================================================================================
-
-If your company has embedded pyexcel and its components into a revenue generating
-product, please `support me on patreon <https://www.patreon.com/bePatron?u=5537627>`_ to
-maintain the project and develop it further.
-
-If you are an individual, you are welcome to support me too on patreon and for however long
-you feel like to. As a patreon, you will receive
-`early access to pyexcel related contents <https://www.patreon.com/pyexcel/posts>`_.
-
-With your financial support, I will be able to invest
-a little bit more time in coding, documentation and writing interesting posts.
-
-
 Development guide
 ================================================================================
 
@@ -136,19 +143,40 @@ Then install relevant development requirements:
 #. pip install -r requirements.txt
 #. pip install -r tests/requirements.txt
 
+Once you have finished your changes, please provide test case(s), relevant documentation
+and update CHANGELOG.rst.
+
+.. note::
+
+    As to rnd_requirements.txt, usually, it is created when a dependent
+    library is not released. Once the dependecy is installed
+    (will be released), the future
+    version of the dependency in the requirements.txt will be valid.
+
+
+How to test your contribution
+------------------------------
+
+Although `nose` and `doctest` are both used in code testing, it is adviable that unit tests are put in tests. `doctest` is incorporated only to make sure the code examples in documentation remain valid across different development releases.
+
+On Linux/Unix systems, please launch your tests like this::
+
+    $ make
+
+On Windows systems, please issue this command::
+
+    > test.bat
+
+How to update test environment and update documentation
+---------------------------------------------------------
 
-In order to update test environment, and documentation, additional steps are
-required:
+Additional steps are required:
 
 #. pip install moban
+#. git clone https://github.com/moremoban/setupmobans.git # generic setup
 #. git clone https://github.com/pyexcel/pyexcel-commons.git commons
 #. make your changes in `.moban.d` directory, then issue command `moban`
 
-What is rnd_requirements.txt
--------------------------------
-
-Usually, it is created when a dependent library is not released. Once the dependecy is installed(will be released), the future version of the dependency in the requirements.txt will be valid.
-
 What is pyexcel-commons
 ---------------------------------
 
@@ -159,18 +187,17 @@ What is .moban.d
 
 `.moban.d` stores the specific meta data for the library.
 
-How to test your contribution
-------------------------------
+Acceptance criteria
+-------------------
 
-Although `nose` and `doctest` are both used in code testing, it is adviable that unit tests are put in tests. `doctest` is incorporated only to make sure the code examples in documentation remain valid across different development releases.
+#. Has Test cases written
+#. Has all code lines tested
+#. Passes all Travis CI builds
+#. Has fair amount of documentation if your change is complex
+#. Please update CHANGELOG.rst
+#. Please add yourself to CONTRIBUTORS.rst
+#. Agree on NEW BSD License for your contribution
 
-On Linux/Unix systems, please launch your tests like this::
-
-    $ make
-
-On Windows systems, please issue this command::
-
-    > test.bat
 
 
 License

docs/source/conf.py

@@ -13,7 +13,7 @@
 ]
 
 intersphinx_mapping = {
-    'pyexcel': ('http://pyexcel.readthedocs.org/en/latest/', None),
+    'pyexcel': ('http://pyexcel.readthedocs.io/en/latest/', None),
 }
 spelling_word_list_filename = 'spelling_wordlist.txt'
 templates_path = ['_templates']
@@ -22,8 +22,8 @@
 
 project = u'pyexcel-io'
 copyright = u'2015-2017 Onni Software Ltd.'
-version = '0.3.4'
-release = '0.4.0'
+version = '0.4.3'
+release = '0.4.3'
 exclude_patterns = []
 pygments_style = 'sphinx'
 html_theme = 'default'

docs/source/index.rst

@@ -10,7 +10,6 @@
 :Source code: http://github.com/pyexcel/pyexcel-io.git
 :Issues: http://github.com/pyexcel/pyexcel-io/issues
 :License: New BSD License
-:Development: |release|
 :Released: |version|
 :Generated: |today|
 
@@ -77,7 +76,8 @@ For individual excel file formats, please install them as you wish:
    `pyexcel-ods3`_          ods                     `ezodf`_,       2.6, 2.7, 3.3, 3.4
                                                     lxml            3.5, 3.6
    `pyexcel-ods`_           ods                     `odfpy`_        same as above
-   `pyexcel-odsr`_          ods(read only)          lxml            same as above
+   `pyexcel-odsr`_          read only for ods, fods lxml            same as above
+   `pyexcel-htmlr`_         html(read only)         lxml,html5lib   same as above
    ======================== ======================= =============== ==================
 
 .. _pyexcel-io: https://github.com/pyexcel/pyexcel-io
@@ -87,6 +87,7 @@ For individual excel file formats, please install them as you wish:
 .. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3
 .. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr
 .. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw
+.. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr
 
 .. _xlrd: https://github.com/python-excel/xlrd
 .. _xlwt: https://github.com/python-excel/xlwt
@@ -126,6 +127,7 @@ get_data(.., library='pyexcel-ods')
     ============= ======= ======== ======= ======== ======== ========
     `pyexcel-io`_ `xls`_  `xlsx`_  `ods`_  `ods3`_  `odsr`_  `xlsxw`_
     ============= ======= ======== ======= ======== ======== ========
+    0.4.0         0.4.0   0.4.0    0.4.0   0.4.0    0.4.0    0.4.0
     0.3.0+        0.3.0+  0.3.0    0.3.0+  0.3.0+   0.3.0    0.3.0
     0.2.2+        0.2.2+  0.2.2+   0.2.1+  0.2.1+            0.0.1
     0.2.0+        0.2.0+  0.2.0+   0.2.0   0.2.0             0.0.1

docs/source/pyinstaller.rst

@@ -73,6 +73,6 @@ pyexcel-odsr
 In order to package pyexcel-odsr, you need to specify::
 
     --hidden-import pyexcel_odsr
-    --hidden-import pyexcel_ods4.odsr
+    --hidden-import pyexcel_odsr.odsr
 
 

pyexcel_io.yaml

@@ -1,8 +1,9 @@
 overrides: "pyexcel.yaml"
 name: "pyexcel-io"
 nick_name: io
-version: 0.4.0
-release: 0.3.4
+version: 0.4.3
+current_version: 0.4.3
+release: 0.4.3
 dependencies:
   - ordereddict;python_version<"2.7"
   - lml==0.0.1

pyexcel_io/_compact.py

@@ -54,10 +54,13 @@ def next(self):
 
 def isstream(instance):
     """ check if a instance is a stream """
-    i_am_not_mmap_obj = True
-    if not PY26:
+    try:
         import mmap
         i_am_not_mmap_obj = not isinstance(instance, mmap.mmap)
+    except ImportError:
+        # Python 2.6 or Google App Engine
+        i_am_not_mmap_obj = True
+
     return hasattr(instance, 'read') and i_am_not_mmap_obj
 
 

pyexcel_io/database/importers/django.py

@@ -19,14 +19,16 @@
 
 class DjangoModelWriter(SheetWriter):
     """ import data into a django model """
-    def __init__(self, importer, adapter, batch_size=None):
+    def __init__(self, importer, adapter, batch_size=None,
+                 bulk_save=True):
         SheetWriter.__init__(self, importer, adapter, adapter.name)
         self.__batch_size = batch_size
         self.__model = adapter.model
         self.__column_names = adapter.column_names
         self.__mapdict = adapter.column_name_mapping_dict
         self.__initializer = adapter.row_initializer
         self.__objs = []
+        self.__bulk_save = bulk_save
 
     def write_row(self, array):
         if is_empty_array(array):
@@ -44,20 +46,23 @@ def write_row(self, array):
                 # skip the row
 
     def close(self):
-        try:
-            self.__model.objects.bulk_create(
-                self.__objs, batch_size=self.__batch_size)
-        except Exception as bulk_create_exception:
-            log.info(constants.MESSAGE_DB_EXCEPTION)
-            log.info(bulk_create_exception)
+        if self.__bulk_save:
+            try:
+                self.__model.objects.bulk_create(
+                    self.__objs, batch_size=self.__batch_size)
+            except Exception as bulk_create_exception:
+                log.info(constants.MESSAGE_DB_EXCEPTION)
+                log.exception(bulk_create_exception)
+                raise
+
+        else:
             for an_object in self.__objs:
                 try:
                     an_object.save()
                 except Exception as single_save_exception:
                     log.info(constants.MESSAGE_IGNORE_ROW)
-                    log.info(single_save_exception)
-                    log.info(an_object)
-                    continue
+                    log.exception(single_save_exception)
+                    raise
 
 
 class DjangoBookWriter(BookWriter):