diff --git a/README.rst b/README.rst index 1cee654..2eba442 100644 --- a/README.rst +++ b/README.rst @@ -22,10 +22,6 @@ README :alt: MIT License :target: http://opensource.org/licenses/MIT -.. image:: https://requires.io/github/sdpython/sparkouille/requirements.svg?branch=master - :target: https://requires.io/github/sdpython/sparkouille/requirements/?branch=master - :alt: Requirements Status - .. image:: https://codecov.io/github/sdpython/sparkouille/coverage.svg?branch=master :target: https://codecov.io/github/sdpython/sparkouille?branch=master diff --git a/_doc/sphinxdoc/source/api/index.rst b/_doc/sphinxdoc/source/api/index.rst index 9204a00..cb0806c 100644 --- a/_doc/sphinxdoc/source/api/index.rst +++ b/_doc/sphinxdoc/source/api/index.rst @@ -5,3 +5,4 @@ API .. toctree:: datasets + fctmr diff --git a/_doc/sphinxdoc/source/conf.py b/_doc/sphinxdoc/source/conf.py index bcfd052..fc96cd7 100644 --- a/_doc/sphinxdoc/source/conf.py +++ b/_doc/sphinxdoc/source/conf.py @@ -3,7 +3,9 @@ import os import datetime import re -import sphinx_redactor_theme +# import sphinx_redactor_theme +# import karma_sphinx_theme +import sphinx_bootstrap_theme sys.path.insert(0, os.path.abspath(os.path.join(os.path.split(__file__)[0]))) @@ -24,22 +26,31 @@ from pyquickhelper.helpgen.default_conf import set_sphinx_variables, get_default_stylesheet set_sphinx_variables(__file__, "sparkouille", "Xavier Dupré", 2018, - "sphinx_redactor_theme", sphinx_redactor_theme.get_html_theme_path(), + "bootstrap", sphinx_bootstrap_theme.get_html_theme_path(), locals(), extlinks=dict( issue=('https://github.com/sdpython/sparkouille/issues/%s', 'issue')), title="sparkouille", book=True) +if False: + html_sidebars['**'] = ['globaltoc.html', 'localtoc.html', 'relations.html', + 'sourcelink.html', 'searchbox.html'] + html_sidebars['*'] = html_sidebars['**'] + html_sidebars[''] = html_sidebars['*'] + +html_theme_options = { + 'touch_icon': '_static/project_ico.ico', +} + blog_root = "http://www.xavierdupre.fr/app/sparkouille/helpsphinx/" html_context = { - 'css_files': get_default_stylesheet() + ['_static/my-styles.css'], + 'css_files': get_default_stylesheet() + ['_static/my-styles.css', '_static/gallery.css'], } -html_logo = "project_ico.png" -html_sidebars = {} +html_logo = "project_ico.png" -language = "en" +language = "fr" mathdef_link_only = True diff --git a/_doc/sphinxdoc/source/i_index.rst b/_doc/sphinxdoc/source/i_index.rst new file mode 100644 index 0000000..0d7cd51 --- /dev/null +++ b/_doc/sphinxdoc/source/i_index.rst @@ -0,0 +1,27 @@ + +=========== +En diagonal +=========== + +.. only:: html + + .. toctree:: + :maxdepth: 2 + + issues_todoextlist + completed_todoextlist + filechanges + all_report + README + HISTORY + license + blog/blogindex + index_module + +.. only:: not html + + .. toctree:: + :maxdepth: 2 + + license + blog/blogindex diff --git a/_doc/sphinxdoc/source/index.rst b/_doc/sphinxdoc/source/index.rst index 1f6edaf..b361e58 100644 --- a/_doc/sphinxdoc/source/index.rst +++ b/_doc/sphinxdoc/source/index.rst @@ -28,10 +28,6 @@ Spark approximatif :alt: MIT License :target: http://opensource.org/licenses/MIT -.. image:: https://requires.io/github/sdpython/sparkouille/requirements.svg?branch=master - :target: https://requires.io/github/sdpython/sparkouille/requirements/?branch=master - :alt: Requirements Status - .. image:: https://codecov.io/github/sdpython/sparkouille/coverage.svg?branch=master :target: https://codecov.io/github/sdpython/sparkouille?branch=master @@ -58,16 +54,14 @@ Lectures, notebooks, helpers pour :epkg:`Spark`. lectures/index_pig tutorial/index api/index - i_index i_ex all_notebooks - blog/blogindex - index_modules + i_index +----------------------+---------------------+---------------------+--------------------+------------------------+------------------------------------------------+ | :ref:`l-modules` | :ref:`l-functions` | :ref:`l-classes` | :ref:`l-methods` | :ref:`l-staticmethods` | :ref:`l-properties` | +----------------------+---------------------+---------------------+--------------------+------------------------+------------------------------------------------+ | :ref:`modindex` | :ref:`l-EX2` | :ref:`search` | :ref:`l-license` | :ref:`l-changes` | :ref:`l-README` | +----------------------+---------------------+---------------------+--------------------+------------------------+------------------------------------------------+ -| :ref:`genindex` | :ref:`l-FAQ2` | :ref:`l-notebooks` | | :ref:`l-statcode` | `Unit Test Coverage `_ | +| :ref:`genindex` | :ref:`l-FAQ2` | :ref:`l-notebooks` | :ref:`l-HISTORY` | :ref:`l-statcode` | `Unit Test Coverage `_ | +----------------------+---------------------+---------------------+--------------------+------------------------+------------------------------------------------+ diff --git a/_doc/sphinxdoc/source/nbcov-2018-04-09.png b/_doc/sphinxdoc/source/nbcov-2018-04-09.png deleted file mode 100644 index ebe5a86..0000000 Binary files a/_doc/sphinxdoc/source/nbcov-2018-04-09.png and /dev/null differ diff --git a/_unittests/ut_fctmr/test_parallel_fctmr.py b/_unittests/ut_fctmr/test_parallel_fctmr.py index e923455..cacadd9 100644 --- a/_unittests/ut_fctmr/test_parallel_fctmr.py +++ b/_unittests/ut_fctmr/test_parallel_fctmr.py @@ -57,7 +57,7 @@ def map_custom(input): class TestParallelFctMr(ExtTestCase): - def test_parallel_mapper(self): + def test_parallel_mapper_sig(self): def func2(x): return x + 1 li = numpy.array(list(range(0, 1000000)), dtype=numpy.float64) @@ -72,6 +72,21 @@ def func2(x): self.assertEqual(out[:10], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]) + def test_parallel_mapper_nosig(self): + def func2(x): + return x + 1 + li = numpy.array(list(range(0, 1000000)), dtype=numpy.float64) + out = list(parallel_mapper( + func2, li, nogil=False, nopython=False)) + self.assertEqual(out[:10], [1.0, 2.0, 3.0, 4.0, + 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]) + out = map_custom(li) + self.assertEqual(out[:10], numpy.array( + [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0])) + out = list(map(func2, li)) + self.assertEqual(out[:10], [1.0, 2.0, 3.0, 4.0, + 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]) + if __name__ == "__main__": unittest.main() diff --git a/appveyor.yml b/appveyor.yml index d92e1b0..b3827bf 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -21,7 +21,7 @@ install: - "%PYTHON%\\Scripts\\pymy_install3 --set=pyquickhelper" - "%PYTHON%\\Scripts\\pymy_install3 --task=tool --source=zip graphviz" - "%PYTHON%\\Scripts\\pip install pyquickhelper --no-deps" - - "%PYTHON%\\Scripts\\pip install wcwidth guzzle_sphinx_theme backcall pyensae yahoo_historical qgrid" + - "%PYTHON%\\Scripts\\pip install wcwidth backcall pyensae yahoo_historical qgrid sphinx_bootstrap_theme" - "%PYTHON%\\Scripts\\pymy_install3 scikit-learn scipy cffi statsmodels cytoolz Pillow numba" - "set PATH=%PATH%;C:\\projects\\jyquickhelper\\build\\update_modules\\Graphviz\\bin" build: off diff --git a/requirements.txt b/requirements.txt index cdd83e7..1960013 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ pymyinstall pyquickhelper>=1.7.2508 pyspark qgrid -sphinx-redactor-theme +sphinx_bootstrap_theme sphinxjp.themes.basicstrap sphinxjp.themes.sphinxjp wheel diff --git a/src/sparkouille/fctmr/parallel_fctmr.py b/src/sparkouille/fctmr/parallel_fctmr.py index fe22ae2..537bffe 100644 --- a/src/sparkouille/fctmr/parallel_fctmr.py +++ b/src/sparkouille/fctmr/parallel_fctmr.py @@ -73,30 +73,53 @@ def loop(nb, inputs, outputs): nopython=nopython, cache=True)(loop) inputs = create_array_numba(chunk_size, sigin) outputs = create_array_numba(chunk_size, sigout) + + done = 0 + for obs in gen: + if done < len(inputs): + inputs[done] = obs + done += 1 + else: + loop_jit(done, inputs, outputs) + for out in outputs: + yield out + done = 0 + if 0 < done < len(inputs): + loop_jit(done, inputs, outputs) + for out in outputs: + yield out + else: - fct_jit = jit(nogil=nogil, parallel=parallel, - nopython=nopython, cache=True)(fct) def loop(nb, inputs, outputs): for i in prange(nb): outputs[i] = fct_jit(inputs[i]) - loop_jit = njit(nogil=nogil, parallel=parallel, - nopython=nopython, cache=True)(loop) - inputs = [None for i in range(0, chunk_size)] - outputs = [None for i in range(0, chunk_size)] - - done = 0 - for obs in gen: - if done < len(inputs): - inputs[done] = obs - done += 1 - else: + loop_jit = None + fct_jit = None + + inputs = None + outputs = None + + done = 0 + for obs in gen: + if inputs is None: + inputs = [obs] * chunk_size + outputs = [fct(obs)] * chunk_size + loop_jit = njit(nogil=nogil, parallel=parallel, + nopython=nopython, cache=True)(loop) + fct_jit = jit(nogil=nogil, parallel=parallel, + nopython=nopython, cache=True)(fct) + + if done < len(inputs): + inputs[done] = obs + done += 1 + else: + loop_jit(done, inputs, outputs) + for out in outputs: + yield out + done = 0 + if 0 < done < len(inputs): loop_jit(done, inputs, outputs) for out in outputs: yield out - done = 0 - if 0 < done < len(inputs): - loop_jit(done, inputs, outputs) - for out in outputs: - yield out diff --git a/src/sparkouille/fctmr/simplefctmr.py b/src/sparkouille/fctmr/simplefctmr.py index f712ce2..8278f9e 100644 --- a/src/sparkouille/fctmr/simplefctmr.py +++ b/src/sparkouille/fctmr/simplefctmr.py @@ -67,7 +67,7 @@ def take(gen, count=5, skip=0): @return generator .. exref:: - :title: mapper + :title: take :tag: progfonc .. runpython:: @@ -96,7 +96,7 @@ def ffilter(fct, gen): @return generator .. exref:: - :title: mapper + :title: filter :tag: progfonc .. runpython:: @@ -164,7 +164,7 @@ def combiner(fctkey1, gen1, fctkey2, gen2, how='inner'): @return generator .. exref:: - :title: reducer + :title: combiner or join :tag: progfonc .. runpython::