diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..e3e41df --- /dev/null +++ b/.coveragerc @@ -0,0 +1,9 @@ +[run] +source = drmaa +omit = + */python?.?/* + */lib-python/?.?/*.py + */lib_pypy/_*.py + */site-packages/ordereddict.py + */site-packages/nose/* + */unittest2/* diff --git a/.gitignore b/.gitignore index d2d6f36..5039d46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,11 @@ -*.py[cod] +# Compiled python +*.pyc +*.pyo +__pycache__ + + +.DS_Store +.coverage # C extensions *.so diff --git a/.travis.yml b/.travis.yml index fd36c3e..9905e7c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,12 +2,14 @@ language: python python: - 2.6 - 2.7 - - 3.2 + - 3.3 notifications: email: false # Install stuff +virtualenv: + system_site_packages: true before_install: - travis/install_sge.sh - export GRID_MAP_REDIS_PORT=12345 @@ -15,9 +17,15 @@ before_install: - export SGE_CELL=default - export DRMAA_LIBRARY_PATH=/usr/lib/libdrmaa.so.1.0 install: + - pip install python-coveralls --use-mirrors + - pip install nose-cov --use-mirrors - python setup.py install # Run test script: - - nosetests -d -v + - nosetests -d -v --with-cov --cov drmaa --cov-config .coveragerc --logging-level=DEBUG + +# Calculate coverage +after_success: + - coveralls --config_file .coveragerc diff --git a/MANIFEST.in b/MANIFEST.in index d337c8c..7cf3035 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ include license.txt include *.rst recursive-include examples * recursive-include test * +recursive-include docs * diff --git a/README.rst b/README.rst index cae669d..fbf5bd3 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,61 @@ -drmaa-python +DRMAA Python ------------ - -.. image:: https://travis-ci.org/dan-blanchard/drmaa-python.png - :target: https://travis-ci.org/dan-blanchard/drmaa-python +.. image:: https://travis-ci.org/drmaa-python/drmaa-python.png + :target: https://travis-ci.org/drmaa-python/drmaa-python :alt: Travis build status -DRMAA bindings for Python. Now Python 3 compatible. +.. image:: https://coveralls.io/repos/drmaa-python/drmaa-python/badge.png + :target: https://coveralls.io/r/drmaa-python/drmaa-python + :alt: Test coverage + +.. image:: https://pypip.in/d/drmaa/badge.png + :target: https://crate.io/packages/drmaa + :alt: PyPI downloads + +.. image:: https://pypip.in/v/drmaa/badge.png + :target: https://crate.io/packages/drmaa + :alt: Latest version on PyPI + +`Distributed Resource Management Application API `__ +(DRMAA) bindings for Python. For more information +`read the docs `__. + +Requirements +~~~~~~~~~~~~ + +- Python 2.6+ +- A DRMAA-compatible cluster (e.g., Grid Engine) + +License +~~~~~~~ + +- BSD (3 Clause) + +Changelog +~~~~~~~~~ + +- v0.7.1 + + - Add `Read The Docs documentation `__ + - Add ``const`` module identifiers back into package namespace + - Remove ``b`` prefixes from strings inserted into error messages. + +- v0.7.0 + - String attribute issues with Python 3 have all been resolved, and now each + function that takes a string can handle unicode strings, and returns + unicode strings. + - All code has been updated to use future imports for ``unicode_literals`` + and ``print_function``, so we're effectively writing Python 3 code now. + - PEP8 compliance changes all over the place, except those that would break + names required by underlying C DRMAA library. + - Now automatically run unit tests of Travis-CI with SGE, and all tests pass + for Python 2.6, 2.7, and 3.3. SGE is installed using scripts I describe + in `this gist `__. + - Unit tests are now in a top-level directory instead of a sub-directory + under the drmaa package. + - There is now a `session.py` module that contains most of the code that was + in ``__init__.py`` before, and ``__init__`` just imports things and sets + ``__all__`` and ``__version__``, as is typically recommended now. + - Drops support for Python 2.5. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..6345ed8 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/DRMAAPython.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/DRMAAPython.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/DRMAAPython" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/DRMAAPython" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..c40f88c --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,271 @@ +# -*- coding: utf-8 -*- +# +# DRMAA Python documentation build configuration file, created by +# sphinx-quickstart on Wed Nov 27 13:52:19 2013. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import os +import sys + + +# Handle modules that will not import properly because they rely on C +class Mock(object): + def __init__(self, *args, **kwargs): + pass + + def __call__(self, *args, **kwargs): + return Mock() + + @classmethod + def __getattr__(cls, name): + if name in ('__file__', '__path__'): + return '/dev/null' + elif name[0] == name[0].upper(): + mockType = type(name, (), {}) + mockType.__module__ = __name__ + return mockType + else: + return Mock() + +MOCK_MODULES = ['drmaa.wrappers'] +for mod_name in MOCK_MODULES: + sys.modules[mod_name] = Mock() + + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('..')) + +import drmaa + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', + 'sphinx.ext.pngmath', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'DRMAA Python' +copyright = u'2013, Dan Blanchard, David Ressman, Enrico Sirola' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = drmaa.__version__ +# The full version, including alpha/beta/rc tags. +release = drmaa.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'DRMAAPythondoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'DRMAAPython.tex', u'DRMAA Python Documentation', + u'Dan Blanchard, David Ressman, Enrico Sirola', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'drmaapython', u'DRMAA Python Documentation', + [u'Dan Blanchard, David Ressman, Enrico Sirola'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'DRMAAPython', u'DRMAA Python Documentation', + u'Dan Blanchard, David Ressman, Enrico Sirola', 'DRMAAPython', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' diff --git a/docs/drmaa.rst b/docs/drmaa.rst new file mode 100644 index 0000000..89ad02f --- /dev/null +++ b/docs/drmaa.rst @@ -0,0 +1,7 @@ +:mod:`drmaa` Package +---------------------- + +.. automodule:: drmaa + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..c49d8ad --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,26 @@ +.. DRMAA Python documentation master file, created by + sphinx-quickstart on Wed Nov 27 13:52:19 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to DRMAA Python's documentation! +======================================== + + + +Contents: + +.. toctree:: + :maxdepth: 2 + + tutorials + drmaa + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..edbf62f --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,190 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\DRMAAPython.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\DRMAAPython.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/docs/tutorials.rst b/docs/tutorials.rst new file mode 100644 index 0000000..e11cb3f --- /dev/null +++ b/docs/tutorials.rst @@ -0,0 +1,405 @@ +Distributed Resource Management Application API +=============================================== + +This guide is a tutorial for getting started programming with DRMAA. It is +basically a one to one translation of the original in C for Grid Engine. It +assumes that you already know what DRMAA is and that you have drmaa-python +installed. If not, have a look at Installing. The following code segments are +also included in the repository. + +Starting and Stopping a Session +------------------------------- + +The following code segments (example1.py and example1.1.py) shows the most basic +DRMAA python binding program. + +.. code-block:: python + + #!/usr/bin/env python + + import drmaa + + def main(): + """Create a drmaa session and exit""" + with drmaa.Session() as s: + print('A session was started successfully') + + if __name__=='__main__': + main() + + +The first thing to notice is that every call to a DRMAA function will return an +error code. In this tutorial, we ignore all error codes. + +Now let's look at the functions being called. First, on line 7, we initialise a +Session object by calling DRMAA.Session(). The Session is automatically +initialized via initialize(), and it creates a session and starts an event +client listener thread. The session is used for organizing jobs submitted +through DRMAA, and the thread is used to receive updates from the queue master +about the state of jobs and the system in general. Once initialize() has been +called successfully, it is the responsibility of the calling application to also +call exit() before terminating. If an application does not call exit() before +terminating, session state may be left behind in the user's home directory, and +the queue master may be left with a dead event client handle, which can decrease +queue master performance. + +At the end of our program, on line 9, we call exit(). exit() cleans up the +session and stops the event client listener thread. Most other DRMAA functions +must be called before exit(). Some functions, like getContact(), can be called +after exit(), but these functions only provide general information. Any function +that does work, such as runJob() or wait() must be called before exit() is +called. If such a function is called after exit() is called, it will return an +error. + + +.. code-block:: python + + #!/usr/bin/env python + + import drmaa + + def main(): + """Create a session, show that each session has an id, + use session id to disconnect, then reconnect. Then exit""" + s = drmaa.Session() + s.initialize() + print('A session was started successfully') + response = s.contact + print('session contact returns: %s' % response) + s.exit() + print('Exited from session') + + s.initialize(response) + print('Session was restarted successfullly') + s.exit() + + + if __name__=='__main__': + main() + +This example is very similar to Example 1. The difference is that it uses the +Grid Engine feature of reconnectable sessions. The DRMAA concept of a session is +translated into a session tag in the Grid Engine job structure. That means that +every job knows to which session it belongs. With reconnectable sessions, it's +possible to initialize the DRMAA library to a previous session, allowing the +library access to that session's job list. The only limitation, though, is that +jobs which end between the calls to exit() and init() will be lost, as the +reconnecting session will no longer see these jobs, and so won't know about +them. + +Through line 9, this example is very similar to Example 1. On line 10, however, +we use the contact attribute to get the contact information for this session. On +line 12 we then exit the session. On line 15, we use the stored contact +information to reconnect to the previous session. Had we submitted jobs before +calling exit(), those jobs would now be available again for operations such as +wait() and synchronize(). Finally, on line 17 we exit the session a second time. + +Running a Job +------------- + +The following code segment (example2.py and example2.1.py) shows how to use the +DRMAA python binding to submit a job to Grid Engine. It submits a small shell +script (sleeper.sh) which takes two arguments: + +.. code-block:: bash + + #!/bin/bash + echo "Hello world, the answer is $1" + sleep 3s + echo "$2 Bye world!" + +.. code-block:: python + + #!/usr/bin/env python + + import drmaa + import os + + def main(): + """Submit a job. + Note, need file called sleeper.sh in current directory. + """ + with drmaa.Session() as s: + print('Creating job template') + jt = s.createJobTemplate() + jt.remoteCommand = os.path.join(os.getcwd(), 'sleeper.sh') + jt.args = ['42', 'Simon says:'] + jt.joinFiles=True + + jobid = s.runJob(jt) + print('Your job has been submitted with id %s' % jobid) + + print('Cleaning up') + s.deleteJobTemplate(jt) + + if __name__=='__main__': + main() + +The beginning and end of this program are the same as the previous one. What's +different is in lines 13-23. On line 13 we ask DRMAA to allocate a job template +for us. A job template is a structure used to store information about a job to +be submitted. The same template can be reused for multiple calls to runJob() or +runBulkJob(). + +On line 14 we set the REMOTE_COMMAND attribute. This attribute tells DRMAA where +to find the program we want to run. Its value is the path to the executable. The +path be be either relative or absolute. If relative, it is relative to the WD +attribute, which if not set defaults to the user's home directory. For more +information on DRMAA attributes, please see the attributes man page. Note that +for this program to work, the script "sleeper.sh" must be in the current +directory. + +On line 15 we set the V_ARGV attribute. This attribute tells DRMAA what +arguments to pass to the executable. For more information on DRMAA attributes, +please see the attributes man page. + +On line 18 we submit the job with runJob(). DRMAA will place the id assigned to +the job into the character array we passed to runJob(). The job is now running +as though submitted by qsub or bsub. At this point calling exit() and/or +terminating the program will have no effect on the job. + +To clean things up, we delete the job template on line 22. This frees the memory +DRMAA set aside for the job template, but has no effect on submitted jobs. +Finally, on line 23, we call exit(). + +If instead of a single job we had wanted to submit an array job, we could have +replaced the else on line 18 and 19 with the following: + +.. code-block:: python + + jobid = s.runBulkJobs(jt, 1, 30, 2) + print('Your job has been submitted with id %s' % jobid) + + +This code segment submits an array job with 15 tasks numbered 1, 3, 5, 7, etc. +An important difference to note is that runBulkJobs() returns the job ids in an +array. On line 19, we print all the job ids. + +Waiting for a Job +----------------- + +Now we're going to extend our example to include waiting for a job to finish +(example3.py, example3.1.py and example3.2.py). + +.. code-block:: python + + #!/usr/bin/env python + + import drmaa + import os + + def main(): + """Submit a job and wait for it to finish. + Note, need file called sleeper.sh in home directory. + """ + with drmaa.Session() as s: + print('Creating job template') + jt = s.createJobTemplate() + jt.remoteCommand = os.path.join(os.getcwd(), 'sleeper.sh') + jt.args = ['42', 'Simon says:'] + jt.joinFiles = True + + jobid = s.runJob(jt) + print('Your job has been submitted with id %s' % jobid) + + retval = s.wait(jobid, drmaa.Session.TIMEOUT_WAIT_FOREVER) + print('Job: {0} finished with status {1}'.format(retval.jobId, retval.hasExited)) + + print('Cleaning up') + s.deleteJobTemplate(jt) + + if __name__=='__main__': + main() + + +This example is very similar to Example 2 except for line 21. On line 21 we call +wait() to wait for the job to end. We have to give wait() both the id of the job +for which we want to wait and a place to write the id of the job for which we +actually waited because the job id we pass in could be JOB_IDS_SESSION_ANY, in +which case wait() must have a way of tell us which job is the one that made it +return. We also have to pass to wait() how long we are willing to wait for the +job to finish. This could be a number of seconds, or it could be either +TIMEOUT_WAIT_FOREVER or TIMEOUT_NO_WAIT. Lastly, we collect the exit status. +Assuming the wait worked, we query the job's exit status on line 22. + +An alternative to wait() when working with multiple jobs, such as jobs submitted +by runBulkJobs() or multiple calls to runJob() is synchronize(). synchronize() +waits for a set of jobs to finish. To use synchronize(), we could replace lines +18-22 with the following: + +.. code-block:: python + + joblist = s.runBulkJobs(jt, 1, 30, 2) + print('Your job has been submitted with id %s' % joblist) + + s.synchronize(joblist, drmaa.Session.TIMEOUT_WAIT_FOREVER, True) + + +Line 18 now call runBulkJobs() so that we have several jobs for which to wait. +On line 21, instead of calling wait(), we call synchronize(). synchronize() +takes only three interesting parameters. The first is the list of ids for which +to wait. This list must be a NULL-terminated array of strings. If the special +id, JOB_IDS_SESSION_ALL, appears in the array, synchronize() will wait for all +jobs submitted via DRMAA during this session, i.e. since initialize() was +called. The second is how long to wait for all the jobs in the list to finish. +This is the same as the timeout parameter for wait(). The third is whether this +call to synchronize() should clean up after the job. After a job completes, it +leaves behind accounting information, such as exist status and usage, until +either wait() or synchronize() with dispose set to true is called. It is the +responsibility of the application to make sure one of these two functions is +called for every job. Not doing so creates a memory leak. Note that calling +synchronize() with dispose set to true flushes all accounting information for +all jobs in the list. If you want to use synchronize() and still recover the +accounting information, set dispose to false and call wait() for each job. To do +this in Example 3, we would replace lines 18--22 with the following: + +.. code-block:: python + + joblist = s.runBulkJobs(jt, 1, 30, 2) + print('Your job has been submitted with id %s' % joblist) + + s.synchronize(joblist, drmaa.Session.TIMEOUT_WAIT_FOREVER, False) + for curjob in joblist: + print('Collecting job ' + curjob) + retval = s.wait(curjob, drmaa.Session.TIMEOUT_WAIT_FOREVER) + print('Job: {0} finished with status {1}'.format(retval.jobId, + retval.hasExited)) + + +What's different is that on line 22 we set dispose to false, and then on lines +23-26 we wait once for each job, printing the exit status and usage information +as we did in Example 3. + +We pass joblist to synchronise to wait for each job specifically. Otherwise, the +wait() could end up waiting for a job submitted after the call to synchronize(). + +Controlling a Job +----------------- + +Now let's look at an example of how to control a job from DRMAA (example4.py): + +.. code-block:: python + + #!/usr/bin/env python + + import drmaa + import os + + def main(): + """Submit a job, then kill it. + Note, need file called sleeper.sh in home directory. + """ + with drmaa.Session() as s: + print('Creating job template') + jt = s.createJobTemplate() + jt.remoteCommand = os.path.join(os.getcwd(), 'sleeper.sh') + jt.args = ['42', 'Simon says:'] + jt.joinFiles = True + + jobid = s.runJob(jt) + print('Your job has been submitted with id %s' % jobid) + # options are: SUSPEND, RESUME, HOLD, RELEASE, TERMINATE + s.control(jobid, drmaa.JobControlAction.TERMINATE) + + print('Cleaning up') + s.deleteJobTemplate(jt) + + if __name__=='__main__': + main() + + +This example is very similar to Example 2 except for line 21. On line 21 we use +control() to delete the job we just submitted. Aside from deleting the job, we +could have also used control() to suspend, resume, hold, or release it. For more +information, see the control man page. + +Note that control() can be used to control jobs not submitted through DRMAA. Any +valid SGE job id could be passed to control() as the id of the job to delete. + +Getting Job Status +------------------ + +Here's an example of using DRMAA to query the status of a job (example5.py): + +.. code-block:: python + + #!/usr/bin/env python + + import drmaa + import time + import os + + def main(): + """Submit a job, and check its progress. + Note, need file called sleeper.sh in home directory. + """ + with drmaa.Session() as s: + print('Creating job template') + jt = s.createJobTemplate() + jt.remoteCommand = os.path.join(os.getcwd(), 'sleeper.sh') + jt.args = ['42', 'Simon says:'] + jt.joinFiles=True + + jobid = s.runJob(jt) + print('Your job has been submitted with id %s' % jobid) + + # Who needs a case statement when you have dictionaries? + decodestatus = {drmaa.JobState.UNDETERMINED: 'process status cannot be determined', + drmaa.JobState.QUEUED_ACTIVE: 'job is queued and active', + drmaa.JobState.SYSTEM_ON_HOLD: 'job is queued and in system hold', + drmaa.JobState.USER_ON_HOLD: 'job is queued and in user hold', + drmaa.JobState.USER_SYSTEM_ON_HOLD: 'job is queued and in user and system hold', + drmaa.JobState.RUNNING: 'job is running', + drmaa.JobState.SYSTEM_SUSPENDED: 'job is system suspended', + drmaa.JobState.USER_SUSPENDED: 'job is user suspended', + drmaa.JobState.DONE: 'job finished normally', + drmaa.JobState.FAILED: 'job finished, but failed'} + + for ix in range(10): + print('Checking %s of 10 times' % ix) + print decodestatus(s.jobStatus(jobid)) + time.sleep(5) + + print('Cleaning up') + s.deleteJobTemplate(jt) + + if __name__=='__main__': + main() + +Again, this example is very similar to Example 2, this time with the exception +of lines 22-40. On line 38, we use jobStatus() to get the status of the job. +Line 43 determine what the job status is and report it. + +Getting DRM information +----------------------- + +Lastly, let's look at how to query the DRMAA library for information about the +DRMS and the DRMAA implementation itself (example6.py): + +.. code-block:: python + + #!/usr/bin/env python + + import drmaa + + def main(): + """Query the system.""" + with drmaa.Session() as s: + print('A DRMAA object was created') + print('Supported contact strings: %s' % s.contact) + print('Supported DRM systems: %s' % s.drmsInfo) + print('Supported DRMAA implementations: %s' % s.drmaaImplementation) + print('Version %s' % s.version) + + print('Exiting') + + if __name__=='__main__': + main() + +On line 9, we get the contact string list. This is the list of contact strings +that will be understood by this DRMAA instance. Normally on of these strings is +used to select to which DRM this DRMAA instance should be bound. On line 10, we +get the list of supported DRM systems. On line 11, we get the list of supported +DRMAA implementations. On line 12, we get the version number of the DRMAA C +binding specification supported by this DRMAA implementation. Finally, on line +15, we end the session with exit(). \ No newline at end of file diff --git a/drmaa/__init__.py b/drmaa/__init__.py index 4f9dd3f..87421a2 100644 --- a/drmaa/__init__.py +++ b/drmaa/__init__.py @@ -23,6 +23,9 @@ This package is an implementation of the DRMAA 1.0 Python language binding specification (http://www.ogf.org/documents/GFD.143.pdf). See http://drmaa-python.googlecode.com for package info and download. + +:author: Enrico Sirola (enrico.sirola@statpro.com) +:author: Dan Blanchard (dblanchard@ets.org) """ from __future__ import absolute_import, print_function, unicode_literals @@ -58,10 +61,10 @@ UnsupportedAttributeException, InvalidArgumentException, InvalidAttributeValueException, OutOfMemoryException) from .session import JobInfo, JobTemplate, Session +from .version import __version__, VERSION __docformat__ = "restructuredtext en" -__version__ = "0.7" __all__ = ['JobInfo', 'JobTemplate', 'Session', 'AlreadyActiveSessionException', diff --git a/drmaa/version.py b/drmaa/version.py new file mode 100644 index 0000000..7bc36a6 --- /dev/null +++ b/drmaa/version.py @@ -0,0 +1,26 @@ +# ----------------------------------------------------------- +# Copyright (C) 2008 StatPro Italia s.r.l. +# +# StatPro Italia +# Via G. B. Vico 4 +# I-20123 Milano +# ITALY +# +# phone: +39 02 96875 1 +# fax: +39 02 96875 605 +# +# This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the license for more details. +# ----------------------------------------------------------- +# +''' +This module exists solely for version information so I only have to change it +in one place. Based on the suggestion `here. `_ + +:author: Dan Blanchard (dblanchard@ets.org) +''' + +__version__ = '0.7.1' +VERSION = tuple(int(x) for x in __version__.split('.')) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5e40900 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[wheel] +universal = 1 diff --git a/setup.py b/setup.py index d42a162..ecb5490 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + # ----------------------------------------------------------- # Copyright (C) 2009 StatPro Italia s.r.l. # @@ -23,30 +25,29 @@ packages = find_packages() package_data = dict([ (x, ['test/*.py']) for x in packages]) -setup( - name="drmaa", - version="0.7", - packages=packages, - package_data=package_data, - author="David Ressman", - author_email="davidr@ressman.org", - description="a python DRMAA library", - license="BSD", - keywords="python grid hpc drmaa", - url="https://github.com/drmaa-python/drmaa-python", - tests_require='nose', - test_suite='nose.collector', - classifiers="""\ -Development Status :: 4 - Beta -Operating System :: OS Independent -Intended Audience :: System Administrators -Intended Audience :: Developers -Intended Audience :: Science/Research -License :: OSI Approved :: BSD License -Programming Language :: Python :: 2 -Programming Language :: Python :: 3 -Topic :: Software Development :: Libraries :: Python Modules -Topic :: System :: Distributed Computing""".split('\n'), -) - +# To get around the fact that you can't import stuff from packages in setup.py +exec(compile(open('drmaa/version.py').read(), 'drmaa/version.py', 'exec')) +# (we use the above instead of execfile for Python 3.x compatibility) +setup(name="drmaa", + version=__version__, + packages=packages, + package_data=package_data, + author="Dan Blanchard", + author_email="dblanchard@ets.org", + description="a python DRMAA library", + license="BSD", + keywords="python grid hpc drmaa", + url="https://github.com/drmaa-python/drmaa-python", + tests_require='nose', + test_suite='nose.collector', + classifiers=["Development Status :: 4 - Beta", + "Operating System :: OS Independent", + "Intended Audience :: System Administrators", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Libraries", + "Topic :: System :: Distributed Computing"])