Merge pull request #2 from stefco/dev

r 0.2.0 Merge dev

Makefile

@@ -2,32 +2,47 @@
 # Modified for PyPI usage
 #
 
+# virtualenv prefix
+ifdef VIRTUAL_ENV
+VIR_ENV_PRE   = $(VIRTUAL_ENV)/bin/
+else
+ifneq ($(shell test -d env/bin), 0)
+VIR_ENV_PRE   = $(shell pwd -P)/env/bin/
+endif
+endif
+
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-TWINEBUILD    = twine
+SPHINXBUILD   = $(VIR_ENV_PRE)sphinx-build
+PYTHON        = $(VIR_ENV_PRE)python
+TWINE         = $(VIR_ENV_PRE)twine
+PIP           = $(VIR_ENV_PRE)pip
 PAPER         =
+CONFDIR       = docs
 BUILDDIR      = _build
 PYPIBUILDDIR  = build
 PYPIDISTDIR   = dist
+MODULENAME    = geco_stat
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
 PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(CONFDIR)
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 
-.PHONY: help check check-twine check-sphinx check-env clean pypi build upload env html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
+.PHONY: help check check-twine check-sphinx check-env clean distclean pypi build upload env html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck test doctest coverage gettext
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  check      to check dependencies"
 	@echo "  env        to create a virtualenv for development"
-	@echo "  pypi       to build and upload packages to PyPI"
+	@echo "  check      to check dependencies"
+	@echo "  pypi       to build and upload packages to PyPI all at once"
 	@echo "  build      to build packages for upload to PyPI"
 	@echo "  upload     to upload finished PyPI packages"
 	@echo "  html       to make standalone HTML files"
+	@echo "  clean      to remove generated files, but keep the env folder"
+	@echo "  distclean  to remove generated files as well as the env folder"
 	@echo "  dirhtml    to make HTML files named index.html in directories"
 	@echo "  singlehtml to make a single large HTML file"
 	@echo "  pickle     to make pickle files"
@@ -49,42 +64,50 @@ help:
 	@echo "  xml        to make Docutils-native XML files"
 	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
 	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  test       to run all doctests as well as unit tests"
 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 	@echo "  coverage   to run coverage check of the documentation (if enabled)"
 
 # User-friendly check for all dependencies.
 check: check-env check-twine check-sphinx
-	echo Dependency checks passed!
+	printf "\nDependency checks passed!\n"
 
 # User-friendly check for twine. indentation puts it in the test proper... v confusing...
 check-twine:
-ifeq ($(shell which $(TWINEBUILD) >/dev/null 2>&1; echo $$?), 1)
-	$(error The '$(TWINEBUILD)' command was not found. Make sure you have Twine installed, which you can do with 'pip install twine'. See this link for more details on using Twine and PyPI: http://python-packaging-user-guide.readthedocs.org/en/latest/distributing/#setup-py)
+ifeq ($(shell which $(TWINE) >/dev/null 2>&1; echo $$?), 1)
+	$(error The '$(TWINE)' command was not found. Make sure you have Twine installed, which you can do with 'pip install twine'. See this link for more details on using Twine and PyPI: http://python-packaging-user-guide.readthedocs.org/en/latest/distributing/#setup-py)
 endif
 
 # User-friendly check for sphinx-build
 check-sphinx:
 ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
-	$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+	$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don not have Sphinx installed, grab it from http://sphinx-doc.org/)
 endif
 
 # User-friendly check for virtualenv
 check-env:
-ifndef VIRTUAL_ENV
-	$(error 'VIRTUAL_ENV is undefined; this means you are not working in a virtual environment, which you should ideally be doing. Run "make env" to install a virtual environment using python virtualenv and follow instructions to activate it.')
+ifndef VIR_ENV_PRE
+	$(error 'VIR_ENV_PRE is undefined; this means you are not working in a virtual environment and do not have one installed in this directory. Run "make env" to install a virtual environment using python virtualenv and follow instructions to activate it.')
+else
+ifdef VIRTUAL_ENV
+	printf "\nvirtual environment ACTIVE, VIRTUAL_ENV is set.\n\nRun\n\n\tdeactivate\n\nto deactivate.\n\n"
+else
+	printf "\nvirtual environment INACTIVE, VIRTUAL_ENV is not set.\n\nRun\n\n\tsource env/bin/activate\n\nto activate.\n\n"
+endif
+	printf "environment used by make: $(VIR_ENV_PRE) \n"
 endif
 
 env:
 	printf "\nConfiguring a virtual environment for python...\n\n"
 # Install a virtualenv to allow for local development and usage.
 	if ! [ -e "./env" ]; then virtualenv env; fi
 # Install some stuff required for pip packaging and development
-	env/bin/pip install -U "pip>=1.4" "setuptools>=0.9" "wheel>=0.21" twine
+	$(PIP) install -U "pip>=1.4" "setuptools>=0.9" "wheel>=0.21" twine
 # Install sphinx itself
-	env/bin/pip install -U "sphinx" "recommonmark"
+	$(PIP) install -U "sphinx" "recommonmark"
 # Install required packages
-	env/bin/pip install -U "numpy" "matplotlib" "h5py"
-	printf "\nDone setting up! Run\n\n\tsource env/bin/activate\n\nto start working in this virtual environment, and run\n\n\tdeactivate\n\nwhen finished to return to your normal setup.\n\nFor nice documentation on virtualenv, visit:\nhttps://www.dabapps.com/blog/introduction-to-pip-and-virtualenv-python/\n"
+	env/bin/pip install -U "numpy" "matplotlib" "h5py" "tendo"
+	printf "\nDone setting up! To use the virtual environment interactively, run\n\n\tsource env/bin/activate\n\nto start working in this virtual environment, and run\n\n\tdeactivate\n\nwhen finished to return to your normal setup.\n\nFor nice documentation on virtualenv, visit:\nhttps://www.dabapps.com/blog/introduction-to-pip-and-virtualenv-python/\n"
 
 clean:
 # Delete autogenerated sphinx files
@@ -97,6 +120,8 @@ clean:
 	rm -rf *egg-info
 # Delete autogenerated .pyc files
 	rm -rf *.pyc
+
+distclean: clean
 # Delete virtualenv
 	echo "Deleting virtualenv..."
 	rm -rf env
@@ -105,12 +130,12 @@ clean:
 pypi: build upload
 
 build: check-env
-	python setup.py sdist
-	python setup.py bdist_wheel
-	@echo "Build finished. You can now upload by running make upload (did you update release?)"
+	$(PYTHON) setup.py sdist
+	$(PYTHON) setup.py bdist_wheel --universal
+	@echo "Build finished. You can now upload by running make upload (did you update version.py?)"
 
 upload: check-env check-twine
-	$(TWINEBUILD) upload $(PYPIDISTDIR)/*
+	$(TWINE) upload $(PYPIDISTDIR)/*
 
 html: check-sphinx
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@@ -232,6 +257,10 @@ linkcheck: check-sphinx
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in $(BUILDDIR)/linkcheck/output.txt."
 
+test: doctest
+	printf "\nRunning unit tests...\n\n"
+	$(PYTHON) -c "import $(MODULENAME); $(MODULENAME).run_unit_tests()"
+
 doctest: check-sphinx
 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
 	@echo "Testing of doctests in the sources finished, look at the " \

README.md

@@ -1,7 +1,9 @@
-# Introduction to GECo Statistics Reports
+# GECo Statistics Reports
 
+[![PyPI Version](https://img.shields.io/pypi/v/geco-stat.svg)](https://pypi.python.org/pypi/geco-stat)
+[![PyPI Project Status](https://img.shields.io/pypi/status/geco-stat.svg)](https://pypi.python.org/pypi/geco-stat)
 [![Documentation Status](https://readthedocs.org/projects/geco-statistics/badge/?version=latest)](http://geco-statistics.readthedocs.org/en/latest/?badge=latest)
-[ ![Codeship Status for stefco/geco_stat](https://codeship.com/projects/e9762300-bd59-0133-0ed3-2a1d867cc1c8/status?branch=master)](https://codeship.com/projects/136547)
+[![Codeship Test Status](https://img.shields.io/codeship/e9762300-bd59-0133-0ed3-2a1d867cc1c8/master.svg)](https://codeship.com/projects/136547)
 
 ## Purpose
 
@@ -14,26 +16,45 @@ ways to combine reports, so that very long timeseries can be efficiently
 analyzed in parallel and then recombined into single, monolithic reports
 covering entire eras.
 
-Future versions of `geco_statistics` will also provide tools for automating
-report generation, reducing the amount of effort that has to go into creating
-reports on LIGO's functioning.
-
-## Dependencies
-
-This is a python-2.6 compatible Python module intended for use in
-LIGO production environments. Its python dependencies are minimal, which means
-it should work out of the box on most python distributions for viewing and
-manipulating existing reports generated using the package. At time of writing,
-generating *new* reports requires tools installed in LIGO production
-environments; it is intended primarily to be run on LIGO servers, though
-future implementations may integrate remote access tools.
-
-**If you are just interested in viewing existing diagnostic report data**, you
-can use Python, though iPython is recommended for interactive use. [You can
-install iPython from the project website.](http://ipython.org) If you are
-running this module on a LIGO production environment, no additional
-dependencies should be necessary.
-
-You can get around any of these problems by just running `create_virtualenv.sh`
-(though this does require virtualenv) and then running
-`source env/bin/activate`, after which `python` will work just fine.
+## Getting Started
+
+If you just want to use the latest stable version, you can install it with `pip`:
+
+```bash
+pip install geco_stat
+```
+
+If you are on an environment without root privileges, you can do this by
+creating a virtual environment using `virtualenv` in the folder where you
+would like to work:
+
+```bash
+cd /path/to/working_directory
+virtualenv env
+
+# use the virtualenv version of python and pip
+source env/bin/activate
+pip install geco_stat
+```
+
+When finished with your virtual environment, you can exit it by simply running
+`deactivate`.
+
+To use the module from this repository for development, run:
+```bash
+git clone git@github.com:stefco/geco_stat.git
+```
+
+There is a Makefile included for automating most tasks. To get started, run
+`make env`, which will configure a virtual environment for development.
+
+You can simply type `make` to see a list of targets for `make` (i.e. commands you
+can have `make` execute).
+
+You can run `make test` to run unit and doctests, `make build` to build a
+package for upload to PyPI, `make upload` to upload it, `make check` to make
+sure everything is set up (and to see whether you are in a virtual environment),
+`make html` to generate documentation html files (though they will have a
+different theme than those on Read The Docs), `make clean` to delete
+autogenerated files, and `make distclean` to delete *all* autogenerated files
+and folders (including the `env` directory).

conf.py → docs/conf.py

@@ -41,7 +41,7 @@
 templates_path = ['_templates']
 
 # Add geco_stat to the project path
-sys.path.insert(0,os.path.abspath('./'))
+sys.path.insert(0,os.path.abspath('../'))
 
 # Add markdown support
 from recommonmark.parser import CommonMarkParser

index.rst → docs/index.rst

@@ -11,8 +11,9 @@ Contents:
 .. toctree::
    :maxdepth: 2
 
-   README.md
+   introduction.rst
    geco_stat.rst
+   useful_links.rst
 
 Indices and tables
 ==================

docs/introduction.rst

@@ -0,0 +1,62 @@
+Introduction
+============
+
+Purpose
+-------
+
+`geco_stat` can be used to easily generate histograms, statistics, and
+anomaly reports for timing signals generated by LIGO interferometers. It is
+meant to simplify the handling of massive amounts of diagnostic data by
+providing a simple, organized interface to relevant statistics, easy
+progress tracking for large jobs, effortless data visualization, and convenient
+ways to combine reports, so that very long timeseries can be efficiently
+analyzed in parallel and then recombined into single, monolithic reports
+covering entire eras.
+
+Future versions of `geco_statistics` will also provide tools for automating
+report generation, reducing the amount of effort that has to go into creating
+reports on LIGO's functioning.
+
+Intended Use Cases
+------------------
+
+This is a python-2.6 compatible Python module intended for use in
+LIGO production environments. Its python dependencies are minimal, which means
+it should work out of the box on most python distributions for viewing and
+manipulating existing reports generated using the package. At time of writing,
+generating *new* reports requires tools installed in LIGO production
+environments; it is intended primarily to be run on LIGO servers, though
+future implementations may integrate remote access tools.
+
+**If you are just interested in viewing existing diagnostic report data**, you
+can use Python, though iPython is recommended for interactive use. `You can
+install iPython from the project website`_. If you are
+running this module on a LIGO production environment, no additional
+dependencies should be necessary.
+
+.. _You can install iPython from the project website: http://ipython.org
+
+Getting Started
+---------------
+
+Install with ``pip``:
+
+::
+
+    pip install --upgrade geco-stats
+
+If you don't have root privileges (as is probably the case on a LIGO server),
+change to the directory where you would like to work and run:
+
+::
+
+    virtualenv env
+    source env/bin/activate
+    pip install --upgrade geco-stats
+
+When you are done using your ``virtualenv``, you can run ``deactivate`` to go
+back to using your regular system install of ``python`` and associated tools.
+
+If you are developing or working from source, visit the `github repository`_.
+
+.. _github repository: https://github.com/stefco/geco_stat

docs/useful_links.rst

@@ -0,0 +1,77 @@
+Bibliography (Useful Links)
+===========================
+
+The following links have proved useful in creating, documenting, testing, and
+deploying this package.
+
+Anomaly and Outlier Detection
+-----------------------------
+
+* http://scikit-learn.org/stable/modules/outlier_detection.html
+
+Documentation and Testing using Sphinx
+--------------------------------------
+
+* http://thomas-cokelaer.info/tutorials/sphinx/docstring_python.html
+* https://docs.python.org/2/library/doctest.html
+* http://www.sphinx-doc.org/en/stable/extensions.html
+* http://www.sphinx-doc.org/en/stable/ext/doctest.html
+* http://www.sphinx-doc.org/en/stable/ext/autosummary.html
+* http://www.sphinx-doc.org/en/stable/ext/math.html
+
+Distribution using PyPI and pip
+-------------------------------
+
+* https://hynek.me/articles/sharing-your-labor-of-love-pypi-quick-and-dirty/
+* https://pypi.python.org/pypi?%3Aaction=list_classifiers
+* https://python-packaging.readthedocs.org/en/latest/minimal.html
+* Setting up a ``requirements.txt`` file that automatically gets required
+  dependencies from the ``setup.py`` file: https://caremad.io/2013/07/setup-vs-requirement/
+* A little bit more discussion of the ``setup.py`` ``install_requires``
+  variable: https://packaging.python.org/en/latest/requirements/
+* Including extra requirements for nonstandard usage:
+  https://pythonhosted.org/setuptools/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies
+
+Running a function from the Command Line
+----------------------------------------
+
+* https://docs.python.org/2/tutorial/modules.html#executing-modules-as-scripts
+* http://stackoverflow.com/questions/34952745/how-can-one-enable-a-shell-command-line-argument-for-a-python-package-installed
+* How to make sure that only a single instance of your code is running:
+  http://blog.tplus1.com/blog/2012/08/08/python-allow-only-one-running-instance-of-a-script/
+* A stackoverflow question on the topic with some nice alternative approaches:
+  http://stackoverflow.com/questions/380870/python-single-instance-of-program
+* Documentation on ``sys.excepthook``, used to call cleanup code right before
+  exiting when your program crashes: https://docs.python.org/2/library/sys.html
+
+Using Read the Docs for Documentation
+-------------------------------------
+
+* https://read-the-docs.readthedocs.org/en/latest/getting_started.html
+* Astropy is a very good (and thorough) example of advanced Read The Docs
+  support in Python: https://github.com/astropy/astropy
+
+Writing in Restructured Text (reST)
+-----------------------------------
+
+* I'm used to markdown, so this comparison of Restructured Text (reST) was very
+  helpful: http://www.unexpected-vortices.com/doc-notes/markdown-and-rest-compared.html
+
+Python
+------
+
+* A very good guide to python method decorators: https://julien.danjou.info/blog/2013/guide-python-static-class-abstract-methods
+* Detailed description of ``super()``: https://rhettinger.wordpress.com/2011/05/26/super-considered-super/
+
+Vim Fun
+-------
+
+* Negative regex matching: http://vim.wikia.com/wiki/Search_for_lines_not_containing_pattern_and_other_helpful_searches
+
+Misc.
+-----
+
+* How to schedule jobs on UNIX systems using `chrontab`: http://kvz.io/blog/2007/07/29/schedule-tasks-on-linux-using-crontab/
+* How to schedule larger cluster jobs using Condor: https://www.lsc-group.phys.uwm.edu/lscdatagrid/doc/condorview.html
+* http://stackoverflow.com/questions/592620/check-if-a-program-exists-from-a-bash-script
+