Merge branch 'release/1.0.0'

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.9.5
+current_version = 1.0.0
 commit = True
 tag = True
 message = Bump up to version {new_version}.

.circleci/config.yml

@@ -32,11 +32,11 @@ jobs:
 
       - restore_cache:
           keys:
-            - python-env-v2-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "requirements-dev.txt" }}
-            - python-env-v2-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}
-            - python-env-v2-{{ arch }}-{{ .Environment.CIRCLE_JOB }}
-            - python-env-v2-{{ arch }}
-            - python-env-v2
+            - python-env-v3-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "requirements-dev.txt" }}
+            - python-env-v3-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}
+            - python-env-v3-{{ arch }}-{{ .Environment.CIRCLE_JOB }}
+            - python-env-v3-{{ arch }}
+            - python-env-v3
 
       - run:
           name: install dependencies
@@ -52,19 +52,22 @@ jobs:
               sudo apt update -qq && sudo apt install -qq libhdf5-dev
             fi
             pip install -U pip==18
+            pip install -U codecov
             pip install -U -e . -r requirements-dev.txt
 
       - save_cache:
-          key: python-env-v2-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "requirements-dev.txt" }}
+          key: python-env-v3-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "requirements-dev.txt" }}
           paths:
             - "venv"
 
       - run:
           name: run tests
           command: |
             . venv/bin/activate
-            coverage run --source=signac/ -m unittest discover tests/ -v
-            coverage report -i --include="signac*"
+            coverage run -m unittest discover tests/ -v
+            pip install python-rapidjson==0.7 && coverage run -m unittest discover tests/ -v
+            coverage report -i
+            codecov
 
       - store_artifacts:
           path: test-reports

.coveragerc

@@ -1,2 +1,7 @@
 [run]
-omit = */configobj/*.py
+source=signac
+omit=
+    signac/common/configobj/*.py
+    signac/common/tempdir.py
+    signac/common/tqdm/*.py
+    signac/common/six.py

README.md

@@ -1,58 +1,71 @@
-# signac - simple data management
+# <img src="https://raw.githubusercontent.com/glotzerlab/signac/master/doc/images/logo.png" width="75" height="75"> signac - simple data management
 
-[![DOI](https://zenodo.org/badge/72946496.svg)](https://zenodo.org/badge/latestdoi/72946496)
-[![PyPi](https://img.shields.io/pypi/v/signac.svg)](https://pypi.org/project/signac/)
-[![Anaconda-Server Badge](https://anaconda.org/conda-forge/signac/badges/version.svg)](https://anaconda.org/conda-forge/signac)
-[![conda-forge-downloads](https://img.shields.io/conda/dn/conda-forge/signac.svg)](https://anaconda.org/conda-forge/signac)
-[![RTD](https://readthedocs.org/projects/signac/badge/?version=latest)](https://docs.signac.io)
-[![License](https://img.shields.io/github/license/csadorf/signac.svg)](https://github.com/glotzerlab/signac/blob/master/LICENSE.txt)
+[![PyPI](https://img.shields.io/pypi/v/signac.svg)](https://pypi.org/project/signac/)
+[![conda-forge](https://img.shields.io/conda/vn/conda-forge/signac.svg?style=flat)](https://anaconda.org/conda-forge/signac)
+![CircleCI](https://img.shields.io/circleci/project/github/glotzerlab/signac/master.svg)
+[![RTD](https://img.shields.io/readthedocs/signac.svg?style=flat)](https://docs.signac.io)
+[![License](https://img.shields.io/github/license/glotzerlab/signac.svg)](https://github.com/glotzerlab/signac/blob/master/LICENSE.txt)
+[![PyPI-downloads](https://img.shields.io/pypi/dm/signac.svg?style=flat)](https://pypistats.org/packages/signac)
+[![Gitter](https://img.shields.io/gitter/room/signac/Lobby.svg?style=flat)](https://gitter.im/signac/Lobby)
 
-## About
 
-The [signac framework](http://www.signac.io) aids in the management of large and heterogeneous data spaces.
+The [**signac** framework](https://signac.io) helps users manage and scale file-based workflows, facilitating data reuse, sharing, and reproducibility.
 
 It provides a simple and robust data model to create a well-defined indexable storage layout for data and metadata.
 This makes it easier to operate on large data spaces, streamlines post-processing and analysis and makes data collectively accessible.
 
-**The documentation is available at: [https://docs.signac.io](https://docs.signac.io)**
+## Resources
+
+- [Framework documentation](https://docs.signac.io/):
+  Examples, tutorials, topic guides, and package Python APIs.
+- [Chat Support](https://gitter.im/signac/Lobby):
+  Get help and ask questions on the **signac** gitter channel.
+- [**signac** website](https://signac.io/):
+  Framework overview and news.
 
 ## Installation
 
-The recommendend installation method for **signac** is through **conda** or **pip**.
-The software is tested for python versions 2.7.x and 3.x and is built for all major platforms.
+The recommended installation method for **signac** is through **conda** or **pip**.
+The software is tested for Python versions 2.7 and 3.4+ and is built for all major platforms.
 
 To install **signac** *via* the [conda-forge](https://conda-forge.github.io/) channel, execute:
 
-    conda install -c conda-forge signac
+```bash
+conda install -c conda-forge signac
+```
 
 To install **signac** *via* **pip**, execute:
 
-    pip install signac
+```bash
+pip install signac
+```
 
 **Detailed information about alternative installation methods can be found in the [documentation](https://docs.signac.io/en/latest/installation.html).**
 
 ## Quickstart
 
 The framework facilitates a project-based workflow.
-Setup a new project:
+Set up a new project:
 
-    $ mkdir my_project
-    $ cd my_project
-    $ signac init MyProject
+```bash
+$ mkdir my_project
+$ cd my_project
+$ signac init MyProject
+```
 
 and access the project handle:
 
-    >>> project = signac.get_project()
-
-## Documentation
-
-The documentation is hosted at [https://docs.signac.io](https://docs.signac.io).
+```python
+>>> project = signac.get_project()
+```
 
 ## Testing
 
 You can test this package by executing:
 
-    $ python -m unittest discover tests/
+```bash
+$ python -m unittest discover tests/
+```
 
 ## Acknowledgment
 

changelog.txt

@@ -4,6 +4,62 @@ Changelog
 
 The **signac** package follows `semantic versioning <https://semver.org/>`_.
 
+Version 1.0
+===========
+
+Highlights
+----------
+
+ - Native integration of HDF5 files with the ``H5Store`` and ``H5StoreManager``, which are exposed as the ``job.data``, ``job.stores``, ``project.data``, and ``project.stores`` properties respectively.
+ - The newly added ``signac.get_job()`` function makes it easier to obtain instances of ``Job`` by calling the function from within a job's workspace directory or by directly providing the path to the job's workspace directory. This is especially useful for interactive work or when accessing jobs which are outside of the current project.
+ - Simplified export of project and job data to pandas dataframes *via* the ``to_dataframe()`` function.
+ - Projects and job search results are displayed nicely in Jupyter Notebooks.
+ - Support for compressed Collection files.
+
+
+[1.0.0] -- 2019-02-28
+---------------------
+
+Added
++++++
+
+ - Official support for Python 3.7.
+ - The ``H5Store`` and ``H5StoreManager`` classes, which are useful for storing (numerical) array-like data with an HDF5-backend. These classes are exposed within the root namespace.
+ - The ``job.data`` and ``project.data`` properties which present an instance of ``H5Store`` to access numerical data within the job workspace and project root directory.
+ - The ``job.stores`` and ``project.stores`` properties, which present an instance of ``H5StoreManager`` to manage multiple instances of ``H5Store`` to store numerical array-like data within the project workspace and project root directory.
+ - The ``signac.get_job()`` and the ``signac.Project.get_job()`` functions that allow users to get a job handle by switching into or providing the job's workspace directory.
+ - The ``job`` variable is automatically set when opening a ``signac shell`` from within a job's workspace directory.
+ - Add the ``signac shell -c`` option which allows the direct specification of Python commands to be executed within the shell.
+ - Automatic cast of ``numpy`` arrays to lists when storing them within a ``JSONDict``, e.g., a ``job.statepoint`` or ``job.document``.
+ - Enable ``Collection`` class to manage collections stored in compressed files (gzip, zip, etc.).
+ - Enable deleting of ``JSONDict`` keys through the attribute interface, e.g., ``del job.doc.foo``.
+ - Pretty HTML representation of instances of ``Project`` and ``JobsCursor`` targeted at Jupyter Notebooks (requires pandas, automatically enabled when installed).
+ - The ``to_dataframe()`` function to export the job state point and document data of a ``Project`` or a ``JobsCursor``, e.g., the result of ``Project.find_jobs()``, as a ``pandas.Dataframe`` (requires pandas).
+
+Changed
++++++++
+
+ - Dots (`.`) in keys are no longer allowed for ``JSONDict`` and ``Collection`` keys (previously deprecated).
+ - The ``JSONDict`` module is exposed in the root namespace, which is useful for storing text-serializable data  with a JSON-backend similar to the ``job.statepoint`` or ``job.document``, etc.
+ - The ``Job.init()`` method returns the job to allow one-line job creation and initialization.
+ - The ``search`` argument was added to the ``signac.get_project()`` function, which when True (the default), will cause signac to search for a project within *and above* a specified root directory, not only within the root directory. The behavior without any arguments remains unchanged.
+
+Fixed
++++++
+
+ - Fix ``Collection.update()`` behavior such that existing documents with identical primary key are updated. Previously, a ``KeyError`` would be raised.
+ - Fix issue where the ``Job.move()`` would trigger a confusing ``DestinationExists`` exception when trying to move jobs across devices / file systems.
+ - Fix issue that caused failures when the ``python-rapidjson`` package is installed. The ``python-rapidjson`` package is used as the primary JSON-backend when installed.
+ - Fix issue where schema with multiple keys would subset incorrectly if the list of jobs or statepoints was provided as an iterator rather than a sequence.
+
+Removed
++++++++
+
+ - Removes the obsolete and deprecated ``core.search_engine`` module.
+ - The previously deprecated ``Project.find_statepoints()`` and ``Project.find_job_documents()`` functions have been removed.
+ - The ``Project.find_jobs()`` no longer accepts the obsolete ``index`` argument.
+
+
 Version 0.9
 ===========
 
@@ -17,8 +73,9 @@ Highlights
  - Adds functions for the import and export of data spaces.
  - Add functions for the management of data on the project level, as opposed to the job level.
 
+
 [0.9.5] -- 2019-01-31
---------------------
+---------------------
 
 Fixed
 +++++
@@ -34,8 +91,8 @@ Fixed
 Added
 +++++
 
- - Adds the ``$ signac import`` command and the ``Project.import_from()`` method for the import of data spaces into a project workspace, such as a directory, a tarball, or a zip-file.
- - Adds the ``$ signac export`` command and the ``Project.export_to()`` method for the export of project workspaces to an external location, such as a directory, a tarball, or a zip-file.
+ - Adds the ``$ signac import`` command and the ``Project.import_from()`` method for the import of data spaces into a project workspace, such as a directory, a tarball, or a zip file.
+ - Adds the ``$ signac export`` command and the ``Project.export_to()`` method for the export of project workspaces to an external location, such as a directory, a tarball, or a zip file.
  - Adds functionality for the rapid initialization of temporary projects with the ``signac.TemporaryProject`` context manager.
  - Adds the ``signac.Project.temporary_project()`` context manager which creates a temporary project within the root project workspace.
  - Add ``signac`` to the default namespace when invoking ``signac shell``.
@@ -55,7 +112,6 @@ Fixed
  - Fix issue where changes to lists as part of synchronized dictionary, for example a state point or document would not be saved.
  - Fix non-deterministic issue occuring on network file systems when trying to open jobs where the user has no write access to the job workspace directory.
 
-
 [0.9.3] -- 2018-06-14
 ---------------------
 
@@ -92,7 +148,6 @@ Fixed
   - Fix critical issue in the ``JSONDict`` implementation that would previously overwrite the underlying file when attempting to store values that are not JSON serializable.
   - Fix issue where the ``Project.export()`` function would ignore the update argument when the index to export to would be a MongoDB collection.
 
-
 [0.9.1] -- 2017-11-07
 ---------------------
 
@@ -450,7 +505,6 @@ Version 0.4
 [0.4.0] -- 2016-08-05
 ---------------------
 
-
 Added
 +++++
 
@@ -475,7 +529,6 @@ Removed
  - Remove previously deprecated *crawl* keyword argument in index export functions.
  - Remove previously deprecated function common.config.write_config().
 
-
 Version 0.3
 ===========
 

contributors.txt

@@ -6,3 +6,4 @@ Jens Glaser
 Bradley Dice
 Tim Moore
 Pengji Zhou
+Eric Harper

doc/api.rst

@@ -38,14 +38,13 @@ The Project
     Project.create_access_module
     Project.create_linked_view
     Project.detect_schema
+    Project.data
     Project.doc
     Project.document
     Project.dump_statepoints
     Project.export_to
-    Project.find_job_documents
     Project.find_job_ids
     Project.find_jobs
-    Project.find_statepoints
     Project.fn
     Project.get_id
     Project.get_statepoint
@@ -88,6 +87,7 @@ The Job class
 
     Job.clear
     Job.close
+    Job.data
     Job.doc
     Job.document
     Job.fn
@@ -101,6 +101,7 @@ The Job class
     Job.reset_statepoint
     Job.sp
     Job.statepoint
+    Job.stores
     Job.sync
     Job.update_statepoint
     Job.workspace
@@ -112,21 +113,50 @@ The Job class
     :show-inheritance:
 
 
+.. currentmodule:: signac
+
 The Collection
 ==============
 
-.. currentmodule:: signac.contrib.collection
-
 .. autoclass:: Collection
    :members:
 
+
+The JSONDict
+============
+
+This class implements the interface for the job's :attr:`~signac.contrib.job.Job.statepoint` and :attr:`~signac.contrib.job.Job.document` attributes, but can also be used stand-alone:
+
+.. autoclass:: JSONDict
+   :members:
+
+
+The H5Store
+===========
+
+This class implements the interface to the job's :attr:`~signac.contrib.job.Job.data` attribute, but can also be used stand-alone:
+
+.. autoclass:: H5Store
+    :members:
+
+
+The H5StoreManager
+==================
+
+This class implements the interface to the job's :attr:`~signac.contrib.job.Job.stores` aatribute, but can also be used stand-alone:
+
+.. autoclass:: H5StoreManager
+    :members:
+    :show-inheritance:
+
+
 Top-level functions
 ===================
 
 .. automodule:: signac
     :members:
     :show-inheritance:
-    :exclude-members: Project,Collection,RegexFileCrawler,MasterCrawler,SignacProjectCrawler
+    :exclude-members: Project,Collection,RegexFileCrawler,MasterCrawler,SignacProjectCrawler,JSONDict,H5Store,H5StoreManager
 
 
 Submodules

doc/conf.py

@@ -81,7 +81,7 @@ def __getattr__(cls, name):
 # The short X.Y version.
 version = '0.9'
 # The full version, including alpha/beta/rc tags.
-release = '0.9.5'
+release = '1.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -309,5 +309,7 @@ def __getattr__(cls, name):
 
 intersphinx_mapping = {
     'python': ('https://docs.python.org/3.6', None),
-    'pymongo': ('https://api.mongodb.com/python/current/', None)
+    'pymongo': ('https://api.mongodb.com/python/current/', None),
+    'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None),
+    'h5py': ('http://docs.h5py.org/en/stable/', None),
 }

setup.py

@@ -13,7 +13,7 @@
 
 setup(
     name='signac',
-    version='0.9.5',
+    version='1.0.0',
     packages=find_packages(),
     zip_safe=True,
 
@@ -35,6 +35,7 @@
         "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: Implementation :: PyPy",
     ],