1. python 3 compliance preparation. 2. QC properties display in viewe…

…r. 3. Made configuration loading implicit rather than explicit.

MANIFEST.in

@@ -1,5 +1,6 @@
 recursive-include pyesdoc *.py
 include LICENSE
 include README.rst
+include pyesdoc.conf
 include requirements.txt
 include runtests.sh

README.rst

@@ -28,16 +28,7 @@ Who uses pyesdoc ?
 Developers building value added software ontop of the ES-DOC eco-system.
 
 
-What are the contents of pyesdoc ?
---------------------------------------
-
-    \docs           		Library documentation.
-    \jobs           		Jobs to be run from command line.
-    \pyesdoc           		Source code.
-    \tests  	            Unit tests.
-
-
 Further Information ?
 --------------------------------------
 
-Please refer to the [splash page](http:es-doc.org) for further information.
+See http:es-doc.org.

ipython-notebooks/creating-cmip6-experiment-definitions-with-pyesdoc.ipynb

@@ -51,7 +51,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 3,
    "metadata": {
     "collapsed": true
    },
@@ -76,7 +76,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 4,
    "metadata": {
     "collapsed": false
    },
@@ -90,7 +90,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 5,
    "metadata": {
     "collapsed": false
    },
@@ -105,19 +105,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 6,
    "metadata": {
     "collapsed": false
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "1e19715f-d743-4f4e-b8e9-2b1765f638e8\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Create experimental requirement.\n",
     "tc = pyesdoc.create(cim.v2.TemporalConstraint, project=_PROJECT, author=author, source=_SOURCE)\n",
@@ -133,7 +125,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 8,
    "metadata": {
     "collapsed": false
    },
@@ -158,7 +150,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 9,
    "metadata": {
     "collapsed": true
    },
@@ -174,7 +166,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {
     "collapsed": true
    },
@@ -193,11 +185,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "metadata": {
     "collapsed": false
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "doc.requirements[0].canonical_name --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.requirements[0].description --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.requirements[0].duration.date_type --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.requirements[0].duration.units --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.requirements[0].meta.metadata_author.email --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.requirements[0].meta.metadata_author.name --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.requirements[0].name --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.canonical_name --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.description --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.long_name --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.meta.metadata_author.email --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.meta.metadata_author.name --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.name --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n",
+      "doc.rationale --> is of invalid type (actual = <type 'str'>, expected=<type 'unicode'>)\n"
+     ]
+    }
+   ],
    "source": [
     "# Validate document & print errors\n",
     "errors = pyesdoc.validate(experiment)\n",
@@ -214,7 +227,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "metadata": {
     "collapsed": false
    },
@@ -226,7 +239,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 14,
    "metadata": {
     "collapsed": false
    },
@@ -238,7 +251,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 15,
    "metadata": {
     "collapsed": false
    },
@@ -250,7 +263,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "metadata": {
     "collapsed": false
    },
@@ -262,7 +275,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 17,
    "metadata": {
     "collapsed": false,
     "scrolled": false
@@ -282,7 +295,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 18,
    "metadata": {
     "collapsed": false
    },
@@ -294,7 +307,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 19,
    "metadata": {
     "collapsed": false
    },
@@ -307,7 +320,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 20,
    "metadata": {
     "collapsed": false
    },
@@ -327,11 +340,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 21,
    "metadata": {
     "collapsed": false
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "ValueError",
+     "evalue": "Cannot publish invalid documents.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-21-d62f75e5ef07>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Publish to ES-DOC web-service\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mpyesdoc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpublish\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexperiment\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m/Users/macg/dev/esdoc/repos/esdoc-py-client/pyesdoc/publishing.pyc\u001b[0m in \u001b[0;36mpublish\u001b[0;34m(doc)\u001b[0m\n\u001b[1;32m    198\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Cannot publish null documents.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_valid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 200\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Cannot publish invalid documents.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    201\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;31m# Increment version.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mValueError\u001b[0m: Cannot publish invalid documents."
+     ]
+    }
+   ],
    "source": [
     "# Publish to ES-DOC web-service\n",
     "pyesdoc.publish(experiment)"

jobs/run_archive_echo.py

@@ -26,6 +26,8 @@ def _main():
     """Main entry point.
 
     """
+    pyesdoc.archive.init()
+
     document = pyesdoc.archive.load(options.uid, options.version)
     if not document:
         pyesdoc.rt.log("document not found")

jobs/run_archive_populate.py

@@ -26,6 +26,7 @@ def _main():
     """Main entry point.
 
     """
+    pyesdoc.archive.init()
     pyesdoc.archive.populate(options.throttle, options.project)
 
 

pyesdoc/__init__.py

@@ -11,6 +11,14 @@
 .. moduleauthor:: Earth System Documentation (ES-DOC) <dev@es-doc.org>
 
 """
+# Standard package info.
+__title__ = 'pyesdoc'
+__version__ = '0.9.3.6'
+__author__ = 'ES-DOC'
+__license__ = 'GPL'
+__copyright__ = 'Copyright 2015 ES-DOC'
+
+
 from pyesdoc.constants import *
 from pyesdoc.exceptions import *
 from pyesdoc.factory import create
@@ -47,9 +55,3 @@
 from pyesdoc import validation
 
 
-# Standard package info.
-__title__ = 'pyesdoc'
-__version__ = '0.9.3.2'
-__author__ = 'ES-DOC'
-__license__ = 'GPL'
-__copyright__ = 'Copyright 2015 ES-DOC'

pyesdoc/archive/config.py

@@ -10,30 +10,48 @@
 
 
 """
-import os
-
 from pyesdoc.utils import config
 
 
 
-# Archive location comes from config.
-DIR_ARCHIVE = config.archive.location
-
-# Set of supported projects.
-PROJECTS = config.archive.projects
-
-# Set of supported sources.
-SOURCES = config.archive.sources
-
-
 def get_project_sources():
     """Returns iterable of unique project/source pairs.
 
     """
+    config.init()
+
     result = []
-    for project in PROJECTS:
+    for project in get_projects():
         for source in (f.source for f in project.feeds):
             if (project.name, source) not in result:
                 result.append((project.name, source))
 
     return tuple(result)
+
+
+def get_directory():
+    """Returns archive directory as defined withn pyesdoc.conf.
+
+    """
+    config.init()
+
+    return config.data.archive.location
+
+
+def get_projects():
+    """Returns set of archive projects.
+
+    """
+    config.init()
+
+    return config.data.archive.projects
+
+
+def get_sources():
+    """Returns set of archive sources.
+
+    """
+    config.init()
+
+    return config.data.archive.sources
+

pyesdoc/archive/folder_info.py

@@ -51,7 +51,7 @@ def __repr__(self):
         """Instance representation.
 
         """
-        return "{0}".format(self.path)
+        return "SSSS{0}".format(self.path)
 
     @property
     def exists(self):

pyesdoc/archive/manager.py

@@ -1,19 +1,20 @@
 # -*- coding: utf-8 -*-
 
 """
-.. module:: model.py
+.. module:: archive/manager.py
    :copyright: @2013 Earth System Documentation (http://es-doc.org)
    :license: GPL/CeCIL
    :platform: Unix, Windows
-   :synopsis: A model of a document archive.
+   :synopsis: Manages access to the document archive.
 
 .. moduleauthor:: Mark Conway-Greenslade <momipsl@ipsl.jussieu.fr>
 
 
 """
 import os
 
-from pyesdoc.archive import config, constants
+from pyesdoc.archive import config
+from pyesdoc.archive import constants
 from pyesdoc.archive.file_info import ArchiveFileInfo
 from pyesdoc.archive.folder_info import ArchiveFolderInfo
 
@@ -22,13 +23,17 @@
 # Set of managed folders.
 _FOLDERS = set()
 
-# Initialized managed folders from configuration file.
-for project, source in config.get_project_sources():
-    path = os.path.join(config.DIR_ARCHIVE, project)
-    path = os.path.join(path, source)
-    if not os.path.exists(path):
-        os.makedirs(path)
-    _FOLDERS.add(ArchiveFolderInfo(project, source, path))
+
+def init():
+    """Initializes archive manager.
+
+    """
+    for project, source in config.get_project_sources():
+        path = os.path.join(config.get_directory(), project)
+        path = os.path.join(path, source)
+        if not os.path.exists(path):
+            os.makedirs(path)
+        _FOLDERS.add(ArchiveFolderInfo(project, source, path))
 
 
 def delete_error_files(project=None, source=None):