diff --git a/ckan/ckan_nose_plugin.py b/ckan/ckan_nose_plugin.py
index 58b9d0061a0..7018e31c23c 100644
--- a/ckan/ckan_nose_plugin.py
+++ b/ckan/ckan_nose_plugin.py
@@ -1,10 +1,13 @@
 from nose.plugins import Plugin
 from inspect import isclass
+import hashlib
 import os
 import sys
+import re
 import pkg_resources
 from paste.deploy import loadapp
 from pylons import config
+import unittest
 import time
 
 class CkanNose(Plugin):
@@ -54,12 +57,35 @@ def options(self, parser, env):
             action='store_true',
             dest='docstrings',
             help='set this to display test docstrings instead of module names')
+        parser.add_option(
+            '--segments',
+            dest='segments',
+            help='A string containing a hex digits that represent which of'
+                 'the 16 test segments to run. i.e 15af will run segments 1,5,a,f')
+
+    def wantClass(self, cls):
+        name = cls.__name__
+
+        wanted = (not cls.__name__.startswith('_')
+                  and (issubclass(cls, unittest.TestCase)
+                       or re.search('(?:^|[\b_\./-])[Tt]est', name)
+                      ))
+        
+        if self.segments and str(hashlib.md5(name).hexdigest())[0] not in self.segments:
+            return False
+
+        return wanted
+
+    def finalize(self, report):
+        if self.segments:
+            print 'Segments: %s' % self.segments
 
     def configure(self, settings, config):
         CkanNose.settings = settings
         if settings.is_ckan:
             self.enabled = True
             self.is_first_test = True
+        self.segments = settings.segments
 
     def describeTest(self, test):
         if not CkanNose.settings.docstrings:
@@ -70,22 +96,22 @@ def startTest(self, test):
         """
         startTest: start timing.
         """
-        self._started = time.time()
+##        self._started = time.time()
 
     def stopTest(self, test):
         """
         stopTest: stop timing, canonicalize the test name, and save
         the run time.
         """
-        runtime = time.time() - self._started
-
-        # CTB: HACK!
-        f = open('times.txt', 'a')
-
-        testname = str(test)
-        #if ' ' in testname:
-        #    testname = testname.split(' ')[1]
-
-        f.write('%s,%s\n' % (testname, str(runtime)))
-
-        f.close()
+##        runtime = time.time() - self._started
+##
+##        # CTB: HACK!
+##        f = open('times.txt', 'a')
+##
+##        testname = str(test)
+##        #if ' ' in testname:
+##        #    testname = testname.split(' ')[1]
+##
+##        f.write('%s,%s\n' % (testname, str(runtime)))
+##
+##        f.close()
diff --git a/ckan/config/environment.py b/ckan/config/environment.py
index b4c79d9b8ca..33e9360ff9d 100644
--- a/ckan/config/environment.py
+++ b/ckan/config/environment.py
@@ -75,6 +75,10 @@ def find_controller(self, controller):
         plugin.update_config(config)
 
     # This is set up before globals are initialized
+    site_id = os.environ.get('CKAN_SITE_ID')
+    if site_id:
+        config['ckan.site_id'] = site_id
+
     site_url = config.get('ckan.site_url', '')
     ckan_host = config['ckan.host'] = urlparse(site_url).netloc
     if config.get('ckan.site_id') is None:
@@ -125,7 +129,12 @@ def template_loaded(template):
     warnings.filterwarnings('ignore', "^Did not recognize type 'BIGINT' of column 'size'", sqlalchemy.exc.SAWarning)
     warnings.filterwarnings('ignore', "^Did not recognize type 'tsvector' of column 'search_vector'", sqlalchemy.exc.SAWarning)
 
-    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
+    ckan_db = os.environ.get('CKAN_DB') 
+
+    if ckan_db:
+        engine = sqlalchemy.create_engine(ckan_db)
+    else:
+        engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
 
     if not model.meta.engine:
         model.init_model(engine)
diff --git a/ckan/config/routing.py b/ckan/config/routing.py
index 149c01bd2a7..9d89d82d3de 100644
--- a/ckan/config/routing.py
+++ b/ckan/config/routing.py
@@ -27,8 +27,8 @@ def make_map():
     PUT_POST = dict(method=['PUT','POST'])
     OPTIONS = dict(method=['OPTIONS'])
 
-    from lib.plugins import register_package_plugins
-    from lib.plugins import register_group_plugins
+    from ckan.lib.plugins import register_package_plugins
+    from ckan.lib.plugins import register_group_plugins
 
     map = Mapper(directory=config['pylons.paths']['controllers'],
                  always_scan=config['debug'])
diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index 7c8332f47dd..d85cd0f0ee1 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -14,7 +14,7 @@
 import ckan.forms
 import ckan.logic.action.get
 
-from lib.plugins import lookup_group_plugin
+from ckan.lib.plugins import lookup_group_plugin
 
 log = logging.getLogger(__name__)
 
@@ -233,10 +233,6 @@ def edit(self, id, data=None, errors=None, error_summary=None):
             old_data = get_action('group_show')(context, data_dict)
             c.grouptitle = old_data.get('title')
             c.groupname = old_data.get('name')
-            schema = self._db_to_form_schema()
-            if schema and not data:
-                old_data, errors = validate(old_data, schema, context=context)
-
             data = data or old_data
         except NotFound:
             abort(404, _('Group not found'))
diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py
index 90260ab48e3..5d8c9cefa4d 100644
--- a/ckan/controllers/package.py
+++ b/ckan/controllers/package.py
@@ -23,7 +23,7 @@
 import ckan.logic.action.get
 from home import CACHE_PARAMETER
 
-from lib.plugins import lookup_package_plugin
+from ckan.lib.plugins import lookup_package_plugin
 
 log = logging.getLogger(__name__)
 
@@ -428,9 +428,6 @@ def edit(self, id, data=None, errors=None, error_summary=None):
             return self._save_edit(id, context)
         try:
             old_data = get_action('package_show')(context, {'id':id})
-            schema = self._db_to_form_schema(package_type=package_type)
-            if schema and not data:
-                old_data, errors = validate(old_data, schema, context=context)
             data = data or old_data
             # Merge all elements for the complete package dictionary
             c.pkg_dict = dict(old_data.items() + data.items())
diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py
index 18c9594cf94..548f7674f9a 100644
--- a/ckan/lib/plugins.py
+++ b/ckan/lib/plugins.py
@@ -207,15 +207,6 @@ def history_template(self):
     def package_form(self):
         return 'package/new_package_form.html'
 
-    def form_to_db_schema(self):
-        schema =  logic.schema.package_form_schema()
-        schema['groups'] = {
-                'name': [not_empty, val.group_id_or_name_exists, unicode],
-                'id':   [ignore_missing, unicode],
-            }
-        return schema
-
-
     def form_to_db_schema_options(self, options):
         ''' This allows us to select different schemas for different
         purpose eg via the web interface or via the api or creation vs
diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py
index a36547206af..fd17862a2b2 100644
--- a/ckan/logic/action/create.py
+++ b/ckan/logic/action/create.py
@@ -1,7 +1,7 @@
 import logging
 from pylons.i18n import _
 
-import lib.plugins as lib_plugins
+import ckan.lib.plugins as lib_plugins
 import ckan.logic as logic
 import ckan.rating as ratings
 import ckan.plugins as plugins
diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py
index bf28588a769..65eef45e270 100644
--- a/ckan/logic/action/get.py
+++ b/ckan/logic/action/get.py
@@ -15,13 +15,16 @@
 import ckan.logic as logic
 import ckan.logic.action
 import ckan.lib.dictization.model_dictize as model_dictize
+import ckan.lib.navl.dictization_functions
 import ckan.model.misc as misc
 import ckan.plugins as plugins
 import ckan.lib.search as search
+import ckan.lib.plugins as lib_plugins
 
 log = logging.getLogger('ckan.logic')
 
 # define some shortcuts
+validate = ckan.lib.navl.dictization_functions.validate
 table_dictize = ckan.lib.dictization.table_dictize
 render = ckan.lib.base.render
 Authorizer = ckan.authz.Authorizer
@@ -370,6 +373,7 @@ def package_relationships_list(context, data_dict):
 def package_show(context, data_dict):
 
     model = context['model']
+    context['session'] = model.Session
     name_or_id = data_dict.get("id") or data_dict['name_or_id']
 
     pkg = model.Package.get(name_or_id)
@@ -386,6 +390,11 @@ def package_show(context, data_dict):
     for item in plugins.PluginImplementations(plugins.IPackageController):
         item.read(pkg)
 
+    schema = lib_plugins.lookup_package_plugin(package_dict['type']).db_to_form_schema()
+
+    if schema:
+        package_dict, errors = validate(package_dict, schema, context=context)
+
     return package_dict
 
 def resource_show(context, data_dict):
@@ -436,6 +445,11 @@ def group_show(context, data_dict):
     for item in plugins.PluginImplementations(plugins.IGroupController):
         item.read(group)
 
+    schema = lib_plugins.lookup_group_plugin(group_dict['type']).db_to_form_schema()
+
+    if schema:
+        package_dict, errors = validate(group_dict, schema, context=context)
+
     return group_dict
 
 def group_package_show(context, data_dict):
diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py
index 2fc23f2328f..39b801261c7 100644
--- a/ckan/logic/action/update.py
+++ b/ckan/logic/action/update.py
@@ -12,7 +12,7 @@
 import ckan.lib.dictization.model_save as model_save
 import ckan.lib.navl.dictization_functions
 import ckan.lib.navl.validators as validators
-import lib.plugins as lib_plugins
+import ckan.lib.plugins as lib_plugins
 
 log = logging.getLogger(__name__)
 
@@ -138,7 +138,7 @@ def package_update(context, data_dict):
 
     model = context['model']
     user = context['user']
-    name_or_id = data_dict.get("id") or data_dict['name_or_id']
+    name_or_id = data_dict.get("id") or data_dict['name']
     model.Session.remove()
     model.Session()._context = context
 
diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py
index aa0d84ea0c4..e77b8948dcd 100644
--- a/ckan/logic/schema.py
+++ b/ckan/logic/schema.py
@@ -115,6 +115,7 @@ def default_package_schema():
         'url': [ignore_missing, unicode],#, URL(add_http=False)],
         'version': [ignore_missing, unicode, package_version_validator],
         'state': [ignore_not_package_admin, ignore_missing],
+        'type': [ignore_missing, unicode],
         '__extras': [ignore],
         '__junk': [empty],
         'resources': default_resource_schema(),
@@ -125,6 +126,7 @@ def default_package_schema():
         'groups': {
             'id': [ignore_missing, unicode],
             'name': [ignore_missing, unicode],
+            'title': [ignore_missing, unicode],
             '__extras': [ignore],
         }
     }
@@ -153,8 +155,7 @@ def package_form_schema():
     schema['log_message'] = [ignore_missing, unicode, no_http]
     schema['groups'] = {
             'id': [ignore_missing, unicode],
-            '__extras': [empty],
-            'name': [ignore, unicode],
+            '__extras': [ignore],
     }
     schema['tag_string'] = [ignore_missing, tag_string_convert]
     schema['extras_validation'] = [duplicate_extras_key, ignore]
diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py
index b532389e05a..58c69ee3aef 100644
--- a/ckan/logic/validators.py
+++ b/ckan/logic/validators.py
@@ -312,6 +312,9 @@ def ignore_not_package_admin(key, data, errors, context):
     model = context['model']
     user = context.get('user')
 
+    if 'ignore_auth' in context:
+        return
+
     if user and Authorizer.is_sysadmin(user):
         return
 
diff --git a/ckan/public/img/glyphicons-halflings-white.png b/ckan/public/img/glyphicons-halflings-white.png
new file mode 100644
index 00000000000..a20760bfde5
Binary files /dev/null and b/ckan/public/img/glyphicons-halflings-white.png differ
diff --git a/ckan/public/img/glyphicons-halflings.png b/ckan/public/img/glyphicons-halflings.png
new file mode 100644
index 00000000000..92d4445dfd0
Binary files /dev/null and b/ckan/public/img/glyphicons-halflings.png differ
diff --git a/ckan/public/scripts/vendor/recline/css/data-explorer.css b/ckan/public/scripts/vendor/recline/css/data-explorer.css
index 208ce906a56..b27e3f82081 100644
--- a/ckan/public/scripts/vendor/recline/css/data-explorer.css
+++ b/ckan/public/scripts/vendor/recline/css/data-explorer.css
@@ -23,39 +23,32 @@
   height: 30px;
 }
 
-.header .recline-query-editor form {
-  height: 30px;
-  margin-bottom: 0;
+.header .recline-query-editor .input-prepend {
+  margin-bottom: auto;
 }
 
-.header .recline-query-editor label {
-  float: none;
+.header .recline-query-editor .text-query input {
+  float: left;  
 }
 
-.header .recline-query-editor label {
-  float: none;
+.recline-query-editor .text-query .btn-group {
+  display: inline;
+  float:left;
+  margin-left:-2px;
 }
 
-.header .recline-query-editor input.text-query {
-  float: left;  
-  margin-top: 1px;
-  margin-right: 5px;
-  width: 200px;
+.recline-query-editor .text-query .btn-group .dropdown-toggle {
+  -moz-border-radius:0px 3px 3px 0px;
+  -webkit-border-radius:0px 3px 3px 0px;
+  border-radius:0px 3px 3px 0px;
 }
 
-.header .recline-query-editor .pagination input {
-  width: 30px;
-  height: 18px;
-  padding: 2px 4px;
-  margin-top: -4px;
+.recline-query-editor .text-query .btn-group ul {
+  margin-left:-110px;
 }
 
 .header .recline-query-editor .pagination a {
-  line-height: 28px;
-}
-
-.header .recline-query-editor form .btn {
-  vertical-align: top;
+  line-height: 26px;
 }
 
 .data-view-container {
@@ -79,6 +72,11 @@
   * Data Table
   *********************************************************/
 
+.data-table .btn-group .dropdown-toggle {
+  padding: 1px 3px;
+  line-height: auto;
+}
+
 .data-table-container {
   overflow: auto;
   height: 550px;
@@ -86,7 +84,6 @@
 
 .data-table {
   border: 1px solid #ccc;
-  font-size: 12px;
   width: 100%;
 }
 
@@ -100,44 +97,46 @@
   width: 20px;
 }
 
+/* direct borrowing from twitter buttons */
+.data-table th,
+.transform-column-view .expression-preview-table-wrapper th
+{
+  background-color: #e6e6e6;
+  background-repeat: no-repeat;
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), color-stop(25%, #ffffff), to(#e6e6e6));
+  background-image: -webkit-linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);
+  background-image: -moz-linear-gradient(top, #ffffff, #ffffff 25%, #e6e6e6);
+  background-image: -ms-linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);
+  background-image: -o-linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);
+  background-image: linear-gradient(#ffffff, #ffffff 25%, #e6e6e6);
+  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffff', endColorstr='#e6e6e6', GradientType=0);
+  text-shadow: 0 1px 1px rgba(255, 255, 255, 0.75);
+  color: #333;
+  border: 1px solid #ccc;
+  border-bottom-color: #bbb;
+  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+  -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+  box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+  -webkit-transition: 0.1s linear all;
+  -moz-transition: 0.1s linear all;
+  -ms-transition: 0.1s linear all;
+  -o-transition: 0.1s linear all;
+  transition: 0.1s linear all;
+}
+
+
 /**********************************************************
   * Data Table Menus
   *********************************************************/
 
-a.column-header-menu, a.root-header-menu {
+.column-header-menu, a.root-header-menu {
   float: right;
-  display: block;
-  margin: 0 4px 0 0;
-  width: 17px;
-  height: 19px;
-  background-image: url(images/menu-dropdown.png);
-  background-repeat: no-repeat;
-}
-
-a.row-header-menu:hover, a.root-header-menu:hover {
-  background-position: -17px 0px;
-  text-decoration: none;
-}
-
-a.row-header-menu {
-  float: left;
-  display: block;
-  margin: -2px 0 -4px 0;
-  width: 17px;
-  height: 18px;
-  background-image: url(images/menu-dropdown.png);
-  background-repeat: no-repeat;
 }
 
 .read-only a.row-header-menu {
   display: none;
 }
 
-a.column-header-menu:hover {
-  background-position: -17px 0px;
-  text-decoration: none;
-}
-
 .column-header-recon-stats-bar {
   margin-top: 10px;
   height: 4px;
@@ -203,45 +202,6 @@ div.data-table-cell-content-numeric > a.data-table-cell-edit {
   color: red;
 }
 
-.data-table-menu-overlay {
-  position: fixed;
-  top: 0;
-  left: 0;
-  width: 100%;
-  height: 100%;
-}
-
-ul.data-table-menu {
-  display: none;
-  outline-style: none;
-  background: white;
-  color: black;
-  font-size: 12px;
-  height: auto;
-  list-style: none;
-  overflow: hidden;
-  position: absolute;
-  text-align: left;
-  width: 120px;
-  z-index: 666;
-  border: 1px solid #CCC;
-  border-right: 1px solid #666;
-  border-bottom: 1px solid #666;
-  margin: 0; padding: 0; }
-  ul.data-table-menu * {
-    margin: 0;
-    padding: 0; }
-  ul.data-table-menu a {
-    line-height: 14px;
-    color: black;
-    display: block;
-    padding: 5px 7px;
-    text-decoration: none; }
-  ul.data-table-menu li {
-    height: 24px; }
-  ul.data-table-menu li:hover {
-    background-color: #DBE8F8 }
-
 /* TODO: not sure the rest of this is needed */
 .data-table-cell-editor, .data-table-topic-popup {
   overflow: auto;
diff --git a/ckan/public/scripts/vendor/recline/recline.js b/ckan/public/scripts/vendor/recline/recline.js
index 3102ad36c70..ee6f6fa33b6 100644
--- a/ckan/public/scripts/vendor/recline/recline.js
+++ b/ckan/public/scripts/vendor/recline/recline.js
@@ -131,14 +131,13 @@ this.recline.Model = this.recline.Model || {};
 
 // ## A Dataset model
 //
-// A model must have the following (Backbone) attributes:
+// A model has the following (non-Backbone) attributes:
 //
 // * fields: (aka columns) is a FieldList listing all the fields on this
-//   Dataset (this can be set explicitly, or, on fetch() of Dataset
-//   information from the backend, or as is perhaps most common on the first
-//   query)
-// * currentDocuments: a DocumentList containing the Documents we have currently loaded for viewing (you update currentDocuments by calling getRows)
-// * docCount: total number of documents in this dataset (obtained on a fetch for this Dataset)
+//   Dataset (this can be set explicitly, or, will be set by Dataset.fetch() or Dataset.query()
+// * currentDocuments: a DocumentList containing the Documents we have
+//   currently loaded for viewing (you update currentDocuments by calling query)
+// * docCount: total number of documents in this dataset
 my.Dataset = Backbone.Model.extend({
   __type__: 'Dataset',
   initialize: function(model, backend) {
@@ -166,7 +165,7 @@ my.Dataset = Backbone.Model.extend({
   query: function(queryObj) {
     this.trigger('query:start');
     var self = this;
-    this.queryState.set(queryObj, {silent: true});
+    this.queryState.set(queryObj);
     var dfd = $.Deferred();
     this.backend.query(this, this.queryState.toJSON()).done(function(rows) {
       var docs = _.map(rows, function(row) {
@@ -255,18 +254,6 @@ my.backends = {};
 var util = function() {
   var templates = {
     transformActions: '<li><a data-action="transform" class="menuAction" href="JavaScript:void(0);">Global transform...</a></li>'
-    , columnActions: ' \
-      <li class="write-op"><a data-action="bulkEdit" class="menuAction" href="JavaScript:void(0);">Transform...</a></li> \
-      <li class="write-op"><a data-action="deleteColumn" class="menuAction" href="JavaScript:void(0);">Delete this column</a></li> \
-      <li><a data-action="sortAsc" class="menuAction" href="JavaScript:void(0);">Sort ascending</a></li> \
-      <li><a data-action="sortDesc" class="menuAction" href="JavaScript:void(0);">Sort descending</a></li> \
-      <li><a data-action="hideColumn" class="menuAction" href="JavaScript:void(0);">Hide this column</a></li> \
-    '
-    , rowActions: '<li><a data-action="deleteRow" class="menuAction write-op" href="JavaScript:void(0);">Delete this row</a></li>'
-    , rootActions: ' \
-        {{#columns}} \
-        <li><a data-action="showColumn" data-column="{{.}}" class="menuAction" href="JavaScript:void(0);">Show column: {{.}}</a></li> \
-        {{/columns}}'
     , cellEditor: ' \
       <div class="menu-container data-table-cell-editor"> \
         <textarea class="data-table-cell-editor-editor" bind="textarea">{{value}}</textarea> \
@@ -453,7 +440,10 @@ my.FlotGraph = Backbone.View.extend({
         <label>Graph Type</label> \
         <div class="input editor-type"> \
           <select> \
-          <option value="line">Line</option> \
+          <option value="lines-and-points">Lines and Points</option> \
+          <option value="lines">Lines</option> \
+          <option value="points">Points</option> \
+          <option value="bars">Bars</option> \
           </select> \
         </div> \
         <label>Group Column (x-axis)</label> \
@@ -514,7 +504,7 @@ my.FlotGraph = Backbone.View.extend({
     this.chartConfig = _.extend({
         group: null,
         series: [],
-        graphType: 'line'
+        graphType: 'lines-and-points'
       },
       configFromHash,
       config
@@ -536,11 +526,16 @@ my.FlotGraph = Backbone.View.extend({
 
   onEditorSubmit: function(e) {
     var select = this.el.find('.editor-group select');
-    this._getEditorData();
+    $editor = this;
+    var series = this.$series.map(function () {
+      return $(this).val();
+    });
+    this.chartConfig.series = $.makeArray(series)
+    this.chartConfig.group = this.el.find('.editor-group select').val();
+    this.chartConfig.graphType = this.el.find('.editor-type select').val();
     // update navigation
-    // TODO: make this less invasive (e.g. preserve other keys in query string)
     var qs = my.parseHashQueryString();
-    qs['graph'] = this.chartConfig;
+    qs['graph'] = JSON.stringify(this.chartConfig);
     my.setHashQueryString(qs);
     this.redraw();
   },
@@ -556,28 +551,127 @@ my.FlotGraph = Backbone.View.extend({
     if ((!areWeVisible || this.model.currentDocuments.length == 0)) {
       return
     }
+    var series = this.createSeries();
+    var options = this.getGraphOptions(this.chartConfig.graphType);
+    this.plot = $.plot(this.$graph, series, options);
+    this.setupTooltips();
     // create this.plot and cache it
-    if (!this.plot) {
-      // only lines for the present
-      options = {
-        id: 'line',
-        name: 'Line Chart'
-      };
-      this.plot = $.plot(this.$graph, this.createSeries(), options);
-    } 
-    this.plot.setData(this.createSeries());
-    this.plot.resize();
-    this.plot.setupGrid();
-    this.plot.draw();
+//    if (!this.plot) {
+//      this.plot = $.plot(this.$graph, series, options);
+//    } else {
+//      this.plot.parseOptions(options);
+//      this.plot.setData(this.createSeries());
+//      this.plot.resize();
+//      this.plot.setupGrid();
+//      this.plot.draw();
+//    }
   },
 
-  _getEditorData: function() {
-    $editor = this
-    var series = this.$series.map(function () {
-      return $(this).val();
+  // needs to be function as can depend on state
+  getGraphOptions: function(typeId) { 
+    var self = this;
+    var tickFormatter = function (val) {
+      if (self.model.currentDocuments.models[val]) {
+        var out = self.model.currentDocuments.models[val].get(self.chartConfig.group);
+        // if the x-axis value was in fact a number we want that not the 
+        if (typeof(out) == 'number') {
+          return val;
+        } else {
+          return out;
+        }
+      }
+      return val;
+    }
+    // TODO: we should really use tickFormatter and 1 interval ticks if (and
+    // only if) x-axis values are non-numeric
+    // However, that is non-trivial to work out from a dataset (datasets may
+    // have no field type info). Thus at present we only do this for bars.
+    var options = { 
+      lines: {
+         series: { 
+           lines: { show: true }
+         }
+      }
+      , points: {
+        series: {
+          points: { show: true }
+        },
+        grid: { hoverable: true, clickable: true }
+      }
+      , 'lines-and-points': {
+        series: {
+          points: { show: true },
+          lines: { show: true }
+        },
+        grid: { hoverable: true, clickable: true },
+      }
+      , bars: {
+        series: {
+          lines: {show: false},
+          bars: {
+            show: true,
+            barWidth: 1,
+            align: "center",
+            fill: true
+          }
+        },
+        grid: { hoverable: true, clickable: true },
+        xaxis: {
+          tickSize: 1,
+          tickLength: 1,
+          tickFormatter: tickFormatter
+        }
+      }
+    }
+    return options[typeId];
+  },
+
+  setupTooltips: function() {
+    var self = this;
+    function showTooltip(x, y, contents) {
+      $('<div id="flot-tooltip">' + contents + '</div>').css( {
+        position: 'absolute',
+        display: 'none',
+        top: y + 5,
+        left: x + 5,
+        border: '1px solid #fdd',
+        padding: '2px',
+        'background-color': '#fee',
+        opacity: 0.80
+      }).appendTo("body").fadeIn(200);
+    }
+
+    var previousPoint = null;
+    this.$graph.bind("plothover", function (event, pos, item) {
+      if (item) {
+        if (previousPoint != item.datapoint) {
+          previousPoint = item.datapoint;
+          
+          $("#flot-tooltip").remove();
+          var x = item.datapoint[0];
+          var y = item.datapoint[1];
+          // convert back from 'index' value on x-axis (e.g. in cases where non-number values)
+          if (self.model.currentDocuments.models[x]) {
+            x = self.model.currentDocuments.models[x].get(self.chartConfig.group);
+          } else {
+            x = x.toFixed(2);
+          }
+          y = y.toFixed(2);
+          
+          var content = _.template('<%= group %> = <%= x %>, <%= series %> = <%= y %>', {
+            group: self.chartConfig.group,
+            x: x,
+            series: item.series.label,
+            y: y
+          });
+          showTooltip(item.pageX, item.pageY, content);
+        }
+      }
+      else {
+        $("#flot-tooltip").remove();
+        previousPoint = null;            
+      }
     });
-    this.chartConfig.series = $.makeArray(series)
-    this.chartConfig.group = this.el.find('.editor-group select').val();
   },
 
   createSeries: function () {
@@ -703,19 +797,19 @@ my.DataGrid = Backbone.View.extend({
 
   onColumnHeaderClick: function(e) {
     this.state.currentColumn = $(e.target).closest('.column-header').attr('data-field');
-    util.position('data-table-menu', e);
-    util.render('columnActions', 'data-table-menu');
   },
 
   onRowHeaderClick: function(e) {
     this.state.currentRow = $(e.target).parents('tr:first').attr('data-id');
-    util.position('data-table-menu', e);
-    util.render('rowActions', 'data-table-menu');
   },
   
   onRootHeaderClick: function(e) {
-    util.position('data-table-menu', e);
-    util.render('rootActions', 'data-table-menu', {'columns': this.hiddenFields});
+    var tmpl = ' \
+        {{#columns}} \
+        <li><a data-action="showColumn" data-column="{{.}}" href="JavaScript:void(0);">Show column: {{.}}</a></li> \
+        {{/columns}}';
+    var tmp = $.mustache(tmpl, {'columns': this.hiddenFields});
+    this.el.find('.root-header-menu .dropdown-menu').html(tmp);
   },
 
   onMenuClick: function(e) {
@@ -757,7 +851,6 @@ my.DataGrid = Backbone.View.extend({
           })
       }
     }
-    util.hide('data-table-menu');
     actions[$(e.target).attr('data-action')]();
   },
 
@@ -810,26 +903,32 @@ my.DataGrid = Backbone.View.extend({
   // ======================================================
   // #### Templating
   template: ' \
-    <div class="data-table-menu-overlay" style="display: none; z-index: 101; ">&nbsp;</div> \
-    <ul class="data-table-menu"></ul> \
-    <table class="data-table table-striped" cellspacing="0"> \
+    <table class="data-table table-striped table-condensed" cellspacing="0"> \
       <thead> \
         <tr> \
           {{#notEmpty}} \
             <th class="column-header"> \
-              <div class="column-header-title"> \
-                <a class="root-header-menu"></a> \
-                <span class="column-header-name"></span> \
+              <div class="btn-group root-header-menu"> \
+                <a class="btn dropdown-toggle" data-toggle="dropdown"><span class="caret"></span></a> \
+                <ul class="dropdown-menu data-table-menu"> \
+                </ul> \
               </div> \
+              <span class="column-header-name"></span> \
             </th> \
           {{/notEmpty}} \
           {{#fields}} \
             <th class="column-header {{#hidden}}hidden{{/hidden}}" data-field="{{id}}"> \
-              <div class="column-header-title"> \
-                <a class="column-header-menu"></a> \
-                <span class="column-header-name">{{label}}</span> \
-              </div> \
+              <div class="btn-group column-header-menu"> \
+                <a class="btn dropdown-toggle" data-toggle="dropdown"><i class="icon-cog"></i><span class="caret"></span></a> \
+                <ul class="dropdown-menu data-table-menu"> \
+                  <li class="write-op"><a data-action="bulkEdit" href="JavaScript:void(0);">Transform...</a></li> \
+                  <li class="write-op"><a data-action="deleteColumn" href="JavaScript:void(0);">Delete this column</a></li> \
+                  <li><a data-action="sortAsc" href="JavaScript:void(0);">Sort ascending</a></li> \
+                  <li><a data-action="sortDesc" href="JavaScript:void(0);">Sort descending</a></li> \
+                  <li><a data-action="hideColumn" href="JavaScript:void(0);">Hide this column</a></li> \
+                </ul> \
               </div> \
+              <span class="column-header-name">{{label}}</span> \
             </th> \
           {{/fields}} \
         </tr> \
@@ -911,7 +1010,14 @@ my.DataGridRow = Backbone.View.extend({
   },
 
   template: ' \
-      <td><a class="row-header-menu"></a></td> \
+      <td> \
+        <div class="btn-group row-header-menu"> \
+          <a class="btn dropdown-toggle" data-toggle="dropdown"><span class="caret"></span></a> \
+          <ul class="dropdown-menu data-table-menu"> \
+            <li class="write-op"><a data-action="deleteRow" href="JavaScript:void(0);">Delete this row</a></li> \
+          </ul> \
+        </div> \
+      </td> \
       {{#cells}} \
       <td data-field="{{field}}"> \
         <div class="data-table-cell-content"> \
@@ -989,6 +1095,212 @@ my.DataGridRow = Backbone.View.extend({
 this.recline = this.recline || {};
 this.recline.View = this.recline.View || {};
 
+// Views module following classic module pattern
+(function($, my) {
+
+// View (Dialog) for doing data transformations on whole dataset.
+my.DataTransform = Backbone.View.extend({
+  className: 'transform-view',
+  template: ' \
+    <div class="dialog-header"> \
+      Recursive transform on all rows \
+    </div> \
+    <div class="dialog-body"> \
+      <div class="grid-layout layout-full"> \
+        <p class="info">Traverse and transform objects by visiting every node on a recursive walk using <a href="https://github.com/substack/js-traverse">js-traverse</a>.</p> \
+        <table> \
+        <tbody> \
+        <tr> \
+          <td colspan="4"> \
+            <div class="grid-layout layout-tight layout-full"> \
+              <table rows="4" cols="4"> \
+              <tbody> \
+              <tr style="vertical-align: bottom;"> \
+                <td colspan="4"> \
+                  Expression \
+                </td> \
+              </tr> \
+              <tr> \
+                <td colspan="3"> \
+                  <div class="input-container"> \
+                    <textarea class="expression-preview-code"></textarea> \
+                  </div> \
+                </td> \
+                <td class="expression-preview-parsing-status" width="150" style="vertical-align: top;"> \
+                  No syntax error. \
+                </td> \
+              </tr> \
+              <tr> \
+                <td colspan="4"> \
+                  <div id="expression-preview-tabs" class="refine-tabs ui-tabs ui-widget ui-widget-content ui-corner-all"> \
+                    <span>Preview</span> \
+                    <div id="expression-preview-tabs-preview" class="ui-tabs-panel ui-widget-content ui-corner-bottom"> \
+                      <div class="expression-preview-container" style="width: 652px; "> \
+                      </div> \
+                    </div> \
+                  </div> \
+                </td> \
+              </tr> \
+              </tbody> \
+              </table> \
+            </div> \
+          </td> \
+        </tr> \
+        </tbody> \
+        </table> \
+      </div> \
+    </div> \
+    <div class="dialog-footer"> \
+      <button class="okButton button">&nbsp;&nbsp;Update All&nbsp;&nbsp;</button> \
+      <button class="cancelButton button">Cancel</button> \
+    </div> \
+  ',
+
+  initialize: function() {
+    this.el = $(this.el);
+  },
+
+  render: function() {
+    this.el.html(this.template);
+  }
+});
+
+
+// View (Dialog) for doing data transformations (on columns of data).
+my.ColumnTransform = Backbone.View.extend({
+  className: 'transform-column-view',
+  template: ' \
+    <div class="dialog-header"> \
+      Functional transform on column {{name}} \
+    </div> \
+    <div class="dialog-body"> \
+      <div class="grid-layout layout-tight layout-full"> \
+        <table> \
+        <tbody> \
+        <tr> \
+          <td colspan="4"> \
+            <div class="grid-layout layout-tight layout-full"> \
+              <table rows="4" cols="4"> \
+              <tbody> \
+              <tr style="vertical-align: bottom;"> \
+                <td colspan="4"> \
+                  Expression \
+                </td> \
+              </tr> \
+              <tr> \
+                <td colspan="3"> \
+                  <div class="input-container"> \
+                    <textarea class="expression-preview-code"></textarea> \
+                  </div> \
+                </td> \
+                <td class="expression-preview-parsing-status" width="150" style="vertical-align: top;"> \
+                  No syntax error. \
+                </td> \
+              </tr> \
+              <tr> \
+                <td colspan="4"> \
+                  <div id="expression-preview-tabs" class="refine-tabs ui-tabs ui-widget ui-widget-content ui-corner-all"> \
+                    <span>Preview</span> \
+                    <div id="expression-preview-tabs-preview" class="ui-tabs-panel ui-widget-content ui-corner-bottom"> \
+                      <div class="expression-preview-container" style="width: 652px; "> \
+                      </div> \
+                    </div> \
+                  </div> \
+                </td> \
+              </tr> \
+              </tbody> \
+              </table> \
+            </div> \
+          </td> \
+        </tr> \
+        </tbody> \
+        </table> \
+      </div> \
+    </div> \
+    <div class="dialog-footer"> \
+      <button class="okButton btn primary">&nbsp;&nbsp;Update All&nbsp;&nbsp;</button> \
+      <button class="cancelButton btn danger">Cancel</button> \
+    </div> \
+  ',
+
+  events: {
+    'click .okButton': 'onSubmit'
+    , 'keydown .expression-preview-code': 'onEditorKeydown'
+  },
+
+  initialize: function() {
+    this.el = $(this.el);
+  },
+
+  render: function() {
+    var htmls = $.mustache(this.template, 
+      {name: this.state.currentColumn}
+      )
+    this.el.html(htmls);
+    // Put in the basic (identity) transform script
+    // TODO: put this into the template?
+    var editor = this.el.find('.expression-preview-code');
+    editor.val("function(doc) {\n  doc['"+ this.state.currentColumn+"'] = doc['"+ this.state.currentColumn+"'];\n  return doc;\n}");
+    editor.focus().get(0).setSelectionRange(18, 18);
+    editor.keydown();
+  },
+
+  onSubmit: function(e) {
+    var self = this;
+    var funcText = this.el.find('.expression-preview-code').val();
+    var editFunc = costco.evalFunction(funcText);
+    if (editFunc.errorMessage) {
+      my.notify("Error with function! " + editFunc.errorMessage);
+      return;
+    }
+    util.hide('dialog');
+    my.notify("Updating all visible docs. This could take a while...", {persist: true, loader: true});
+      var docs = self.model.currentDocuments.map(function(doc) {
+       return doc.toJSON();
+      });
+    // TODO: notify about failed docs? 
+    var toUpdate = costco.mapDocs(docs, editFunc).edited;
+    var totalToUpdate = toUpdate.length;
+    function onCompletedUpdate() {
+      totalToUpdate += -1;
+      if (totalToUpdate === 0) {
+        my.notify(toUpdate.length + " documents updated successfully");
+        alert('WARNING: We have only updated the docs in this view. (Updating of all docs not yet implemented!)');
+        self.remove();
+      }
+    }
+    // TODO: Very inefficient as we search through all docs every time!
+    _.each(toUpdate, function(editedDoc) {
+      var realDoc = self.model.currentDocuments.get(editedDoc.id);
+      realDoc.set(editedDoc);
+      realDoc.save().then(onCompletedUpdate).fail(onCompletedUpdate)
+    });
+  },
+
+  onEditorKeydown: function(e) {
+    var self = this;
+    // if you don't setTimeout it won't grab the latest character if you call e.target.value
+    window.setTimeout( function() {
+      var errors = self.el.find('.expression-preview-parsing-status');
+      var editFunc = costco.evalFunction(e.target.value);
+      if (!editFunc.errorMessage) {
+        errors.text('No syntax error.');
+        var docs = self.model.currentDocuments.map(function(doc) {
+          return doc.toJSON();
+        });
+        var previewData = costco.previewTransform(docs, editFunc, self.state.currentColumn);
+        util.render('editPreview', 'expression-preview-container', {rows: previewData});
+      } else {
+        errors.text(editFunc.errorMessage);
+      }
+    }, 1, true);
+  }
+});
+
+})(jQuery, recline.View);
+this.recline = this.recline || {};
+this.recline.View = this.recline.View || {};
+
 (function($, my) {
 // ## DataExplorer
 //
@@ -1099,6 +1411,10 @@ my.DataExplorer = Backbone.View.extend({
         my.clearNotifications();
         self.el.find('.doc-count').text(self.model.docCount || 'Unknown');
         my.notify('Data loaded', {category: 'success'});
+        // update navigation
+        var qs = my.parseHashQueryString();
+        qs['reclineQuery'] = JSON.stringify(self.model.queryState.toJSON());
+        my.setHashQueryString(qs);
       });
     this.model.bind('query:fail', function(error) {
         my.clearNotifications();
@@ -1122,8 +1438,11 @@ my.DataExplorer = Backbone.View.extend({
     // note this.model and dataset returned are the same
     this.model.fetch()
       .done(function(dataset) {
-        self.el.find('.doc-count').text(self.model.docCount || 'Unknown');
-        self.model.query();
+        var queryState = my.parseHashQueryString().reclineQuery;
+        if (queryState) {
+          queryState = JSON.parse(queryState);
+        }
+        self.model.query(queryState);
       })
       .fail(function(error) {
         my.notify(error.message, {category: 'error', persist: true});
@@ -1185,21 +1504,31 @@ my.QueryEditor = Backbone.View.extend({
   className: 'recline-query-editor', 
   template: ' \
     <form action="" method="GET" class="form-inline"> \
-      <input type="text" name="q" value="{{q}}" class="text-query" /> \
+      <div class="input-prepend text-query"> \
+        <span class="add-on"><i class="icon-search"></i></span> \
+        <input type="text" name="q" value="{{q}}" class="span2" placeholder="Search data ..." class="search-query" /> \
+        <div class="btn-group menu"> \
+          <a class="btn dropdown-toggle" data-toggle="dropdown"><i class="icon-cog"></i><span class="caret"></span></a> \
+          <ul class="dropdown-menu"> \
+            <li><a data-action="size" href="">Number of items to show ({{size}})</a></li> \
+            <li><a data-action="from" href="">Show from ({{from}})</a></li> \
+          </ul> \
+        </div> \
+      </div> \
       <div class="pagination"> \
         <ul> \
-          <li class="prev action-pagination-update"><a>&laquo;</a></li> \
-          <li class="active"><a><input name="from" type="text" value="{{from}}" /> &ndash; <input name="to" type="text" value="{{to}}" /> </a></li> \
-          <li class="next action-pagination-update"><a>&raquo;</a></li> \
+          <li class="prev action-pagination-update"><a href="">&laquo;</a></li> \
+          <li class="active"><a>{{from}} &ndash; {{to}}</a></li> \
+          <li class="next action-pagination-update"><a href="">&raquo;</a></li> \
         </ul> \
       </div> \
-      <button type="submit" class="btn" style="">Update &raquo;</button> \
     </form> \
   ',
 
   events: {
-    'submit form': 'onFormSubmit',
-    'click .action-pagination-update': 'onPaginationUpdate'
+    'submit form': 'onFormSubmit'
+    , 'click .action-pagination-update': 'onPaginationUpdate'
+    , 'click .menu li a': 'onMenuItemClick'
   },
 
   initialize: function() {
@@ -1210,10 +1539,8 @@ my.QueryEditor = Backbone.View.extend({
   },
   onFormSubmit: function(e) {
     e.preventDefault();
-    var newFrom = parseInt(this.el.find('input[name="from"]').val());
-    var newSize = parseInt(this.el.find('input[name="to"]').val()) - newFrom;
-    var query = this.el.find('.text-query').val();
-    this.model.set({size: newSize, from: newFrom, q: query});
+    var query = this.el.find('.text-query input').val();
+    this.model.set({q: query});
   },
   onPaginationUpdate: function(e) {
     e.preventDefault();
@@ -1225,6 +1552,20 @@ my.QueryEditor = Backbone.View.extend({
     }
     this.model.set({from: newFrom});
   },
+  onMenuItemClick: function(e) {
+    e.preventDefault();
+    var attrName = $(e.target).attr('data-action');
+    var msg = _.template('New value (<%= value %>)',
+        {value: this.model.get(attrName)}
+        );
+    var newValue = prompt(msg);
+    if (newValue) {
+      newValue = parseInt(newValue);
+      var update = {};
+      update[attrName] = newValue;
+      this.model.set(update);
+    }
+  },
   render: function() {
     var tmplData = this.model.toJSON();
     tmplData.to = this.model.get('from') + this.model.get('size');
@@ -1254,6 +1595,9 @@ my.parseHashUrl = function(hashUrl) {
 
 // Parse a URL query string (?xyz=abc...) into a dictionary.
 my.parseQueryString = function(q) {
+  if (!q) {
+    return {};
+  }
   var urlParams = {},
     e, d = function (s) {
       return unescape(s.replace(/\+/g, " "));
@@ -1281,7 +1625,7 @@ my.composeQueryString = function(queryParams) {
   var queryString = '?';
   var items = [];
   $.each(queryParams, function(key, value) {
-    items.push(key + '=' + JSON.stringify(value));
+    items.push(key + '=' + value);
   });
   queryString += items.join('&');
   return queryString;
@@ -1333,212 +1677,6 @@ my.clearNotifications = function() {
 
 })(jQuery, recline.View);
 
-this.recline = this.recline || {};
-this.recline.View = this.recline.View || {};
-
-// Views module following classic module pattern
-(function($, my) {
-
-// View (Dialog) for doing data transformations on whole dataset.
-my.DataTransform = Backbone.View.extend({
-  className: 'transform-view',
-  template: ' \
-    <div class="dialog-header"> \
-      Recursive transform on all rows \
-    </div> \
-    <div class="dialog-body"> \
-      <div class="grid-layout layout-full"> \
-        <p class="info">Traverse and transform objects by visiting every node on a recursive walk using <a href="https://github.com/substack/js-traverse">js-traverse</a>.</p> \
-        <table> \
-        <tbody> \
-        <tr> \
-          <td colspan="4"> \
-            <div class="grid-layout layout-tight layout-full"> \
-              <table rows="4" cols="4"> \
-              <tbody> \
-              <tr style="vertical-align: bottom;"> \
-                <td colspan="4"> \
-                  Expression \
-                </td> \
-              </tr> \
-              <tr> \
-                <td colspan="3"> \
-                  <div class="input-container"> \
-                    <textarea class="expression-preview-code"></textarea> \
-                  </div> \
-                </td> \
-                <td class="expression-preview-parsing-status" width="150" style="vertical-align: top;"> \
-                  No syntax error. \
-                </td> \
-              </tr> \
-              <tr> \
-                <td colspan="4"> \
-                  <div id="expression-preview-tabs" class="refine-tabs ui-tabs ui-widget ui-widget-content ui-corner-all"> \
-                    <span>Preview</span> \
-                    <div id="expression-preview-tabs-preview" class="ui-tabs-panel ui-widget-content ui-corner-bottom"> \
-                      <div class="expression-preview-container" style="width: 652px; "> \
-                      </div> \
-                    </div> \
-                  </div> \
-                </td> \
-              </tr> \
-              </tbody> \
-              </table> \
-            </div> \
-          </td> \
-        </tr> \
-        </tbody> \
-        </table> \
-      </div> \
-    </div> \
-    <div class="dialog-footer"> \
-      <button class="okButton button">&nbsp;&nbsp;Update All&nbsp;&nbsp;</button> \
-      <button class="cancelButton button">Cancel</button> \
-    </div> \
-  ',
-
-  initialize: function() {
-    this.el = $(this.el);
-  },
-
-  render: function() {
-    this.el.html(this.template);
-  }
-});
-
-
-// View (Dialog) for doing data transformations (on columns of data).
-my.ColumnTransform = Backbone.View.extend({
-  className: 'transform-column-view',
-  template: ' \
-    <div class="dialog-header"> \
-      Functional transform on column {{name}} \
-    </div> \
-    <div class="dialog-body"> \
-      <div class="grid-layout layout-tight layout-full"> \
-        <table> \
-        <tbody> \
-        <tr> \
-          <td colspan="4"> \
-            <div class="grid-layout layout-tight layout-full"> \
-              <table rows="4" cols="4"> \
-              <tbody> \
-              <tr style="vertical-align: bottom;"> \
-                <td colspan="4"> \
-                  Expression \
-                </td> \
-              </tr> \
-              <tr> \
-                <td colspan="3"> \
-                  <div class="input-container"> \
-                    <textarea class="expression-preview-code"></textarea> \
-                  </div> \
-                </td> \
-                <td class="expression-preview-parsing-status" width="150" style="vertical-align: top;"> \
-                  No syntax error. \
-                </td> \
-              </tr> \
-              <tr> \
-                <td colspan="4"> \
-                  <div id="expression-preview-tabs" class="refine-tabs ui-tabs ui-widget ui-widget-content ui-corner-all"> \
-                    <span>Preview</span> \
-                    <div id="expression-preview-tabs-preview" class="ui-tabs-panel ui-widget-content ui-corner-bottom"> \
-                      <div class="expression-preview-container" style="width: 652px; "> \
-                      </div> \
-                    </div> \
-                  </div> \
-                </td> \
-              </tr> \
-              </tbody> \
-              </table> \
-            </div> \
-          </td> \
-        </tr> \
-        </tbody> \
-        </table> \
-      </div> \
-    </div> \
-    <div class="dialog-footer"> \
-      <button class="okButton btn primary">&nbsp;&nbsp;Update All&nbsp;&nbsp;</button> \
-      <button class="cancelButton btn danger">Cancel</button> \
-    </div> \
-  ',
-
-  events: {
-    'click .okButton': 'onSubmit'
-    , 'keydown .expression-preview-code': 'onEditorKeydown'
-  },
-
-  initialize: function() {
-    this.el = $(this.el);
-  },
-
-  render: function() {
-    var htmls = $.mustache(this.template, 
-      {name: this.state.currentColumn}
-      )
-    this.el.html(htmls);
-    // Put in the basic (identity) transform script
-    // TODO: put this into the template?
-    var editor = this.el.find('.expression-preview-code');
-    editor.val("function(doc) {\n  doc['"+ this.state.currentColumn+"'] = doc['"+ this.state.currentColumn+"'];\n  return doc;\n}");
-    editor.focus().get(0).setSelectionRange(18, 18);
-    editor.keydown();
-  },
-
-  onSubmit: function(e) {
-    var self = this;
-    var funcText = this.el.find('.expression-preview-code').val();
-    var editFunc = costco.evalFunction(funcText);
-    if (editFunc.errorMessage) {
-      my.notify("Error with function! " + editFunc.errorMessage);
-      return;
-    }
-    util.hide('dialog');
-    my.notify("Updating all visible docs. This could take a while...", {persist: true, loader: true});
-      var docs = self.model.currentDocuments.map(function(doc) {
-       return doc.toJSON();
-      });
-    // TODO: notify about failed docs? 
-    var toUpdate = costco.mapDocs(docs, editFunc).edited;
-    var totalToUpdate = toUpdate.length;
-    function onCompletedUpdate() {
-      totalToUpdate += -1;
-      if (totalToUpdate === 0) {
-        my.notify(toUpdate.length + " documents updated successfully");
-        alert('WARNING: We have only updated the docs in this view. (Updating of all docs not yet implemented!)');
-        self.remove();
-      }
-    }
-    // TODO: Very inefficient as we search through all docs every time!
-    _.each(toUpdate, function(editedDoc) {
-      var realDoc = self.model.currentDocuments.get(editedDoc.id);
-      realDoc.set(editedDoc);
-      realDoc.save().then(onCompletedUpdate).fail(onCompletedUpdate)
-    });
-  },
-
-  onEditorKeydown: function(e) {
-    var self = this;
-    // if you don't setTimeout it won't grab the latest character if you call e.target.value
-    window.setTimeout( function() {
-      var errors = self.el.find('.expression-preview-parsing-status');
-      var editFunc = costco.evalFunction(e.target.value);
-      if (!editFunc.errorMessage) {
-        errors.text('No syntax error.');
-        var docs = self.model.currentDocuments.map(function(doc) {
-          return doc.toJSON();
-        });
-        var previewData = costco.previewTransform(docs, editFunc, self.state.currentColumn);
-        util.render('editPreview', 'expression-preview-container', {rows: previewData});
-      } else {
-        errors.text(editFunc.errorMessage);
-      }
-    }, 1, true);
-  }
-});
-
-})(jQuery, recline.View);
 // # Recline Backends
 //
 // Backends are connectors to backend data sources and stores
@@ -1786,13 +1924,33 @@ this.recline.Backend = this.recline.Backend || {};
   // );
   // </pre>
   my.GDoc = Backbone.Model.extend({
+    getUrl: function(dataset) {
+      var url = dataset.get('url');
+      if (url.indexOf('feeds/list') != -1) {
+        return url;
+      } else {
+        // https://docs.google.com/spreadsheet/ccc?key=XXXX#gid=0
+        var regex = /.*spreadsheet\/ccc?.*key=([^#?&+]+).*/
+        var matches = url.match(regex);
+        if (matches) {
+          var key = matches[1];
+          var worksheet = 1;
+          var out = 'https://spreadsheets.google.com/feeds/list/' + key + '/' + worksheet + '/public/values?alt=json'
+          return out;
+        } else {
+          alert('Failed to extract gdocs key from ' + url);
+        }
+      }
+    },
     sync: function(method, model, options) {
       var self = this;
       if (method === "read") { 
         var dfd = $.Deferred(); 
         var dataset = model;
 
-        $.getJSON(model.get('url'), function(d) {
+        var url = this.getUrl(model);
+
+        $.getJSON(url, function(d) {
           result = self.gdocsToJavascript(d);
           model.fields.reset(_.map(result.field, function(fieldId) {
               return {id: fieldId};
@@ -1982,64 +2140,3 @@ this.recline.Backend = this.recline.Backend || {};
   recline.Model.backends['memory'] = new my.Memory();
 
 }(jQuery, this.recline.Backend));
-this.recline = this.recline || {};
-this.recline.Backend = this.recline.Backend || {};
-
-(function($, my) {
-  // ## Webstore Backend
-  //
-  // Connecting to [Webstores](http://github.com/okfn/webstore)
-  //
-  // To use this backend ensure your Dataset has a webstore_url in its attributes.
-  my.Webstore = Backbone.Model.extend({
-    sync: function(method, model, options) {
-      if (method === "read") {
-        if (model.__type__ == 'Dataset') {
-          var base = model.get('webstore_url');
-          var schemaUrl = base + '/schema.json';
-          var jqxhr = $.ajax({
-            url: schemaUrl,
-              dataType: 'jsonp',
-              jsonp: '_callback'
-          });
-          var dfd = $.Deferred();
-          my.wrapInTimeout(jqxhr).done(function(schema) {
-            var fieldData = _.map(schema.data, function(item) {
-              item.id = item.name;
-              delete item.name;
-              return item;
-            });
-            model.fields.reset(fieldData);
-            model.docCount = schema.count;
-            dfd.resolve(model, jqxhr);
-          })
-          .fail(function(arguments) {
-            dfd.reject(arguments);
-          });
-          return dfd.promise();
-        }
-      }
-    },
-    query: function(model, queryObj) {
-      var base = model.get('webstore_url');
-      var data = {
-        _limit:  queryObj.size
-        , _offset: queryObj.from
-      };
-      var jqxhr = $.ajax({
-        url: base + '.json',
-        data: data,
-        dataType: 'jsonp',
-        jsonp: '_callback',
-        cache: true
-      });
-      var dfd = $.Deferred();
-      jqxhr.done(function(results) {
-        dfd.resolve(results.data);
-      });
-      return dfd.promise();
-    }
-  });
-  recline.Model.backends['webstore'] = new my.Webstore();
-
-}(jQuery, this.recline.Backend));
diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html
index c1a498be8aa..9cdfbb94705 100644
--- a/ckan/templates/package/resource_read.html
+++ b/ckan/templates/package/resource_read.html
@@ -18,16 +18,14 @@
     <link rel="stylesheet" href="${h.url_for_static('/scripts/vendor/recline/css/data-explorer.css')}" />
     <link rel="stylesheet" href="${h.url_for_static('/scripts/vendor/recline/css/graph-flot.css')}" />
     <style type="text/css">
-      .header .recline-query-editor input.text-query {
-        margin-right: 5px;
-        margin-top: 2px;
-        height: 18px;
+      .recline-query-editor form, .recline-query-editor .text-query {
+        height: 28px;
       }
 
-      .header .recline-query-editor .pagination input {
-        margin-top: auto;
-        margin-bottom: 2px;
+      .header .recline-query-editor .text-query input {
+        height: 20px;
       }
+
       .recline-query-editor .pagination ul {
         margin: 0;
         padding: 0;
diff --git a/ckan/templates/user/read.html b/ckan/templates/user/read.html
index 43716c66a3f..f2f2f8719a8 100644
--- a/ckan/templates/user/read.html
+++ b/ckan/templates/user/read.html
@@ -16,7 +16,7 @@
   </py:def>
   
   <div py:match="content" py:strip="">
-    <div class="row summary">
+    <div class="summary">
       <div class="span-12">
         <dl class="vcard">
           <dt>Name</dt>
diff --git a/ckan/tests/functional/test_datastore.py b/ckan/tests/functional/test_datastore.py
index e7aa7b143f1..302bccdac0d 100644
--- a/ckan/tests/functional/test_datastore.py
+++ b/ckan/tests/functional/test_datastore.py
@@ -39,15 +39,15 @@ def test_read(self):
         assert_equal(res.status, 200)
         assert_equal(res.body, '""')
         headers = dict(res.headers)
-        assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-test.ckan.net/%s?'
-                % resource_id)
+        assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-%s/%s?'
+                % (config['ckan.site_id'], resource_id))
 
         offset = url_for('datastore_read', id=resource_id, url='/_search')
         res = self.app.get(offset)
         assert_equal(res.status, 200)
         headers = dict(res.headers)
-        assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-test.ckan.net/%s/_search?'
-                % resource_id)
+        assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-%s/%s/_search?'
+                % (config['ckan.site_id'], resource_id))
 
     def test_update(self):
         dataset = model.Package.by_name(CreateTestData.pkg_names[0])
@@ -79,14 +79,14 @@ def test_update(self):
         # assert res.status in [401,302], res.status
         assert res.status == 200
         headers = dict(res.headers)
-        assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-test.ckan.net/%s?'
-                % resource_id)
+        assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-%s/%s?'
+                % (config['ckan.site_id'], resource_id))
 
 
         offset = url_for('datastore_write', id=resource_id, url='/_mapping')
         res = self.app.post(offset)
         assert res.status == 200
         headers = dict(res.headers)
-        assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-test.ckan.net/%s/_mapping?'
-                % resource_id)
+        assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-%s/%s/_mapping?'
+                % (config['ckan.site_id'], resource_id))
 
diff --git a/ckan/tests/functional/test_tag_vocab.py b/ckan/tests/functional/test_tag_vocab.py
index 2fa82c9a3e5..f13fa039c74 100644
--- a/ckan/tests/functional/test_tag_vocab.py
+++ b/ckan/tests/functional/test_tag_vocab.py
@@ -118,6 +118,7 @@ def _add_vocab_tag_to_dataset(self, dataset_id, vocab_id, tag_name):
         params = json.dumps({'id': dataset_id})
         response = self.app.post('/api/action/package_show', params=params)
         dataset = json.loads(response.body)['result']
+        dataset['tags'] = []
         dataset['tags'].append({'name': tag_name, 'vocabulary_id': vocab_id})
         params = json.dumps(dataset)
         response = self.app.post('/api/action/package_update', params=params,
@@ -128,7 +129,7 @@ def _remove_vocab_tags(self, dataset_id, vocab_id, tag_name):
         params = json.dumps({'id': dataset_id})
         response = self.app.post('/api/action/package_show', params=params)
         dataset = json.loads(response.body)['result']
-        dataset['tags'] = [t for t in dataset['tags'] if not t['name'] == tag_name]
+        dataset['vocab_tag_selected'] = []
         params = json.dumps(dataset)
         response = self.app.post('/api/action/package_update', params=params,
                                  extra_environ={'Authorization': str(self.sysadmin_user.apikey)})
diff --git a/ckan/tests/lib/test_dictization_schema.py b/ckan/tests/lib/test_dictization_schema.py
index 078b6b04f03..13f14e1d5a2 100644
--- a/ckan/tests/lib/test_dictization_schema.py
+++ b/ckan/tests/lib/test_dictization_schema.py
@@ -37,6 +37,9 @@ def remove_changable_columns(self, dict):
         for key, value in dict.items():
             if key.endswith('id') and key <> 'license_id':
                 dict.pop(key)
+            if key == 'created':
+                dict.pop(key)
+
             if isinstance(value, list):
                 for new_dict in value:
                     self.remove_changable_columns(new_dict)
@@ -71,7 +74,11 @@ def test_1_package_schema(self):
         pprint(errors)
         assert converted_data == {'extras': [{'key': u'genre', 'value': u'"romantic novel"'},
                                             {'key': u'original media', 'value': u'"book"'}],
-                                 'groups': [{'name': u'david'}, {'name': u'roger'}],
+                                   'groups': [{u'name': u'david',
+                                               u'title': u"Dave's books"},
+                                              {u'name': u'roger',
+                                               u'title': u"Roger's books",
+                                               }],
                                  'license_id': u'other-open',
                                  'name': u'anna2',
                                  'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n\nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py
index d1727552b6a..8949ab9101b 100644
--- a/ckan/tests/logic/test_action.py
+++ b/ckan/tests/logic/test_action.py
@@ -2,6 +2,7 @@
 import json
 from pprint import pprint
 from nose.tools import assert_equal, assert_raises
+from pylons import config
 
 import ckan
 from ckan.lib.create_test_data import CreateTestData
@@ -1273,16 +1274,17 @@ def test_27_get_site_user_not_authorized(self):
         user = model.User.get('test.ckan.net')
         assert not user
 
+        site_id = config.get('ckan.site_id')
         user = get_action('get_site_user')({'model': model, 'ignore_auth': True}, {})
-        assert user['name'] == 'test.ckan.net'
+        assert user['name'] == site_id
 
-        user = model.User.get('test.ckan.net')
+        user = model.User.get(site_id)
         assert user
 
         user=get_action('get_site_user')({'model': model, 'ignore_auth': True}, {})
-        assert user['name'] == 'test.ckan.net'
+        assert user['name'] == site_id
 
-        user = model.Session.query(model.User).filter_by(name='test.ckan.net').one()
+        user = model.Session.query(model.User).filter_by(name=site_id).one()
         assert user
 
     def test_28_group_package_show(self):
diff --git a/ckanext/test_tag_vocab_plugin.py b/ckanext/test_tag_vocab_plugin.py
index 8d88650b7bb..043234890b2 100644
--- a/ckanext/test_tag_vocab_plugin.py
+++ b/ckanext/test_tag_vocab_plugin.py
@@ -54,7 +54,7 @@ def form_to_db_schema(self):
         return schema
 
     def db_to_form_schema(self):
-        schema = package_form_schema()
+        schema = default_package_schema()
         schema.update({
             'tags': {
                 '__extras': [keep_extras, free_tags_only]
@@ -72,11 +72,10 @@ def filter(self, stream):
             if routes.get('controller') == 'package' \
                 and routes.get('action') == 'read':
                     # add vocab tags to the bottom of the page
-                    tags = c.pkg_dict.get('tags', [])
+                    tags = c.pkg_dict.get('vocab_tags_selected', [])
                     for tag in tags:
-                        if tag.get('vocabulary_id'):
-                            stream = stream | Transformer('body')\
-                                .append(HTML('<p>%s</p>' % tag.get('name')))
+                        stream = stream | Transformer('body')\
+                            .append(HTML('<p>%s</p>' % tag))
             if routes.get('controller') == 'package' \
                 and routes.get('action') == 'edit':
                     # add vocabs tag select box to edit page
diff --git a/doc/_templates/footer.html b/doc/_templates/footer.html
index 146189a8098..addc6974357 100644
--- a/doc/_templates/footer.html
+++ b/doc/_templates/footer.html
@@ -18,12 +18,12 @@
 
 <p>
 Related Projects:
-<a href="http://thedatahub.org/">TheDataHub.org</a>
+<a href="http://thedatahub.org/">The DataHub</a>
 &mdash;
 <a href="http://datacatalogs.org">DataCatalogs.org</a>
 &mdash;
 <a href="http://openspending.org">OpenSpending.org</a>
 &mdash;
-<a href="http://datapatterns.org">DataPatterns.org</a>
+<a href="http://opendatahandbook.org">Open Data Handbook</a>
 </p>
 
diff --git a/doc/_themes/sphinx-theme-okfn b/doc/_themes/sphinx-theme-okfn
index 1e0978e28d0..5c2ef7385a9 160000
--- a/doc/_themes/sphinx-theme-okfn
+++ b/doc/_themes/sphinx-theme-okfn
@@ -1 +1 @@
-Subproject commit 1e0978e28d002a422cbbdeea8d86f6b94efda25f
+Subproject commit 5c2ef7385a9e1012d0c328722c8cb235f18391a1
diff --git a/doc/administrative-dashboard.rst b/doc/administrative-dashboard.rst
new file mode 100644
index 00000000000..6e90735e055
--- /dev/null
+++ b/doc/administrative-dashboard.rst
@@ -0,0 +1,66 @@
+=============================
+CKAN Administrative Dashboard
+=============================
+
+CKAN provides an administrative dashboard available to Sysadmin Administrators.
+The dashboard allows you to:
+
+* Create and remove sysadmins
+* Edit general system level authorization
+* Manage the 'trash' bin (i.e. datasets or revisions that have been marked as deleted)
+
+The dashboard is located, relative to your site root, at ``/ckan-admin/``.
+
+.. note:: To create your first sysadmin you cannot use Dashboard as you will
+          not yet have access! Instead create a sysadmin using the command line
+          ``paster`` by running the following command::
+          
+            paster sysadmin -h
+
+Setting System-Level Roles
+==========================
+
+Authorization interface is located at: ``/ckan-admin/authz``
+
+This page allows you to see and change the users and authorization groups who
+have 'roles' on the 'System Object'. In a standard installation, there are four
+'roles' which a user can have on the System (or on any object):
+
+* admin (administrator)
+
+  * Having an admin role on the System objects means you are a System Administrator
+    and may carry out **any** operation on any object.
+
+    .. warning:: Once a person is an system administrator, they can carry out
+                 **any** operatoin on the system including **destructive**
+                 ones. Grant System Administrator access with care!
+
+* reader (Read action allowed)
+
+  * Without read access a site user or visitor will not be able to see
+    anything except the login page, even the page which allows them to
+    create an account, so they're locked out forever unless they already
+    have a valid account.
+
+* editor (Update action allowed) 
+* anon-editor
+
+.. note:: these roles can be applied to users on your system as well as to
+          'pseudo-users' like 'visitor', which stands for anyone who accesses
+          the site whether logged in or not (see :doc:`authorization` for more
+          on permissions and roles).
+
+Make Someone a Sysadmin
+=======================
+
+Given the user the role 'admin'.
+
+The Trash
+=========
+
+When you delete datasets or revisions they go into the 'trash'. The contents of
+the trash can be viewed by System Administrators at: ``/ckan-admin/trash``.
+
+Contents of the trash can be removed permanently (and **irreversibly**) by
+going to the trash page and selecting the purge option.
+
diff --git a/doc/api-tutorial.rst b/doc/api-tutorial.rst
new file mode 100644
index 00000000000..4d8cba8d5c7
--- /dev/null
+++ b/doc/api-tutorial.rst
@@ -0,0 +1,118 @@
+=============================
+CKAN API: Quickstart Tutorial
+=============================
+
+A quickstart tutorial for the CKAN API that walks through some of the main
+features. For full details of the API see :doc:`api` and its 
+:ref:`api-reference` section.
+
+The following examples use the demonstration instance at http://test.ckan.org/.
+You may obviously replace this with your own site.
+
+Details of tools for accessing the API including client libraries can be found
+in the :ref:`main API section here <tools-for-api>`.
+
+Example queries
+===============
+
+Search
+------
+
+http://test.ckan.org/api/search/dataset?q=open+street+map
+
+::
+
+ {"count": 4, "results": ["uk-naptan-osm", "osm-uk", "osm", "naptan"]}
+
+Get dataset
+-----------
+
+http://test.ckan.org/api/rest/dataset/osm
+
+::
+
+ {"id": "a3dd8f64-9078-4f04-845c-e3f047125028", "name": "osm", "title": "Open Street Map", ...
+
+
+Create a dataset
+----------------
+
+.. note:: You'll need an :ref:`api key <api-keys>`.
+
+::
+
+ curl http://test.ckan.org/api/rest/dataset -d '{"name":"test", "title":"Test dataset"}' -H "Authorization:your-api-key"
+
+Update a dataset
+----------------
+
+(Using POST or PUT)::
+
+  curl http://test.ckan.org/api/rest/dataset/test -d '{"name":"test", "title":"Changed Test dataset"}' -H "Authorization:your-api-key"
+
+Viewing permissions
+-------------------
+
+To view authorization roles on a dataset::
+
+  curl http://test.ckan.org/api/action/roles_show -d '{"domain_object": "freshwateratlasrivers"}'
+  
+  {
+    "help": "Returns the roles that users (and authorization groups) have on a\n    particular domain_object.\n    \n    If you specify a user (or authorization group) then the resulting roles\n    will be filtered by those of that user (or authorization group).\n\n    domain_object can be a package/group/authorization_group name or id.\n    ", 
+    "result": {
+      "domain_object_id": "9da77628-2ac5-4965-af12-c7c51cc1d99a", 
+      "domain_object_type": "Package", 
+      "roles": [
+        {
+          "authorized_group_id": null, 
+          "context": "Package", 
+          "id": "481b6cd8-350b-4599-bd20-5e3c0ed0a8cb", 
+          "package_id": "9da77628-2ac5-4965-af12-c7c51cc1d99a", 
+          "role": "editor", 
+          "user_id": "4229c297-fe28-4597-a191-3ebbbee6c47a", 
+          "user_object_role_id": "481b6cd8-350b-4599-bd20-5e3c0ed0a8cb"
+        }, 
+        {
+          "authorized_group_id": null, 
+          "context": "Package", 
+          "id": "aba38fa7-2fb4-4f84-98e1-02cb76c5d95a", 
+          "package_id": "9da77628-2ac5-4965-af12-c7c51cc1d99a", 
+          "role": "admin", 
+          "user_id": "e7f30c0d-944b-4a69-84c4-61b08bbf6b98", 
+          "user_object_role_id": "aba38fa7-2fb4-4f84-98e1-02cb76c5d95a"
+        }, 
+        {
+          "authorized_group_id": null, 
+          "context": "Package", 
+          "id": "e06b1293-86ec-4417-8e28-b9499161348e", 
+          "package_id": "9da77628-2ac5-4965-af12-c7c51cc1d99a", 
+          "role": "reader", 
+          "user_id": "41cb1162-3d61-4b16-a3af-4cae27836ac5", 
+          "user_object_role_id": "e06b1293-86ec-4417-8e28-b9499161348e"
+        }
+      ]
+    }, 
+    "success": true
+  }
+
+Looking at the list of "roles" we can see who has what permissions on this
+dataset. User with id="4229..." is an "editor", id="e7f3..." is an "admin",
+"41cb..." is a "reader". By using the user_show call to reveal the names of the
+users, we see that "visitor" (i.e. anyone who is not logged in) is the "reader"
+(no write permission), "logged-in" (any logged-in user) is the "editor" and the
+admin is "OKFN" who is the user who created this dataset in the first place and
+can therefore confer permissions to other users.
+
+Adding permissions
+------------------
+
+To give user "dread" the "admin" authorization role on dataset
+"freshwateratlasrivers"::
+
+  curl http://test.ckan.org/api/action/user_role_update -d '{"user": "dread", "domain_object": "freshwateratlasrivers", "roles": ["admin"]}' -H "Authorization:{your-api-key}"
+
+Javascript examples
+===================
+
+See http://okfnlabs.org/ckanjs/ (demo search and count widgets)
+
diff --git a/doc/api-util.rst b/doc/api-util.rst
new file mode 100644
index 00000000000..450ac9da686
--- /dev/null
+++ b/doc/api-util.rst
@@ -0,0 +1,128 @@
+========
+Util API
+========
+
+The Util API provides various utility APIs -- e.g. auto-completion APIs used by
+front-end javascript.
+
+All Util APIs are read-only. The response format is JSON. Javascript calls may
+want to use the JSONP formatting.
+
+.. Note::
+
+  Some CKAN deployments have the API deployed at a different domain to the main CKAN website. To make sure that the AJAX calls in the Web UI work, you'll need to configue the ckan.api_url. e.g.::
+
+    ckan.api_url = http://api.example.com/
+
+
+dataset autocomplete
+````````````````````
+
+There an autocomplete API for package names which matches on name or title.
+
+This URL:
+
+::
+
+    /api/2/util/dataset/autocomplete?incomplete=a%20novel
+
+Returns:
+
+::
+
+    {"ResultSet": {"Result": [{"match_field": "title", "match_displayed": "A Novel By Tolstoy (annakarenina)", "name": "annakarenina", "title": "A Novel By Tolstoy"}]}}
+
+
+tag autocomplete
+````````````````
+
+There is also an autocomplete API for tags which looks like this:
+
+This URL:
+
+::
+
+    /api/2/util/tag/autocomplete?incomplete=ru
+
+Returns:
+
+::
+
+    {"ResultSet": {"Result": [{"Name": "russian"}]}}
+
+resource format autocomplete
+````````````````````````````
+
+Similarly, there is an autocomplete API for the resource format field
+which is available at:
+
+::
+
+    /api/2/util/resource/format_autocomplete?incomplete=cs
+
+This returns:
+
+::
+
+    {"ResultSet": {"Result": [{"Format": "csv"}]}}
+
+markdown
+````````
+
+Takes a raw markdown string and returns a corresponding chunk of HTML. CKAN uses the basic Markdown format with some modifications (for security) and useful additions (e.g. auto links to datasets etc. e.g. ``dataset:river-quality``).
+
+Example::
+
+    /api/util/markdown?q=<http://ibm.com/>
+
+Returns::
+
+    "<p><a href="http://ibm.com/" target="_blank" rel="nofollow">http://ibm.com/</a>\n</p>"
+
+is slug valid
+`````````````
+
+Checks a name is valid for a new dataset (package) or group, with respect to it being used already.
+
+Example::
+
+    /api/2/util/is_slug_valid?slug=river-quality&type=package
+
+Response::
+
+    {"valid": true}
+
+munge package name
+``````````````````
+
+For taking an readable identifier and munging it to ensure it is a valid dataset id. Symbols and whitespeace are converted into dashes. Example::
+
+    /api/util/dataset/munge_name?name=police%20spending%20figures%202009
+
+Returns::
+
+    "police-spending-figures-2009"
+
+munge title to package name
+```````````````````````````
+
+For taking a title of a package and munging it to a readable and valid dataset id. Symbols and whitespeace are converted into dashes, with multiple dashes collapsed. Ensures that long titles with a year at the end preserves the year should it need to be shortened. Example::
+
+    /api/util/dataset/munge_title_to_name?title=police:%20spending%20figures%202009
+
+Returns::
+
+    "police-spending-figures-2009"
+
+
+munge tag
+`````````
+
+For taking a readable word/phrase and munging it to a valid tag (name). Symbols and whitespeace are converted into dashes. Example::
+
+    /api/util/tag/munge?tag=water%20quality
+
+Returns::
+
+    "water-quality"
+
diff --git a/doc/api-v2.rst b/doc/api-v2.rst
new file mode 100644
index 00000000000..81521ba4182
--- /dev/null
+++ b/doc/api-v2.rst
@@ -0,0 +1,433 @@
+============================
+API Details - Versions 1 & 2
+============================
+
+The CKAN API version 1 & 2 is separated into three parts.
+
+* `Model API`_
+* `Search API`_
+* `Util API`_
+
+The resources, methods, and data formats of each are described below.
+
+Versions 1 & 2
+~~~~~~~~~~~~~~
+
+These are very similar, but when the API returns a reference to an object, Version 1 API will return the Name of the object (e.g. "river-pollution") and Version 2 API will return the ID of the object (e.g. "a3dd8f64-9078-4f04-845c-e3f047125028").
+
+The reason for this is that Names can change, so to reliably refer to the same dataset every time, you will want to use the ID and therefore use API v2. Alternatively, many people prefer to deal with Names, so API v1 suits them.
+
+When making requests, you can call objects by either their Name or ID, interchangeably.
+
+The only exception for this is for Tag objects. Since Tag names are immutable, they are always referred to with their Name.
+
+Locators
+~~~~~~~~
+
+The locator for a given resource can be formed by appending
+the relative path for that resource to the API locator.
+
+  ``Resource Locator = API Locator + Resource Path``
+
+The API locators for the CKAN APIs (by version) are:
+
+ * ``/api`` (version 1)
+ * ``/api/1`` (version 1)
+ * ``/api/2`` (version 2)
+
+The relative paths for each resource are listed in the sections below.
+
+Model API
+~~~~~~~~~
+
+Model resources are available at published locations. They are represented with
+a variety of data formats. Each resource location supports a number of methods.
+
+The data formats of the requests and the responses are defined below.
+
+Model Resources
+```````````````
+
+Here are the resources of the Model API.
+
+
++--------------------------------+-------------------------------------------------------------------+
+| Model Resource                 | Location                                                          |
++================================+===================================================================+
+| Dataset Register               | ``/rest/dataset``                                                 |
++--------------------------------+-------------------------------------------------------------------+
+| Dataset Entity                 | ``/rest/dataset/DATASET-REF``                                     |
++--------------------------------+-------------------------------------------------------------------+
+| Group Register                 | ``/rest/group``                                                   |
++--------------------------------+-------------------------------------------------------------------+
+| Group Entity                   | ``/rest/group/GROUP-REF``                                         |
++--------------------------------+-------------------------------------------------------------------+
+| Tag Register                   | ``/rest/tag``                                                     |
++--------------------------------+-------------------------------------------------------------------+
+| Tag Entity                     | ``/rest/tag/TAG-NAME``                                            |
++--------------------------------+-------------------------------------------------------------------+
+| Rating Register                | ``/rest/rating``                                                  |
++--------------------------------+-------------------------------------------------------------------+
+| Dataset Relationships Register | ``/rest/dataset/DATASET-REF/relationships``                       |
++--------------------------------+-------------------------------------------------------------------+
+| Dataset Relationships Register | ``/rest/dataset/DATASET-REF/RELATIONSHIP-TYPE``                   |
++--------------------------------+-------------------------------------------------------------------+
+| Dataset Relationships Register | ``/rest/dataset/DATASET-REF/relationships/DATASET-REF``           |
++--------------------------------+-------------------------------------------------------------------+
+| Dataset Relationship Entity    | ``/rest/dataset/DATASET-REF/RELATIONSHIP-TYPE/DATASET-REF``       |
++--------------------------------+-------------------------------------------------------------------+
+| Dataset\'s Revisions Entity    | ``/rest/dataset/DATASET-REF/revisions``                           |
++--------------------------------+-------------------------------------------------------------------+
+| Revision Register              | ``/rest/revision``                                                |
++--------------------------------+-------------------------------------------------------------------+
+| Revision Entity                | ``/rest/revision/REVISION-ID``                                    |
++--------------------------------+-------------------------------------------------------------------+
+| License List                   | ``/rest/licenses``                                                |
++--------------------------------+-------------------------------------------------------------------+
+
+Possible values for DATASET-REF are the dataset id, or the current dataset name.
+
+Possible values for RELATIONSHIP-TYPE are described in the Relationship-Type data format.
+
+
+Model Methods
+`````````````
+
+Here are the methods of the Model API.
+
++-------------------------------+--------+------------------+-------------------+
+| Resource                      | Method | Request          | Response          |
++===============================+========+==================+===================+ 
+| Dataset Register              | GET    |                  | Dataset-List      |
++-------------------------------+--------+------------------+-------------------+
+| Dataset Register              | POST   | Dataset          |                   |
++-------------------------------+--------+------------------+-------------------+
+| Dataset Entity                | GET    |                  | Dataset           |
++-------------------------------+--------+------------------+-------------------+
+| Dataset Entity                | PUT    | Dataset          |                   |
++-------------------------------+--------+------------------+-------------------+
+| Group Register                | GET    |                  | Group-List        |
++-------------------------------+--------+------------------+-------------------+
+| Group Register                | POST   | Group            |                   |
++-------------------------------+--------+------------------+-------------------+
+| Group Entity                  | GET    |                  | Group             |
++-------------------------------+--------+------------------+-------------------+
+| Group Entity                  | PUT    | Group            |                   |
++-------------------------------+--------+------------------+-------------------+
+| Tag Register                  | GET    |                  | Tag-List          | 
++-------------------------------+--------+------------------+-------------------+
+| Tag Entity                    | GET    |                  | Dataset-List      |
++-------------------------------+--------+------------------+-------------------+
+| Rating Register               | POST   | Rating           |                   |
++-------------------------------+--------+------------------+-------------------+
+| Rating Entity                 | GET    |                  | Rating            |
++-------------------------------+--------+------------------+-------------------+
+| Dataset Relationships Register| GET    |                  | Pkg-Relationships |
++-------------------------------+--------+------------------+-------------------+
+| Dataset Relationship Entity   | GET    |                  | Pkg-Relationship  |
++-------------------------------+--------+------------------+-------------------+
+| Dataset Relationships Register| POST   | Pkg-Relationship |                   |
++-------------------------------+--------+------------------+-------------------+
+| Dataset Relationship Entity   | PUT    | Pkg-Relationship |                   |
++-------------------------------+--------+------------------+-------------------+
+| Dataset\'s Revisions Entity   | GET    |                  | Pkg-Revisions     |
++-------------------------------+--------+------------------+-------------------+
+| Revision List                 | GET    |                  | Revision-List     |
++-------------------------------+--------+------------------+-------------------+
+| Revision Entity               | GET    |                  | Revision          |
++-------------------------------+--------+------------------+-------------------+
+| License List                  | GET    |                  | License-List      |
++-------------------------------+--------+------------------+-------------------+
+
+In general:
+
+* GET to a register resource will *list* the entities of that type.
+
+* GET of an entity resource will *show* the entity's properties.
+
+* POST of entity data to a register resource will *create* the new entity.
+
+* PUT of entity data to an existing entity resource will *update* it.
+
+It is usually clear whether you are trying to create or update, so in these cases, HTTP POST and PUT methods are accepted by CKAN interchangeably.
+
+Model Formats
+`````````````
+
+Here are the data formats for the Model API:
+
++--------------------+------------------------------------------------------------+
+| Name               | Format                                                     |
++====================+============================================================+
+| Dataset-Ref        | Dataset-Name-String (API v1) OR Dataset-Id-Uuid (API v2)   |
++--------------------+------------------------------------------------------------+
+| Dataset-List       | [ Dataset-Ref, Dataset-Ref, Dataset-Ref, ... ]             |
++--------------------+------------------------------------------------------------+
+| Dataset            | { id: Uuid, name: Name-String, title: String, version:     | 
+|                    | String, url: String, resources: [ Resource, Resource, ...],| 
+|                    | author: String, author_email: String, maintainer: String,  |
+|                    | maintainer_email: String, license_id: String,              |
+|                    | tags: Tag-List, notes: String, extras: { Name-String:      |
+|                    | String, ... } }                                            |
+|                    | See note below on additional fields upon GET of a dataset. |
++--------------------+------------------------------------------------------------+
+| Group-Ref          | Group-Name-String (API v1) OR Group-Id-Uuid (API v2)       |
++--------------------+------------------------------------------------------------+
+| Group-List         | [ Group-Ref, Group-Ref, Group-Ref, ... ]                   |
++--------------------+------------------------------------------------------------+
+| Group              | { name: Group-Name-String, title: String,                  |
+|                    | description: String, datasets: Dataset-List }              |
++--------------------+------------------------------------------------------------+
+| Tag-List           | [ Name-String, Name-String, Name-String, ... ]             |
++--------------------+------------------------------------------------------------+
+| Tag                | { name: Name-String }                                      |
++--------------------+------------------------------------------------------------+
+| Resource           | { url: String, format: String, description: String,        |
+|                    | hash: String }                                             |
++--------------------+------------------------------------------------------------+
+| Rating             | { dataset: Name-String, rating: int }                      |
++--------------------+------------------------------------------------------------+
+| Pkg-Relationships  | [ Pkg-Relationship, Pkg-Relationship, ... ]                |
++--------------------+------------------------------------------------------------+
+| Pkg-Relationship   | { subject: Dataset-Name-String,                            |
+|                    | object: Dataset-Name-String, type: Relationship-Type,      |
+|                    | comment: String }                                          |
++--------------------+------------------------------------------------------------+
+| Pkg-Revisions      | [ Pkg-Revision, Pkg-Revision, Pkg-Revision, ... ]          |
++--------------------+------------------------------------------------------------+
+| Pkg-Revision       | { id: Uuid, message: String, author: String,               |
+|                    | timestamp: Date-Time }                                     |
++--------------------+------------------------------------------------------------+
+|Relationship-Type   | One of: 'depends_on', 'dependency_of',                     |
+|                    | 'derives_from', 'has_derivation',                          |
+|                    | 'child_of', 'parent_of',                                   |
+|                    | 'links_to', 'linked_from'.                                 |
++--------------------+------------------------------------------------------------+
+| Revision-List      | [ revision_id, revision_id, revision_id, ... ]             |
++--------------------+------------------------------------------------------------+
+| Revision           | { id: Uuid, message: String, author: String,               |
+|                    | timestamp: Date-Time, datasets: Dataset-List }             |
++--------------------+------------------------------------------------------------+
+| License-List       | [ License, License, License, ... ]                         |
++--------------------+------------------------------------------------------------+
+| License            | { id: Name-String, title: String, is_okd_compliant:        |
+|                    | Boolean, is_osi_compliant: Boolean, tags: Tag-List,        |
+|                    | family: String, url: String, maintainer: String,           |
+|                    | date_created: Date-Time, status: String }                  |
++--------------------+------------------------------------------------------------+
+
+To send request data, create the JSON-format string (encode in UTF8) put it in the request body and send it using PUT or POST.
+
+Response data will be in the response body in JSON format.
+
+Notes:
+
+ * When you update an object, fields that you don't supply will remain as they were before.
+
+ * To delete an 'extra' key-value pair, supply the key with JSON value: ``null``
+
+ * When you read a dataset, some additional information is supplied that you cannot modify and POST back to the CKAN API. These 'read-only' fields are provided only on the Dataset GET. This is a convenience to clients, to save further requests. This applies to the following fields:
+    
+===================== ================================
+Key                   Description 
+===================== ================================
+id                    Unique Uuid for the Dataset
+revision_id           Latest revision ID for the core Package data (but is not affected by changes to tags, groups, extras, relationships etc)
+metadata_created      Date the Dataset (record) was created
+metadata_modified     Date the Dataset (record) was last modified
+relationships         info on Dataset Relationships
+ratings_average         
+ratings_count            
+ckan_url              full URL of the Dataset
+download_url (API v1) URL of the first Resource
+isopen                boolean indication of whether dataset is open according to Open Knowledge Definition, based on other fields
+notes_rendered        HTML rendered version of the Notes field (which may contain Markdown)
+===================== ================================
+   
+
+
+Search API
+~~~~~~~~~~
+
+Search resources are available at published locations. They are represented with
+a variety of data formats. Each resource location supports a number of methods.
+
+The data formats of the requests and the responses are defined below.
+
+Search Resources
+````````````````
+
+Here are the published resources of the Search API.
+
++---------------------------+--------------------------+
+| Search Resource           | Location                 |
++===========================+==========================+
+| Dataset Search            | ``/search/dataset``      |
++---------------------------+--------------------------+
+| Resource Search           | ``/search/resource``     |
++---------------------------+--------------------------+
+| Revision Search           | ``/search/revision``     |
++---------------------------+--------------------------+
+| Tag Counts                | ``/tag_counts``          |
++---------------------------+--------------------------+
+
+See below for more information about dataset and revision search parameters.
+
+Search Methods
+``````````````
+
+Here are the methods of the Search API.
+
++-------------------------------+--------+------------------------+--------------------------+
+| Resource                      | Method | Request                | Response                 |
++===============================+========+========================+==========================+ 
+| Dataset Search                | POST   | Dataset-Search-Params  | Dataset-Search-Response  | 
++-------------------------------+--------+------------------------+--------------------------+
+| Resource Search               | POST   | Resource-Search-Params | Resource-Search-Response | 
++-------------------------------+--------+------------------------+--------------------------+
+| Revision Search               | POST   | Revision-Search-Params | Revision-List            | 
++-------------------------------+--------+------------------------+--------------------------+
+| Tag Counts                    | GET    |                        | Tag-Count-List           | 
++-------------------------------+--------+------------------------+--------------------------+
+
+It is also possible to supply the search parameters in the URL of a GET request, 
+for example ``/api/search/dataset?q=geodata&amp;allfields=1``.
+
+Search Formats
+``````````````
+
+Here are the data formats for the Search API.
+
++-------------------------+------------------------------------------------------------+
+| Name                    | Format                                                     |
++=========================+============================================================+
+| Dataset-Search-Params   | { Param-Key: Param-Value, Param-Key: Param-Value, ... }    |
+| Resource-Search-Params  | See below for full details of search parameters across the | 
+| Revision-Search-Params  | various domain objects.                                    |
++-------------------------+------------------------------------------------------------+
+| Dataset-Search-Response | { count: Count-int, results: [Dataset, Dataset, ... ] }    |
++-------------------------+------------------------------------------------------------+
+| Resource-Search-Response| { count: Count-int, results: [Resource, Resource, ... ] }  |
++-------------------------+------------------------------------------------------------+
+| Revision-List           | [ Revision-Id, Revision-Id, Revision-Id, ... ]             |
+|                         | NB: Ordered with youngest revision first                   |
++-------------------------+------------------------------------------------------------+
+| Tag-Count-List          | [ [Name-String, Integer], [Name-String, Integer], ... ]    |
++-------------------------+------------------------------------------------------------+
+
+The ``Dataset`` and ``Revision`` data formats are as defined in `Model Formats`_.
+
+**Dataset Parameters**
+
++-----------------------+---------------+----------------------------------+----------------------------------+
+| Param-Key             | Param-Value   | Examples                         |  Notes                           |
++=======================+===============+==================================+==================================+
+| q                     | Search-String || q=geodata                       | Criteria to search the dataset   |
+|                       |               || q=government+sweden             | fields for. URL-encoded search   |
+|                       |               || q=%22drug%20abuse%22            | text. (You can also concatenate  |
+|                       |               || q=tags:"river pollution"        | words with a '+' symbol in a     |
+|                       |               |                                  | URL.) Search results must contain|
+|                       |               |                                  | all the specified words.  You    |
+|                       |               |                                  | can also search within specific  |
+|                       |               |                                  | fields.                          |
++-----------------------+---------------+----------------------------------+----------------------------------+
+| qjson                 | JSON encoded  | ['q':'geodata']                  | All search parameters can be     |
+|                       | options       |                                  | json-encoded and supplied to this|
+|                       |               |                                  | parameter as a more flexible     |
+|                       |               |                                  | alternative in GET requests.     |
++-----------------------+---------------+----------------------------------+----------------------------------+
+|title,                 | Search-String || title=uk&amp;tags=health        | Search in a particular a field.  |
+|tags, notes, groups,   |               || department=environment          |                                  |
+|author, maintainer,    |               || tags=health&tags=pollution      |                                  |
+|update_frequency, or   |               || tags=river%20pollution          |                                  |
+|any 'extra' field name |               |                                  |                                  |
+|e.g. department        |               |                                  |                                  |
++-----------------------+---------------+----------------------------------+----------------------------------+
+| order_by              | field-name    | order_by=name                    | Specify either rank or the field |
+|                       | (default=rank)|                                  | to sort the results by           |
++-----------------------+---------------+----------------------------------+----------------------------------+
+| offset, limit         | result-int    | offset=40&amp;limit=20           | Pagination options. Offset is the|
+|                       | (defaults:    |                                  | number of the first result and   |
+|                       | offset=0,     |                                  | limit is the number of results to|
+|                       | limit=20)     |                                  | return.                          |
++-----------------------+---------------+----------------------------------+----------------------------------+
+| all_fields            | 0 (default)   | all_fields=1                     | Each matching search result is   |
+|                       | or 1          |                                  | given as either a dataset name   |
+|                       |               |                                  | (0) or the full dataset record   |
+|                       |               |                                  | (1).                             |
++-----------------------+---------------+----------------------------------+----------------------------------+
+
+.. Note::
+
+ filter_by_openness and filter_by_downloadable were dropped from CKAN version 1.5 onwards.
+
+
+**Resource Parameters**
+
++-----------------------+---------------+-----------------------------------------+----------------------------------+
+| Param-Key             | Param-Value   | Example                                 |  Notes                           |
++=======================+===============+=========================================+==================================+
+| url, format,          | Search-String || url=statistics.org                     | Criteria to search the dataset   |
+| description           |               || format=xls                             | fields for. URL-encoded search   |
+|                       |               || description=Research+Institute         | text. This search string must be |
+|                       |               |                                         | found somewhere within the field |
+|                       |               |                                         | to match.                        |
+|                       |               |                                         | Case insensitive.                |
++-----------------------+---------------+-----------------------------------------+----------------------------------+
+| qjson                 | JSON encoded  | ['url':'www.statistics.org']            | All search parameters can be     |
+|                       | options       |                                         | json-encoded and supplied to this|
+|                       |               |                                         | parameter as a more flexible     |
+|                       |               |                                         | alternative in GET requests.     |
++-----------------------+---------------+-----------------------------------------+----------------------------------+
+| hash                  | Search-String |hash=b0d7c260-35d4-42ab-9e3d-c1f4db9bc2f0| Searches for an match of the     |
+|                       |               |                                         | hash field. An exact match or    |
+|                       |               |                                         | match up to the length of the    |
+|                       |               |                                         | hash given.                      |
++-----------------------+---------------+-----------------------------------------+----------------------------------+
+| all_fields            | 0 (default)   | all_fields=1                            | Each matching search result is   |
+|                       | or 1          |                                         | given as either an ID (0) or the |
+|                       |               |                                         | full resource record             |
++-----------------------+---------------+-----------------------------------------+----------------------------------+
+| offset, limit         | result-int    | offset=40&amp;limit=20                  | Pagination options. Offset is the|
+|                       | (defaults:    |                                         | number of the first result and   |
+|                       | offset=0,     |                                         | limit is the number of results to|
+|                       | limit=20)     |                                         | return.                          |
++-----------------------+---------------+-----------------------------------------+----------------------------------+
+
+.. Note::
+
+   Powerful searching from the command-line can be achieved with curl and the qjson parameter. In this case you need to remember to escapt the curly braces and use url encoding (e.g. spaces become ``%20``). For example::
+
+     curl 'http://thedatahub.org/api/search/dataset?qjson=\{"author":"The%20Stationery%20Office%20Limited"\}'
+
+
+**Revision Parameters**
+
++-----------------------+---------------+-----------------------------------------------------+----------------------------------+
+| Param-Key             | Param-Value   | Example                                             |  Notes                           |
++=======================+===============+=====================================================+==================================+ 
+| since_time            | Date-Time     | since_time=2010-05-05T19:42:45.854533               | The time can be less precisely   |
+|                       |               |                                                     | stated (e.g 2010-05-05).         |
++-----------------------+---------------+-----------------------------------------------------+----------------------------------+
+| since_id              | Uuid          | since_id=6c9f32ef-1f93-4b2f-891b-fd01924ebe08       | The stated id will not be        |
+|                       |               |                                                     | included in the results.         |
++-----------------------+---------------+-----------------------------------------------------+----------------------------------+
+
+Status Codes
+~~~~~~~~~~~~
+
+Standard HTTP status codes are used to signal method outcomes.
+
+===== =====
+Code  Name
+===== =====
+200   OK                 
+201   OK and new object created (referred to in the Location header)
+301   Moved Permanently  
+400   Bad Request     
+403   Not Authorized     
+404   Not Found          
+409   Conflict (e.g. name already exists)
+500   Service Error           
+===== =====
+
diff --git a/doc/api.rst b/doc/api.rst
index a908c717acf..18e68232f65 100644
--- a/doc/api.rst
+++ b/doc/api.rst
@@ -1,508 +1,86 @@
 .. index:: API
 
-===========================
-Reference: CKAN API
-===========================
+========
+CKAN API
+========
 
 .. toctree::
    :hidden:
    :maxdepth: 1
 
-The CKAN API provides programmatic access to the catalog of metadata stored in CKAN.
-
-Overview
---------
-
-The CKAN data catalog is not only available in a web browser, but also
-via its Application Programming Interface (API).
-
-The API can be used to view and change the catalog. This document describes the 
-resource locations, data formats, and status codes which comprise the CKAN
-API, so that anyone can create software applications that use the API
-service.
-
-The CKAN API follows the RESTful (Representational State 
-Transfer) style. Resource locations are separated both from the methods
-supported by the resources, and from the data formats and status codes
-used by the methods.
-
-Examples
---------
-
-For a tutorial and examples of using the CKAN API, see: http://wiki.ckan.org/Using_the_API
-
-Code Modules for Client Applications
--------------------------------------
-
-There are also some code modules (Python, PHP, Drupal, Perl etc.) that provide 
-convenient wrappers around much of the CKAN API. For full details of these, 
-please consult http://wiki.ckan.org/API
-
-Versions
---------
-
-The CKAN API is versioned, so that backwards incompatible changes can be
-introduced without removing existing support.  A particular version of the API
-can be used by including its version number after the API location and before
-the resource location.
-
-If the API version is not specified in the request, then the API will default to version 1.
-
-Versions 1 & 2
-~~~~~~~~~~~~~~
-
-These are very similar, but when the API returns a reference to an object, Version 1 API will return the Name of the object (e.g. "river-pollution") and Version 2 API will return the ID of the object (e.g. "a3dd8f64-9078-4f04-845c-e3f047125028").
-
-The reason for this is that Names can change, so to reliably refer to the same dataset every time, you will want to use the ID and therefore use API v2. Alternatively, many people prefer to deal with Names, so API v1 suits them.
+The CKAN platform is not only available in a web browser, but also via its
+Application Programming Interface (API). The API provides programmatic access
+to the CKAN system. The API is very powerful and allows you do everything (and more) you can do via the web interface.
 
-When making requests, you can call objects by either their Name or ID, interchangeably.
+Using the API you can do things like:
 
-The only exception for this is for Tag objects. Since Tag names are immutable, they are always referred to with their Name.
+* Access any bit of information in CKAN (if you are authorized!)
+* Edit any piece of information in CKAN
+* Create a whole new web front-end for CKAN (if you want!)
 
-Version 3
-~~~~~~~~~
-
-This version is in beta. To details of trying it out, see :doc:`apiv3`.
+The CKAN API follows the RESTful (Representational State Transfer) style and
+uses JSON by default.
 
+.. _tools-for-api:
 
+Tools for Accessing the API
+===========================
 
-API Details - Versions 1 & 2
-----------------------------
-
-Overview
-~~~~~~~~
+In using the API you will be performing HTTP requests to urls like: ``/api/rest/dataset`` with this returning data in JSON format.
 
-The CKAN data catalog is not only available in a web browser, but also
-via its Application Programming Interface (API).
+There are several ways you can access this URL directly:
 
-The API can be used to view and change the catalog. This document describes the 
-resource locations, data formats, and status codes which comprise the CKAN
-API, so that anyone can create software applications that use the API
-service.
+* Put this URL into your web browser and view or save the resulting response -- there are plugins for browsers like Firefox and Chrome that allow you to see JSON nicely formatted in your browser (e.g. `JSONView for Chrome <https://chrome.google.com/webstore/detail/chklaanhfefbnpoihckbnefhakgolnmc>`_)
+* Use a command-line program such as `curl <http://curl.haxx.se/>`_
+* Write a program in your favourite language that uses an http library to access the URL
 
-The CKAN API follows the RESTful (Representational State 
-Transfer) style. Resource locations are separated both from the methods
-supported by the resources, and from the data formats and status codes
-used by the methods.
+Clients
+-------
 
+Alternatively, you can access the API using one the dedicated tools or libraries written specifically for CKAN. The following clients are available:
 
-The CKAN API version 1 & 2 is separated into three parts.
+* `dpm (data package manager) <http://github.com/okfn/dpm/>`_: command-line client and Python library (maintained by core CKAN team)
+* `ckanclient - CKAN Python Client <http://pypi.python.org/pypi/ckanclient>`_: Python client maintained by the core CKAN team
+* `CKAN Ruby <https://github.com/apohllo/ckan>`_: Ruby Client
+* `Ckan_client-PHP <http://github.com/jeffreybarke/Ckan_client-PHP>`_: PHP client
+* `net-ckan <http://github.com/lukec/net-ckan>`_: PERL client
+* `ckanjs <http://github.com/okfn/ckanjs>`_: sophisticated Javascript client built on Backbone.
+* `Google Refine CKAN Extension <http://ckan.org/2011/07/05/google-refine-extension-for-ckan/>`_: Google Refine client which allows you to get and push data to and from a CKAN instance using Google Refine.
 
-* `Model API`_
-* `Search API`_
-* `Util API`_
+Quickstart Tutorial
+===================
 
-The resources, methods, and data formats of each are described below.
+.. toctree::
+   :maxdepth: 2
 
-Locators
-~~~~~~~~
+   api-tutorial.rst
 
-The locator for a given resource can be formed by appending
-the relative path for that resource to the API locator.
+.. _api-keys:
 
-  ``Resource Locator = API Locator + Resource Path``
+Authentication and API Keys
+===========================
 
-The API locators for the CKAN APIs (by version) are:
+CKAN can be configured to only allow authorized users to carry out certain
+actions (see :doc:`authorization` for more details). The authorization
+configuration is the same for actions done over the API as for those carried
+out in the CKAN web interfac, so a user has the same permissions, whichever way
+he/she accesses CKAN data.
 
- * ``/api`` (version 1)
- * ``/api/1`` (version 1)
- * ``/api/2`` (version 2)
+Thus, all actions **not** permitted to anonymous users, will require a user to
+identify themselves via some authentication method.
 
-The relative paths for each resource are listed in the sections below.
+.. note:: Depending on the authorization settings of the CKAN instance, a user
+          may need to authenticate themselves for all operations including
+          read. However, by default, CKAN allows anonymous read access and this
+          setup is assumed for the API usage examples.
 
-Model API
-~~~~~~~~~
+The standard authentication method utilized by CKAN is API keys and a user
+authenticates his/her user identity by supplying a header in the request
+containing the API key. The header field is either ``Authorization``,
+``X-CKAN-API-Key`` or configured with the `apikey_header_name` option. (Details
+of how to obtain an API key are below).
 
-Model resources are available at published locations. They are represented with
-a variety of data formats. Each resource location supports a number of methods.
-
-The data formats of the requests and the responses are defined below.
-
-Model Resources
-```````````````
-
-Here are the resources of the Model API.
-
-
-+--------------------------------+-------------------------------------------------------------------+
-| Model Resource                 | Location                                                          |
-+================================+===================================================================+
-| Dataset Register               | ``/rest/dataset``                                                 |
-+--------------------------------+-------------------------------------------------------------------+
-| Dataset Entity                 | ``/rest/dataset/DATASET-REF``                                     |
-+--------------------------------+-------------------------------------------------------------------+
-| Group Register                 | ``/rest/group``                                                   |
-+--------------------------------+-------------------------------------------------------------------+
-| Group Entity                   | ``/rest/group/GROUP-REF``                                         |
-+--------------------------------+-------------------------------------------------------------------+
-| Tag Register                   | ``/rest/tag``                                                     |
-+--------------------------------+-------------------------------------------------------------------+
-| Tag Entity                     | ``/rest/tag/TAG-NAME``                                            |
-+--------------------------------+-------------------------------------------------------------------+
-| Rating Register                | ``/rest/rating``                                                  |
-+--------------------------------+-------------------------------------------------------------------+
-| Dataset Relationships Register | ``/rest/dataset/DATASET-REF/relationships``                       |
-+--------------------------------+-------------------------------------------------------------------+
-| Dataset Relationships Register | ``/rest/dataset/DATASET-REF/RELATIONSHIP-TYPE``                   |
-+--------------------------------+-------------------------------------------------------------------+
-| Dataset Relationships Register | ``/rest/dataset/DATASET-REF/relationships/DATASET-REF``           |
-+--------------------------------+-------------------------------------------------------------------+
-| Dataset Relationship Entity    | ``/rest/dataset/DATASET-REF/RELATIONSHIP-TYPE/DATASET-REF``       |
-+--------------------------------+-------------------------------------------------------------------+
-| Dataset\'s Revisions Entity    | ``/rest/dataset/DATASET-REF/revisions``                           |
-+--------------------------------+-------------------------------------------------------------------+
-| Revision Register              | ``/rest/revision``                                                |
-+--------------------------------+-------------------------------------------------------------------+
-| Revision Entity                | ``/rest/revision/REVISION-ID``                                    |
-+--------------------------------+-------------------------------------------------------------------+
-| License List                   | ``/rest/licenses``                                                |
-+--------------------------------+-------------------------------------------------------------------+
-
-Possible values for DATASET-REF are the dataset id, or the current dataset name.
-
-Possible values for RELATIONSHIP-TYPE are described in the Relationship-Type data format.
-
-
-Model Methods
-`````````````
-
-Here are the methods of the Model API.
-
-+-------------------------------+--------+------------------+-------------------+
-| Resource                      | Method | Request          | Response          |
-+===============================+========+==================+===================+ 
-| Dataset Register              | GET    |                  | Dataset-List      |
-+-------------------------------+--------+------------------+-------------------+
-| Dataset Register              | POST   | Dataset          |                   |
-+-------------------------------+--------+------------------+-------------------+
-| Dataset Entity                | GET    |                  | Dataset           |
-+-------------------------------+--------+------------------+-------------------+
-| Dataset Entity                | PUT    | Dataset          |                   |
-+-------------------------------+--------+------------------+-------------------+
-| Group Register                | GET    |                  | Group-List        |
-+-------------------------------+--------+------------------+-------------------+
-| Group Register                | POST   | Group            |                   |
-+-------------------------------+--------+------------------+-------------------+
-| Group Entity                  | GET    |                  | Group             |
-+-------------------------------+--------+------------------+-------------------+
-| Group Entity                  | PUT    | Group            |                   |
-+-------------------------------+--------+------------------+-------------------+
-| Tag Register                  | GET    |                  | Tag-List          | 
-+-------------------------------+--------+------------------+-------------------+
-| Tag Entity                    | GET    |                  | Dataset-List      |
-+-------------------------------+--------+------------------+-------------------+
-| Rating Register               | POST   | Rating           |                   |
-+-------------------------------+--------+------------------+-------------------+
-| Rating Entity                 | GET    |                  | Rating            |
-+-------------------------------+--------+------------------+-------------------+
-| Dataset Relationships Register| GET    |                  | Pkg-Relationships |
-+-------------------------------+--------+------------------+-------------------+
-| Dataset Relationship Entity   | GET    |                  | Pkg-Relationship  |
-+-------------------------------+--------+------------------+-------------------+
-| Dataset Relationships Register| POST   | Pkg-Relationship |                   |
-+-------------------------------+--------+------------------+-------------------+
-| Dataset Relationship Entity   | PUT    | Pkg-Relationship |                   |
-+-------------------------------+--------+------------------+-------------------+
-| Dataset\'s Revisions Entity   | GET    |                  | Pkg-Revisions     |
-+-------------------------------+--------+------------------+-------------------+
-| Revision List                 | GET    |                  | Revision-List     |
-+-------------------------------+--------+------------------+-------------------+
-| Revision Entity               | GET    |                  | Revision          |
-+-------------------------------+--------+------------------+-------------------+
-| License List                  | GET    |                  | License-List      |
-+-------------------------------+--------+------------------+-------------------+
-
-In general:
-
-* GET to a register resource will *list* the entities of that type.
-
-* GET of an entity resource will *show* the entity's properties.
-
-* POST of entity data to a register resource will *create* the new entity.
-
-* PUT of entity data to an existing entity resource will *update* it.
-
-It is usually clear whether you are trying to create or update, so in these cases, HTTP POST and PUT methods are accepted by CKAN interchangeably.
-
-Model Formats
-`````````````
-
-Here are the data formats for the Model API:
-
-+--------------------+------------------------------------------------------------+
-| Name               | Format                                                     |
-+====================+============================================================+
-| Dataset-Ref        | Dataset-Name-String (API v1) OR Dataset-Id-Uuid (API v2)   |
-+--------------------+------------------------------------------------------------+
-| Dataset-List       | [ Dataset-Ref, Dataset-Ref, Dataset-Ref, ... ]             |
-+--------------------+------------------------------------------------------------+
-| Dataset            | { id: Uuid, name: Name-String, title: String, version:     | 
-|                    | String, url: String, resources: [ Resource, Resource, ...],| 
-|                    | author: String, author_email: String, maintainer: String,  |
-|                    | maintainer_email: String, license_id: String,              |
-|                    | tags: Tag-List, notes: String, extras: { Name-String:      |
-|                    | String, ... } }                                            |
-|                    | See note below on additional fields upon GET of a dataset. |
-+--------------------+------------------------------------------------------------+
-| Group-Ref          | Group-Name-String (API v1) OR Group-Id-Uuid (API v2)       |
-+--------------------+------------------------------------------------------------+
-| Group-List         | [ Group-Ref, Group-Ref, Group-Ref, ... ]                   |
-+--------------------+------------------------------------------------------------+
-| Group              | { name: Group-Name-String, title: String,                  |
-|                    | description: String, datasets: Dataset-List }              |
-+--------------------+------------------------------------------------------------+
-| Tag-List           | [ Name-String, Name-String, Name-String, ... ]             |
-+--------------------+------------------------------------------------------------+
-| Tag                | { name: Name-String }                                      |
-+--------------------+------------------------------------------------------------+
-| Resource           | { url: String, format: String, description: String,        |
-|                    | hash: String }                                             |
-+--------------------+------------------------------------------------------------+
-| Rating             | { dataset: Name-String, rating: int }                      |
-+--------------------+------------------------------------------------------------+
-| Pkg-Relationships  | [ Pkg-Relationship, Pkg-Relationship, ... ]                |
-+--------------------+------------------------------------------------------------+
-| Pkg-Relationship   | { subject: Dataset-Name-String,                            |
-|                    | object: Dataset-Name-String, type: Relationship-Type,      |
-|                    | comment: String }                                          |
-+--------------------+------------------------------------------------------------+
-| Pkg-Revisions      | [ Pkg-Revision, Pkg-Revision, Pkg-Revision, ... ]          |
-+--------------------+------------------------------------------------------------+
-| Pkg-Revision       | { id: Uuid, message: String, author: String,               |
-|                    | timestamp: Date-Time }                                     |
-+--------------------+------------------------------------------------------------+
-|Relationship-Type   | One of: 'depends_on', 'dependency_of',                     |
-|                    | 'derives_from', 'has_derivation',                          |
-|                    | 'child_of', 'parent_of',                                   |
-|                    | 'links_to', 'linked_from'.                                 |
-+--------------------+------------------------------------------------------------+
-| Revision-List      | [ revision_id, revision_id, revision_id, ... ]             |
-+--------------------+------------------------------------------------------------+
-| Revision           | { id: Uuid, message: String, author: String,               |
-|                    | timestamp: Date-Time, datasets: Dataset-List }             |
-+--------------------+------------------------------------------------------------+
-| License-List       | [ License, License, License, ... ]                         |
-+--------------------+------------------------------------------------------------+
-| License            | { id: Name-String, title: String, is_okd_compliant:        |
-|                    | Boolean, is_osi_compliant: Boolean, tags: Tag-List,        |
-|                    | family: String, url: String, maintainer: String,           |
-|                    | date_created: Date-Time, status: String }                  |
-+--------------------+------------------------------------------------------------+
-
-To send request data, create the JSON-format string (encode in UTF8) put it in the request body and send it using PUT or POST.
-
-Response data will be in the response body in JSON format.
-
-Notes:
-
- * When you update an object, fields that you don't supply will remain as they were before.
-
- * To delete an 'extra' key-value pair, supply the key with JSON value: ``null``
-
- * When you read a dataset, some additional information is supplied that you cannot modify and POST back to the CKAN API. These 'read-only' fields are provided only on the Dataset GET. This is a convenience to clients, to save further requests. This applies to the following fields:
-    
-===================== ================================
-Key                   Description 
-===================== ================================
-id                    Unique Uuid for the Dataset
-revision_id           Latest revision ID for the core Package data (but is not affected by changes to tags, groups, extras, relationships etc)
-metadata_created      Date the Dataset (record) was created
-metadata_modified     Date the Dataset (record) was last modified
-relationships         info on Dataset Relationships
-ratings_average         
-ratings_count            
-ckan_url              full URL of the Dataset
-download_url (API v1) URL of the first Resource
-isopen                boolean indication of whether dataset is open according to Open Knowledge Definition, based on other fields
-notes_rendered        HTML rendered version of the Notes field (which may contain Markdown)
-===================== ================================
-   
-
-
-Search API
-~~~~~~~~~~
-
-Search resources are available at published locations. They are represented with
-a variety of data formats. Each resource location supports a number of methods.
-
-The data formats of the requests and the responses are defined below.
-
-Search Resources
-````````````````
-
-Here are the published resources of the Search API.
-
-+---------------------------+--------------------------+
-| Search Resource           | Location                 |
-+===========================+==========================+
-| Dataset Search            | ``/search/dataset``      |
-+---------------------------+--------------------------+
-| Resource Search           | ``/search/resource``     |
-+---------------------------+--------------------------+
-| Revision Search           | ``/search/revision``     |
-+---------------------------+--------------------------+
-| Tag Counts                | ``/tag_counts``          |
-+---------------------------+--------------------------+
-
-See below for more information about dataset and revision search parameters.
-
-Search Methods
-``````````````
-
-Here are the methods of the Search API.
-
-+-------------------------------+--------+------------------------+--------------------------+
-| Resource                      | Method | Request                | Response                 |
-+===============================+========+========================+==========================+ 
-| Dataset Search                | POST   | Dataset-Search-Params  | Dataset-Search-Response  | 
-+-------------------------------+--------+------------------------+--------------------------+
-| Resource Search               | POST   | Resource-Search-Params | Resource-Search-Response | 
-+-------------------------------+--------+------------------------+--------------------------+
-| Revision Search               | POST   | Revision-Search-Params | Revision-List            | 
-+-------------------------------+--------+------------------------+--------------------------+
-| Tag Counts                    | GET    |                        | Tag-Count-List           | 
-+-------------------------------+--------+------------------------+--------------------------+
-
-It is also possible to supply the search parameters in the URL of a GET request, 
-for example ``/api/search/dataset?q=geodata&amp;allfields=1``.
-
-Search Formats
-``````````````
-
-Here are the data formats for the Search API.
-
-+-------------------------+------------------------------------------------------------+
-| Name                    | Format                                                     |
-+=========================+============================================================+
-| Dataset-Search-Params   | { Param-Key: Param-Value, Param-Key: Param-Value, ... }    |
-| Resource-Search-Params  | See below for full details of search parameters across the | 
-| Revision-Search-Params  | various domain objects.                                    |
-+-------------------------+------------------------------------------------------------+
-| Dataset-Search-Response | { count: Count-int, results: [Dataset, Dataset, ... ] }    |
-+-------------------------+------------------------------------------------------------+
-| Resource-Search-Response| { count: Count-int, results: [Resource, Resource, ... ] }  |
-+-------------------------+------------------------------------------------------------+
-| Revision-List           | [ Revision-Id, Revision-Id, Revision-Id, ... ]             |
-|                         | NB: Ordered with youngest revision first                   |
-+-------------------------+------------------------------------------------------------+
-| Tag-Count-List          | [ [Name-String, Integer], [Name-String, Integer], ... ]    |
-+-------------------------+------------------------------------------------------------+
-
-The ``Dataset`` and ``Revision`` data formats are as defined in `Model Formats`_.
-
-**Dataset Parameters**
-
-+-----------------------+---------------+----------------------------------+----------------------------------+
-| Param-Key             | Param-Value   | Examples                         |  Notes                           |
-+=======================+===============+==================================+==================================+
-| q                     | Search-String || q=geodata                       | Criteria to search the dataset   |
-|                       |               || q=government+sweden             | fields for. URL-encoded search   |
-|                       |               || q=%22drug%20abuse%22            | text. (You can also concatenate  |
-|                       |               || q=tags:"river pollution"        | words with a '+' symbol in a     |
-|                       |               |                                  | URL.) Search results must contain|
-|                       |               |                                  | all the specified words.  You    |
-|                       |               |                                  | can also search within specific  |
-|                       |               |                                  | fields.                          |
-+-----------------------+---------------+----------------------------------+----------------------------------+
-| qjson                 | JSON encoded  | ['q':'geodata']                  | All search parameters can be     |
-|                       | options       |                                  | json-encoded and supplied to this|
-|                       |               |                                  | parameter as a more flexible     |
-|                       |               |                                  | alternative in GET requests.     |
-+-----------------------+---------------+----------------------------------+----------------------------------+
-|title,                 | Search-String || title=uk&amp;tags=health        | Search in a particular a field.  |
-|tags, notes, groups,   |               || department=environment          |                                  |
-|author, maintainer,    |               || tags=health&tags=pollution      |                                  |
-|update_frequency, or   |               || tags=river%20pollution          |                                  |
-|any 'extra' field name |               |                                  |                                  |
-|e.g. department        |               |                                  |                                  |
-+-----------------------+---------------+----------------------------------+----------------------------------+
-| order_by              | field-name    | order_by=name                    | Specify either rank or the field |
-|                       | (default=rank)|                                  | to sort the results by           |
-+-----------------------+---------------+----------------------------------+----------------------------------+
-| offset, limit         | result-int    | offset=40&amp;limit=20           | Pagination options. Offset is the|
-|                       | (defaults:    |                                  | number of the first result and   |
-|                       | offset=0,     |                                  | limit is the number of results to|
-|                       | limit=20)     |                                  | return.                          |
-+-----------------------+---------------+----------------------------------+----------------------------------+
-| all_fields            | 0 (default)   | all_fields=1                     | Each matching search result is   |
-|                       | or 1          |                                  | given as either a dataset name   |
-|                       |               |                                  | (0) or the full dataset record   |
-|                       |               |                                  | (1).                             |
-+-----------------------+---------------+----------------------------------+----------------------------------+
-
-.. Note::
-
- filter_by_openness and filter_by_downloadable were dropped from CKAN version 1.5 onwards.
-
-
-**Resource Parameters**
-
-+-----------------------+---------------+-----------------------------------------+----------------------------------+
-| Param-Key             | Param-Value   | Example                                 |  Notes                           |
-+=======================+===============+=========================================+==================================+
-| url, format,          | Search-String || url=statistics.org                     | Criteria to search the dataset   |
-| description           |               || format=xls                             | fields for. URL-encoded search   |
-|                       |               || description=Research+Institute         | text. This search string must be |
-|                       |               |                                         | found somewhere within the field |
-|                       |               |                                         | to match.                        |
-|                       |               |                                         | Case insensitive.                |
-+-----------------------+---------------+-----------------------------------------+----------------------------------+
-| qjson                 | JSON encoded  | ['url':'www.statistics.org']            | All search parameters can be     |
-|                       | options       |                                         | json-encoded and supplied to this|
-|                       |               |                                         | parameter as a more flexible     |
-|                       |               |                                         | alternative in GET requests.     |
-+-----------------------+---------------+-----------------------------------------+----------------------------------+
-| hash                  | Search-String |hash=b0d7c260-35d4-42ab-9e3d-c1f4db9bc2f0| Searches for an match of the     |
-|                       |               |                                         | hash field. An exact match or    |
-|                       |               |                                         | match up to the length of the    |
-|                       |               |                                         | hash given.                      |
-+-----------------------+---------------+-----------------------------------------+----------------------------------+
-| all_fields            | 0 (default)   | all_fields=1                            | Each matching search result is   |
-|                       | or 1          |                                         | given as either an ID (0) or the |
-|                       |               |                                         | full resource record             |
-+-----------------------+---------------+-----------------------------------------+----------------------------------+
-| offset, limit         | result-int    | offset=40&amp;limit=20                  | Pagination options. Offset is the|
-|                       | (defaults:    |                                         | number of the first result and   |
-|                       | offset=0,     |                                         | limit is the number of results to|
-|                       | limit=20)     |                                         | return.                          |
-+-----------------------+---------------+-----------------------------------------+----------------------------------+
-
-.. Note::
-
-   Powerful searching from the command-line can be achieved with curl and the qjson parameter. In this case you need to remember to escapt the curly braces and use url encoding (e.g. spaces become ``%20``). For example::
-
-     curl 'http://thedatahub.org/api/search/dataset?qjson=\{"author":"The%20Stationery%20Office%20Limited"\}'
-
-
-**Revision Parameters**
-
-+-----------------------+---------------+-----------------------------------------------------+----------------------------------+
-| Param-Key             | Param-Value   | Example                                             |  Notes                           |
-+=======================+===============+=====================================================+==================================+ 
-| since_time            | Date-Time     | since_time=2010-05-05T19:42:45.854533               | The time can be less precisely   |
-|                       |               |                                                     | stated (e.g 2010-05-05).         |
-+-----------------------+---------------+-----------------------------------------------------+----------------------------------+
-| since_id              | Uuid          | since_id=6c9f32ef-1f93-4b2f-891b-fd01924ebe08       | The stated id will not be        |
-|                       |               |                                                     | included in the results.         |
-+-----------------------+---------------+-----------------------------------------------------+----------------------------------+
-
-API Keys
-~~~~~~~~
-
-You will need to supply an API Key for certain requests to the CKAN API:
-
-* For any action which makes a change to a resource (i.e. all POST methods on register resources, and PUT/POST methods on entity resources).
-
-* If the particular resource's authorization set-up is not open to 
-  visitors for the action.
-
-To obtain your API key:
-
-1. Log-in to the particular CKAN website: /user/login
-
-2. The user page shows the API Key: /user/me
-
-The key should be passed in the API request header ''Authorization'' (or an alternative may be provided such as ''X-CKAN-API-KEY''). For example::
+For example::
 
   curl http://thedatahub.org/api/rest/package -d '{"name": "test"}' -H 'Authorization: fde34a3c-b716-4c39-8dc4-881ba115c6d4'
 
@@ -512,30 +90,15 @@ key is not authorized for the operation, or the header is somehow malformed,
 then the requested operation will not be carried out and the CKAN API will
 respond with status code 403.
 
-For more information about HTTP Authorization header, please refer to section
-14.8 of `RFC 2616 <http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.8>`_.
-
+Obtaining an API key
+--------------------
 
-Status Codes
-~~~~~~~~~~~~
+1. Log-in to the particular CKAN website: ``/user/login``
 
-Standard HTTP status codes are used to signal method outcomes.
+2. Your user page will show the API Key in your details section at the top of the screen
 
-===== =====
-Code  Name
-===== =====
-200   OK                 
-201   OK and new object created (referred to in the Location header)
-301   Moved Permanently  
-400   Bad Request     
-403   Not Authorized     
-404   Not Found          
-409   Conflict (e.g. name already exists)
-500   Service Error           
-===== =====
-
-JSONP formatted responses
-~~~~~~~~~~~~~~~~~~~~~~~~~
+JSONP Support
+=============
 
 To cater for scripts from other sites that wish to access the API, the data can be returned in JSONP format, where the JSON data is 'padded' with a function call. The function is named in the 'callback' parameter.
 
@@ -546,138 +109,56 @@ Example normal request::
 
 but now with the callback parameter::
 
- GET /api/rest/dataset/pollution_stats?callback=jsoncallback
+ GET /api/rest/dataset/pollution_stats?callback=name-of-callback-function
  returns: jsoncallback({"name": "pollution_stats", ... });
 
-This parameter can apply to all GET requests in the API.
-
-
-Util API
-~~~~~~~~
-
-These utilities don't involve the CKAN database, but can be useful to scripts and code on the Web front-end (i.e. AJAX calls). 
-
-The response format is JSON. Javascript calls may want to use the JSONP formatting.
-
-.. Note::
-
-  Some CKAN deployments have the API deployed at a different domain to the main CKAN website. To make sure that the AJAX calls in the Web UI work, you'll need to configue the ckan.api_url. e.g.::
-
-    ckan.api_url = http://api.example.com/
-
-
-dataset autocomplete
-````````````````````
-
-There an autocomplete API for package names which matches on name or title.
-
-This URL:
+This parameter can apply to all POST requests to the Action API and GET requests to the Search API and v1/v2/v3 APIs.
 
-::
+.. _api-reference: 
 
-    /api/2/util/dataset/autocomplete?incomplete=a%20novel
+API: Reference
+==============
 
-Returns:
+Model, Search and Actions APIs
+------------------------------
 
-::
+These sections describe the resource locations, data formats, and status codes
+which comprise the CKAN API.
 
-    {"ResultSet": {"Result": [{"match_field": "title", "match_displayed": "A Novel By Tolstoy (annakarenina)", "name": "annakarenina", "title": "A Novel By Tolstoy"}]}}
-
-
-tag autocomplete
-````````````````
-
-There is also an autocomplete API for tags which looks like this:
-
-This URL:
-
-::
-
-    /api/2/util/tag/autocomplete?incomplete=ru
-
-Returns:
-
-::
-
-    {"ResultSet": {"Result": [{"Name": "russian"}]}}
-
-resource format autocomplete
-````````````````````````````
-
-Similarly, there is an autocomplete API for the resource format field
-which is available at:
-
-::
-
-    /api/2/util/resource/format_autocomplete?incomplete=cs
-
-This returns:
-
-::
-
-    {"ResultSet": {"Result": [{"Format": "csv"}]}}
-
-markdown
-````````
-
-Takes a raw markdown string and returns a corresponding chunk of HTML. CKAN uses the basic Markdown format with some modifications (for security) and useful additions (e.g. auto links to datasets etc. e.g. ``dataset:river-quality``).
-
-Example::
-
-    /api/util/markdown?q=<http://ibm.com/>
-
-Returns::
-
-    "<p><a href="http://ibm.com/" target="_blank" rel="nofollow">http://ibm.com/</a>\n</p>"
-
-is slug valid
-`````````````
-
-Checks a name is valid for a new dataset (package) or group, with respect to it being used already.
-
-Example::
-
-    /api/2/util/is_slug_valid?slug=river-quality&type=package
-
-Response::
-
-    {"valid": true}
-
-munge package name
-``````````````````
-
-For taking an readable identifier and munging it to ensure it is a valid dataset id. Symbols and whitespeace are converted into dashes. Example::
-
-    /api/util/dataset/munge_name?name=police%20spending%20figures%202009
-
-Returns::
-
-    "police-spending-figures-2009"
+The CKAN API is versioned, so that backwards incompatible changes can be
+introduced without removing existing support. A particular version of the API
+can be used by including its version number after the API location and before
+the resource location.
 
-munge title to package name
-```````````````````````````
+If the API version is not specified in the request, then the API will default
+to version 1.
 
-For taking a title of a package and munging it to a readable and valid dataset id. Symbols and whitespeace are converted into dashes, with multiple dashes collapsed. Ensures that long titles with a year at the end preserves the year should it need to be shortened. Example::
+Version 1 and 2 
+~~~~~~~~~~~~~~~
 
-    /api/util/dataset/munge_title_to_name?title=police:%20spending%20figures%202009
+.. toctree::
+   :maxdepth: 2
 
-Returns::
+   api-v2.rst
 
-    "police-spending-figures-2009"
+Version 3
+~~~~~~~~~
 
+This version is in beta.
 
-munge tag
-`````````
+.. toctree::
+   :maxdepth: 2
 
-For taking a readable word/phrase and munging it to a valid tag (name). Symbols and whitespeace are converted into dashes. Example::
+   apiv3.rst
 
-    /api/util/tag/munge?tag=water%20quality
+Util API
+--------
 
-Returns::
+The Util API provides various utility APIs -- e.g. auto-completion APIs used by
+front-end javascript.
 
-    "water-quality"
+.. toctree::
+   :maxdepth: 2
 
-Action API
-~~~~~~~~~~
+   api-util.rst
 
-See: :doc:`apiv3`
diff --git a/doc/apiv3.rst b/doc/apiv3.rst
index eb58abb4512..9e0657a6ea5 100644
--- a/doc/apiv3.rst
+++ b/doc/apiv3.rst
@@ -1,5 +1,6 @@
-Version 3
-~~~~~~~~~
+=======================================
+Model, Search and Action API: Version 3
+=======================================
 
 API Versions
 ~~~~~~~~~~~~
@@ -239,35 +240,6 @@ Examples::
  curl http://test.ckan.net/api/action/package_list -d '{}'
  curl http://test.ckan.net/api/action/package_show -d '{"id": "fd788e57-dce4-481c-832d-497235bf9f78"}'
 
-Authorization Header
-====================
-
-CKAN can be configured to only allow authorized users to carry out certain actions. For example, in a default installation of CKAN, anyone can read packages, you have to be a logged-in user to edit them and editing permissions for a dataset can only be done by the dataset creator and a 'sysadmin' user.
-
-The authorization configuration is the same between the CKAN web interface and the API, so a user has the same permissions, whichever way he/she accesses CKAN data.
-
-Depending on the authorization settings of the CKAN instance, a user may not need to identify him/herself for simple read operations. This is the case for thedatahub.org and is assumed for the API usage examples.
-
-When using the API, a user authenticates his/her user identity by supplying a header in the request. The header key is either ``Authorization``, ``X-CKAN-API-Key`` or configured with the `apikey_header_name` option. The value of the header is the user's API key, provided on the user's page in the CKAN web interface.
-
-To obtain your API key:
-
-1. Log-in to the particular CKAN website: /user/login
-
-2. The user page shows the API Key: /user/me
-
-The key should be passed in the API request header ''Authorization'' (or an alternative may be provided such as ''X-CKAN-API-KEY''). For example::
-
-  curl http://thedatahub.org/api/rest/package -d '{"name": "test"}' -H 'Authorization: fde34a3c-b716-4c39-8dc4-881ba115c6d4'
-
-If requests that are required to be authorized are not sent with a 
-valid Authorization header, for example the user associated with the 
-key is not authorized for the operation, or the header is somehow malformed,
-then the requested operation will not be carried out and the CKAN API will
-respond with status code 403.
-
-For more information about HTTP Authorization header, please refer to section
-14.8 of `RFC 2616 <http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.8>`_.
 
 
 Responses
@@ -488,19 +460,3 @@ Code  Name
 500   Service Error - unhandled error - the system administrator has been notified
 ===== =====
 
-JSONP formatted responses
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To cater for scripts from other sites that wish to access the API, the data can be returned in JSONP format, where the JSON data is 'padded' with a function call. The function is named in the 'callback' parameter.
-
-Example normal request::
-
- curl http://test.ckan.net/api/action/package_show -d '{"id": "fd788e57-dce4-481c-832d-497235bf9f78"}'
- returns: {"help": null, "success": true, "result": {"name": "uk-quango-data", ...}}
-
-but now with the callback parameter::
-
- curl http://test.ckan.net/api/action/package_show?callback=jsoncallback -d '{"id": "fd788e57-dce4-481c-832d-497235bf9f78"}'
- returns: jsoncallback({"help": null, "success": true, "result": {"name": "uk-quango-data", ...}});
-
-This parameter can apply to all POST requests to the Action API and GET requests to the Search API and v1/v2 APIs.
diff --git a/doc/conf.py b/doc/conf.py
index 61880937649..c8294fe091c 100644
--- a/doc/conf.py
+++ b/doc/conf.py
@@ -26,7 +26,7 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -43,7 +43,14 @@
 # General information about the project.
 project = u'CKAN Data Management System Documentation'
 project_short_name = u'CKAN'
-copyright = u'&copy; Copyright 2009-2012, Open Knowledge Foundation'
+copyright = u'''&copy; 2009-2012, <a href="http://okfn.org/">Open Knowledge Foundation</a>.
+    Licensed under <a
+    href="http://creativecommons.org/licenses/by-sa/3.0/">Creative Commons
+    Attribution ShareAlike (Unported) v3.0 License</a>.<br />
+    <img src="http://i.creativecommons.org/l/by-sa/3.0/80x15.png" alt="CC License Logo" />
+    <a href="http://opendefinition.org/"><img src="http://assets.okfn.org/images/ok_buttons/oc_80x15_blue.png" border="0" 
+      alt="{{ _('Open Content') }}" /></a>
+  '''
 html_show_sphinx = False
 
 # The version info for the project you're documenting, acts as replacement for
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 9c0f81ed5be..f0c533c3973 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -150,7 +150,7 @@ Example::
 
 If there is a page which allows you to download a dump of the entire catalogue then specify the URL and the format here, so that it can be advertised in the web interface. ``dumps_format`` is just a string for display.
 
-For more information on using dumpfiles, see :doc:`database_dumps`.
+For more information on using dumpfiles, see :doc:`database-dumps`.
 
 recaptcha
 ^^^^^^^^^
diff --git a/doc/domain-model-dataset.rst b/doc/domain-model-dataset.rst
new file mode 100644
index 00000000000..7b63bc2e8a4
--- /dev/null
+++ b/doc/domain-model-dataset.rst
@@ -0,0 +1,47 @@
+=======
+Dataset
+=======
+
+A Dataset (known as a (Data) Package in CKAN <=1.4) is the object representing
+datasets in CKAN and, as such, is the central domain object.
+
+When you retrieve a Dataset in the CKAN API it will automatically include
+information from most related objects including Tags, Resources, Relationships,
+Ratings etc.
+
+Schema
+======
+
+Mappings to dublin core are in brackets (dc:...).
+
+* id: unique id
+* name (slug): unique name that is used in urls and for identification
+* title (dc:title): short title for dataset
+* url (home page): home page for this dataset
+* author (dc:creator): original creator of the dataset
+* author_email: 
+* maintainer: current maintainer or publisher of the dataset
+* maintainer_email:
+* license (dc:rights): license under which the dataset is made available
+* version: dataset version
+* notes (description) (dc:description): description and other information about the dataset
+* tags: arbitrary textual tags for the dataset
+* state: state of dataset in CKAN system (active, deleted, pending)
+* resources: list of [[Domain Model/Resource|Resources]]
+* groups: list of [[Domain Model/Group|Groups]] this dataset is a member of
+* "extras" - arbitrary, unlimited additional key/value fields
+
+The schema in code (see default_package_schema):
+https://github.com/okfn/ckan/blob/master/ckan/logic/schema.py
+
+Background
+==========
+
+The CKAN Dataset was originally heavily based on the kind of packaging
+information provided for software but with some modifications. One of our aims
+is to keep things simple and as generic as possible as we have data from a lot
+of different domains.
+
+Thus we've tried to keep the core metadata pretty restricted but allow for
+additional info either via tags or via "extra" arbitrary key/value fields.
+
diff --git a/doc/domain-model-resource.rst b/doc/domain-model-resource.rst
new file mode 100644
index 00000000000..8e8d4c0ac78
--- /dev/null
+++ b/doc/domain-model-resource.rst
@@ -0,0 +1,127 @@
+========
+Resource
+========
+
+A Resource corresponds to a file, API or other online data resource. A Resource
+is associated to a :doc:`domain-model-dataset` (which may have several Resources).
+
+Attributes
+==========
+
+Like Datasets, Resources can have arbitrary set of attributes. Thus, the
+attributes listed here are not exhaustive and may be extended by specific
+extensions.
+
+Standard
+--------
+
+These are standard set of attributes utilized by 'core' CKAN.
+
+* url: the key attribute of a resource (and the only required attribute). The
+  url points to the location online where the content of that resource can be
+  found. For a file this would be the location online of that file (or more
+  generally a url which yields the bitstream representing the contents of that
+  file -- for example some "files" are only generated on demand from a
+  database). For an API this would be the endpoint for the api.
+* name: a name for this resource (could be used in a ckan url)
+* description: A brief description (one sentence) of the Resource. Longer
+  descriptions can go in notes field of the associated Data Package.
+* type: the type of the resource. One of: file | file.upload | api |
+  visualization | code | documentation
+
+  * file - a file (GET of this url should yield a bitstream)
+  * file.upload - a file uploaded to the :doc:`filestore`
+  * api - an API
+  * visualization - a visualization
+  * code - code related to this dataset (for example a reference to a code
+    repository containing processing scripts)
+  * documentation - documentation for this dataset
+
+* format: human created format string with possible nesting e.g. zip:csv. See
+  below for details of the format field.
+* mimetype: standard mimetype (e.g. for zipped csv would be application/zip)
+* mimetype-inner: mimetype of innermost object (so for example would be
+  text/csv)
+* size: size of the resource (content length). Usually only relevant for
+  resources of type file.
+* last_modified: the date when this resource's data was last modified (NB:
+  *not* the date when the metadata was modified).
+* hash: md5 or sha-1 hash
+
+Resource Quality Information
+----------------------------
+
+See http://wiki.ckan.org/Data_Quality. This information, while directly related
+resources, will not be stored on the resource table. It is currently undecided
+whether the Resource object will have this data directly available (e.g. via a
+'quality' attribute).
+
+Attributes for FileStore Archiving and DataStore Usage
+------------------------------------------------------
+
+Resource data may have been archived into the FileStore or stored into the
+DataStore. In these cases the following additional attributes are used:
+
+* cache_url: url for cache of object in :doc:`filestore`
+
+  * Note could be same as resource url if resource directly stored in storage
+
+* cache_last_updated: timestamp when cached version was created
+
+* webstore_url: set to non-empty value if data is in the doc:`datastore` (note
+  unusual naming is a holdover from previous usage)
+* webstore_last_updated: timestamp when webstore was last updated
+
+Resource Format Strings
+=======================
+
+Conventions on format strings:
+
+* file: mime-type or file extensions (for common tile types)
+
+  * Examples: csv (text/csv), xls (application/vnd.ms-excel or application/xls
+    -- there are about 6 alternatives!), html (text/html), pdf
+    (application/pdf) etc
+
+* api: {spec-type}+{mime-type-of-standard-response}
+
+  * Examples: sparql+rdf/xml, rest+json
+
+* service: service/{service-identifier}/{type-of-object-or-file-format}
+
+  * Examples: service/gdocs/spreadsheet (google docs spreadsheet)
+
+Nested Formats
+--------------
+
+It is very common for files to be provided in compressed form (e.g. zip, targz,
+tar). Strictly the format of this object is the format for the compression
+(e.g. zip). However, for users it is the underlying format that will matter. To
+solve this one provides the formats in nested order separate by a colon.
+Formats should be provide in outermost first (i.e. start with format of last
+layer and work inwards). Examples:
+
+* zip:json - a zipped json file e.g. myfile.json.zip 
+* targz:xml - an xml file that has been tar'd and gzipped e.g. myfile.xml.tgz
+* torrent:zip:csv - a csv file that has been zipped and then provided as a torrent
+
+Multiple Formats for Same Resources
+-----------------------------------
+
+It is common for a given API to provide data in multiple formats, for example
+xml and json. In this case use the '||' term. Examples:
+
+* api/xml||json - an API providing both xml and json
+
+Formats for resources that are listings or index pages
+------------------------------------------------------
+
+It is common, at present, to find projects where the data is in lots of files
+with these files listed on an index page. Rather than attempt to create a
+resource entry for each file we have adopted the convention of creating a
+resource for the relevant index page with a special format string beginning
+"index", e.g.:
+
+* index/html (an index page in html format)
+* index/ftp (an index page for a ftp site)
+
diff --git a/doc/domain-model-task-status.rst b/doc/domain-model-task-status.rst
new file mode 100644
index 00000000000..7c5b1b72d81
--- /dev/null
+++ b/doc/domain-model-task-status.rst
@@ -0,0 +1,27 @@
+===========
+Task Status
+===========
+
+The Task Status domain object is essentially a key/value store that is used by
+CKAN Tasks to store the results of each processing task.
+
+Schema
+======
+
+Each task status entry consists of the following required fields:
+
+* **id** [UnicodeText]: A unique ID for each status object. Automatically generated if not provided.
+* **entity_id** [UnicodeText]:  Each task_status entry is assumed to be information about a task that performs some operation on another CKAN domain object (usually either a dataset/package or a resource). This refers to the ID of that object.
+* **entity_type** [UnicodeText]: The type of CKAN domain object that the task operates on (eg: resource).
+* **task_type** [UnicodeText]: The type of CKAN Task (eg: qa, webstorer, archiver, etc).
+* **key** [UnicodeText]: Key descriptor for data being stored.
+* **value** [UnicodeText]: Actual data being stored.
+
+.. note:: each task status entry must be unique on *(entity_id, task_type, key)*.
+
+They also contain a number of optional fields:
+
+* **state** [UnicodeText]: The current (or final) state of the task.
+* **error** [UnicodeText]: Information about any error that occurred during processing.
+* **last_updated** [DateTime]: The time at which this entry was last updated. Defaults to the current time.
+
diff --git a/doc/domain-model.rst b/doc/domain-model.rst
new file mode 100644
index 00000000000..a834f2c492f
--- /dev/null
+++ b/doc/domain-model.rst
@@ -0,0 +1,44 @@
+=================
+CKAN Domain Model
+=================
+
+Overview
+========
+
+* **Datasets**: the central entity in CKAN. Associated to Datasets are 'Resources' (files,
+  APIs etc) and a large variety of metadata. 
+  
+  * **Core metadata**: Datasets have a set of "core" metadata attributes
+  * **Unlimited additional metadata**: Datasets may have an unlimited amount of
+    arbitrary additional metadata in the form of "extra" key/value
+    associations.
+  * **Relationships**: relationships between datasets (such as depends on,
+    child of, derived from etc)
+
+* **Resources**: the actual data or APIs associated to a dataset are entered
+  into Resources.
+
+Additionally:
+
+* **Revisions**: All metadata in the CKAN system is *revisioned* -- i.e. all
+  changes are recorded. This support reverting changes, viewing changes, and,
+  perhaps most importantly going forward, the exchange of metadata changesets
+  between different CKAN instances or CKAN and other catalogues. However, this
+  is not a core part of the schema.
+* **Task Status**: A key/value store used by CKAN Tasks (background processes).
+
+Entity List
+===========
+
+* :doc:`domain-model-dataset`
+* :doc:`domain-model-resource` - a file, API or other resource
+* Group
+* Dataset Relationship - a relationship between Data Datasets.
+* Tag - tags can be applied to packages.
+* Vocabulary - tags can belong to vocabularies.
+
+Part of the domain model but not central:
+
+* Revision - changes to the domain model
+* :doc:`task-status` - key/value information stored by CKAN Tasks
+
diff --git a/doc/index.rst b/doc/index.rst
index 61ad018da4f..9fca3db10fb 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -28,10 +28,13 @@ Customizing and Extending
    :maxdepth: 2
 
    theming
+   extensions
    writing-extensions
    forms
    form-integration
-   extensions
+   linked-data-and-rdf
+   filestore
+   datastore
 
 Publishing Datasets
 ===================
@@ -41,17 +44,15 @@ Publishing Datasets
 
    loading-data
    authorization
-   filestore
-   datastore
+   publisher-profile
 
 The CKAN API
 ============
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 3
 
    api
-   apiv3
 
 General Administration
 ======================
@@ -59,6 +60,7 @@ General Administration
 .. toctree::
    :maxdepth: 2
 
+   administrative-dashboard
    paster
    database-dumps
    configuration
@@ -70,6 +72,10 @@ For CKAN Developers
 .. toctree::
    :maxdepth: 2
 
+   domain-model
+   domain-model-dataset
+   domain-model-resource
+   domain-model-task-status
    i18n
    test
    buildbot
diff --git a/doc/install-from-source.rst b/doc/install-from-source.rst
index 403ff331574..c6dc209c2ca 100644
--- a/doc/install-from-source.rst
+++ b/doc/install-from-source.rst
@@ -211,7 +211,7 @@ If you're using a remote host with password authentication rather than SSL authe
 
 
 8. Setup Solr
-~~~~~~~~~~~~~~
+~~~~~~~~~~~~~
 
 Set up Solr following the instructions on :ref:`solr-single` or :ref:`solr-multi-core` depending on your needs.
 
@@ -222,7 +222,7 @@ Set appropriate values for the ``ckan.site_id`` and ``solr_url`` config variable
        ckan.site_id=my_ckan_instance
        solr_url=http://127.0.0.1:8983/solr
 
-9. Create database tables.
+9. Create database tables
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Now that you have a configuration file that has the correct settings for
diff --git a/doc/linked-data-and-rdf.rst b/doc/linked-data-and-rdf.rst
new file mode 100644
index 00000000000..19442ef73f8
--- /dev/null
+++ b/doc/linked-data-and-rdf.rst
@@ -0,0 +1,52 @@
+===================
+Linked Data and RDF
+===================
+
+CKAN has extensive support for linked data and RDF. In particular, there is
+complete and functional mapping of the CKAN dataset schema to linked data
+formats.
+
+
+Enabling and Configuring Linked Data Support
+============================================
+
+In CKAN <= 1.6 please install the RDF extension: https://github.com/okfn/ckanext-rdf
+
+In CKAN >= 1.6.1, basic RDF support will be available directly in core.
+
+Configuration
+-------------
+
+.. todo:: fill this in (imagine that config instructions for extension will be
+          in extension but for >= 1.6.1 we have stuff in core)
+
+
+Accessing Linked Data
+=====================
+
+To access linked data versions just access the :doc:`api` in the usual way but
+set the Accept header to the format you would like to be returned. For
+example::
+
+ curl -L -H "Accept: application/rdf+xml" http://thedatahub.org/dataset/gold-prices
+ curl -L -H "Accept: text/n3" http://thedatahub.org/dataset/gold-prices
+
+
+Schema Mapping
+==============
+
+There are various vocabularies that can be used for describing datasets:
+
+* Dublin core: these are the most well-known and basic. Dublin core terms includes the class *dct:Dataset*.
+* DCAT_ - vocabulary for catalogues of datasets
+* VoID_ - vocabulary of interlinked datasets. Specifically designed for describing *rdf* datasets. Perfect except for the fact that it is focused on RDF
+* SCOVO_: this is more oriented to statistical datasets but has a *scovo:Dataset* class.
+
+At the present CKAN uses mostly DCAT and Dublin Core.
+
+.. _DCAT: http://vocab.deri.ie/dcat
+.. _VoID: http://rdfs.org/ns/void
+.. _SCOVO: http://sw.joanneum.at/scovo/schema.html
+
+.. todo:: put in an example of converted data to illustrate the schema
+
diff --git a/doc/paster.rst b/doc/paster.rst
index 6d607af3756..c664a74dc17 100644
--- a/doc/paster.rst
+++ b/doc/paster.rst
@@ -198,7 +198,7 @@ To load it in again, you first have to clean the database of existing data (be c
 
 .. warning: The pg_dump file is a complete backup of the database in plain text, and includes API keys and other user data which may be regarded as private. So keep it secure, like your database server.
 
-The pg_dump file notes which PostgreSQL user 'owns' the data on the server. Because the PostgreSQL user (by default) is identified as the current Linux user, and this is setup to be ``ckanINSTANCE`` where ``INSTANCE`` is the name of the CKAN instance. This means if you want to restore the pg_dump as another CKAN instance name (often needed if you move it to another server) then you will need to change the database owner - see :doc:`howto-editing--database-ownership`.
+The pg_dump file notes which PostgreSQL user 'owns' the data on the server. Because the PostgreSQL user (by default) is identified as the current Linux user, and this is setup to be ``ckanINSTANCE`` where ``INSTANCE`` is the name of the CKAN instance. This means if you want to restore the pg_dump as another CKAN instance name (often needed if you move it to another server) then you will need to change the database owner - see :doc:`howto-editing-database-ownership`.
 
 Upgrade migration
 ~~~~~~~~~~~~~~~~~
@@ -210,7 +210,7 @@ When you upgrade CKAN software by any method *other* than the package update des
 Creating dump files
 ~~~~~~~~~~~~~~~~~~~
 
-For information on using ``db`` to create dumpfiles, see :doc:`database_dumps`.
+For information on using ``db`` to create dumpfiles, see :doc:`database-dumps`.
 
 
 ratings: Manage dataset ratings
diff --git a/doc/publisher-profile.rst b/doc/publisher-profile.rst
new file mode 100644
index 00000000000..6c0856f60bc
--- /dev/null
+++ b/doc/publisher-profile.rst
@@ -0,0 +1,69 @@
+==============================
+Publisher Profile and Workflow
+==============================
+
+.. versionadded:: 1.6.1
+
+The Publisher workflow in CKAN is designed to support a setup in which datasets
+are managed by a "Publisher" organization. Users can become members of one (or
+more) Publishers and their membership determines what datasets they have access
+to.
+
+Specifically, the workflow looks like:
+
+* A User joins or creates a Publisher
+
+  * If creator of the Publisher User becomes administrator of the Publisher
+    otherwise they become a Member.
+  * Creation of new Publishers must be approved by a System Administrator
+  * New Members must be approved by the Group Administrator
+  
+* User creates a dataset. On creation User must assign this dataset to a
+  specific publisher (and can only assign to a Publisher of which User is a
+  member)
+
+  * Other members of that Publisher can then edit and update this dataset
+
+This setup is a natural one for many situations. For example:
+
+ * Government. Publishers correspond to Departments or Ministries (or other
+   organizational groups)
+ * Academia: Publishers again correspond to Departments or research groups
+
+
+.. _publisher-configuration:
+Enabling and Configuring the Publisher Profile
+==============================================
+
+To switch CKAN to use the publisher profile workflow you need to set the
+following configuration option::
+
+	ckan.auth.profile = publisher
+
+Setting auth.profile to publisher will enable the publisher authorization
+profile. Setting it to nothing, or if it is not present will force CKAN to use
+the default profile.
+
+
+Technical Overview
+==================
+
+* Publishers are a specialization of CKAN Groups. As such they retain many of
+  their features.
+* Authorization for most actions is granted based on shared membership of a
+  group between the **user** and the **object** being manipulated.
+* You can design custom forms for publisher sign up and editing.
+
+In more detail, these concepts are as follows:
+
+* :doc:`Domain Objects <domain-model>` such as *groups*, *datasets* and *users*
+  can be added as members of groups.
+* Each user within a group has a capacity with which it can interact with the
+  group, currently these are *editor* and *administrator*.
+* Even though groups are hierarchical there must be an intersection of the
+  user's groups and the **object**'s groups for permission to be granted, as
+  long as the capacity is appropriate.  For instance, being an *editor* within
+  a group does not necessarily grant authorization to edit the group.
+* This means that individual permissions do not need to granted on a *user* by
+  *user* basis, instead the user can just be added to the appropriate group.
+
diff --git a/doc/publisher_auth_profile.rst b/doc/publisher_auth_profile.rst
deleted file mode 100644
index ae970da01b3..00000000000
--- a/doc/publisher_auth_profile.rst
+++ /dev/null
@@ -1,36 +0,0 @@
-===============================
-Publisher Authorization Profile
-===============================
-
-CKAN provides authorization profiles which will allow authorization to be performed based on group membership rather than individually assigned actions.
-
-* :ref:`publisher-overview`. An overview of the concepts underlying CKAN's publisher profile.
-* :ref:`publisher-configuration`. Configuring the publisher profile
-* :ref:`publisher-usage`. Using the publisher profile
-
-.. _publisher-overview:
-
-Overview
---------
-
-In a nutshell: authorization for most actions is granted based on shared membership of a group between the **user** and the **object** being manipulated.
-
-In more detail, these concepts are as follows:
-
-* **Objects** such as *groups*, *datasets* and *users* can be added as members of groups.
-* Each user within a group has a capacity with which it can interact with the group, currently these are *editor* and *administrator*.
-* Even though groups are hierarchical there must be an intersection of the user's groups and the **object**'s groups for permission to be granted, as long as the capacity is appropriate.  For instance, being an *editor* within a group does not necessarily grant authorization to edit the group.
-* This means that individual permissions do not need to granted on a *user* by *user* basis, instead the user can just be added to the appropriate group.
-
-.. _publisher-configuration:
-
-Configuration
--------------
-
-To configure CKAN to use the publisher profile a single entry should be added to the CKAN configuration see  :doc:`configuration`
-
-Setting auth.profile to publisher will enable the publisher authorization profile.  Setting it to nothing, or if it is not present will force CKAN to use the default authorization profile.
-
-Example::
-	ckan.auth.profile = publisher
-