From 339479089e15cb451f8d1431fc1f31e9b90b35b6 Mon Sep 17 00:00:00 2001
From: tobes <toby.junk@gmail.com>
Date: Wed, 19 Sep 2012 10:19:16 +0100
Subject: [PATCH 01/37] Fanstatic fixes for resource upload

---
 ckan/public/base/javascript/resource.config | 2 ++
 ckan/public/base/vendor/resource.config     | 7 +++++++
 ckan/templates/package/new_resource.html    | 6 +-----
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/ckan/public/base/javascript/resource.config b/ckan/public/base/javascript/resource.config
index 3af45427c83..776a2d922c8 100644
--- a/ckan/public/base/javascript/resource.config
+++ b/ckan/public/base/javascript/resource.config
@@ -41,3 +41,5 @@ main =
     modules/autocomplete.js
     modules/custom-fields.js
     modules/related-item.js
+    modules/resource-form.js
+    modules/resource-upload-field.js
diff --git a/ckan/public/base/vendor/resource.config b/ckan/public/base/vendor/resource.config
index 9e711134c9f..363eb10fcb4 100644
--- a/ckan/public/base/vendor/resource.config
+++ b/ckan/public/base/vendor/resource.config
@@ -23,6 +23,7 @@ block_html5_shim =
 [depends]
 
 vendor = jquery.js
+fileupload = jquery.js
 
 [groups]
 
@@ -38,3 +39,9 @@ vendor =
     bootstrap/js/bootstrap-tab.js
     bootstrap/js/bootstrap-button.js
     font-awesome/css/font-awesome-ie7.css
+
+
+fileupload =
+    jquery.ui.widget.js
+    jquery-fileupload/jquery.iframe-transport.js
+    jquery-fileupload/jquery.fileupload.js
diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html
index faa93ee9ea7..52a114e39a9 100644
--- a/ckan/templates/package/new_resource.html
+++ b/ckan/templates/package/new_resource.html
@@ -23,11 +23,7 @@ <h2 class="module-heading"><i class="icon-large icon-info-sign"></i> {{ _('What
 
 {% block scripts %}
   {{ super() }}
-  <script src="{% url_for_static "/base/vendor/jquery.ui.widget.js" %}"></script>
-  <script src="{% url_for_static "/base/vendor/jquery-fileupload/jquery.iframe-transport.js" %}"></script>
-  <script src="{% url_for_static "/base/vendor/jquery-fileupload/jquery.fileupload.js" %}"></script>
-  <script src="{% url_for_static "/base/javascript/modules/resource-form.js" %}"></script>
-  <script src="{% url_for_static "/base/javascript/modules/resource-upload-field.js" %}"></script>
+  {% resource 'vendor/fileupload' %}
 {% endblock %}
 
 {% block breadcrumb_content %}

From febac8ae7bcc7659a1f191b4bfae9666211d01d6 Mon Sep 17 00:00:00 2001
From: tobes <toby.junk@gmail.com>
Date: Wed, 19 Sep 2012 10:33:51 +0100
Subject: [PATCH 02/37] Fanstatic fixes remove commented out scripts

---
 ckan/templates/package/resource_read.html     | 29 ------------
 .../snippets/recline-extra-footer.html        | 19 --------
 .../snippets/recline-extra-header.html        | 46 -------------------
 3 files changed, 94 deletions(-)
 delete mode 100644 ckan/templates/snippets/recline-extra-footer.html
 delete mode 100644 ckan/templates/snippets/recline-extra-header.html

diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html
index ecc077a4688..1436871a2d6 100644
--- a/ckan/templates/package/resource_read.html
+++ b/ckan/templates/package/resource_read.html
@@ -116,38 +116,9 @@ <h2 class="module-heading"><i class="icon-medium icon-info-sign"></i> Resource I
   {% endblock %}
 {% endblock %}
 
-{% block styles %}
-  {{ super() }}
-  {#
-  <link rel="stylesheet" href="{% url_for_static '/base/datapreview/vendor/slickgrid/2.0.1/slick.grid.css' %}" />
-  <link rel="stylesheet" href="{% url_for_static '/base/datapreview/vendor/leaflet/0.3.1/leaflet.css' %}" />
-  <!--[if lte IE 8]>
-  <link rel="stylesheet" href="{% url_for_static '/base/datapreview/vendor/leaflet/0.3.1/leaflet.ie.css' %}" />
-  <![endif]-->
-  <link rel="stylesheet" href="{% url_for_static '/base/datapreview/vendor/recline/css/recline.css' %}" />
-  <link rel="stylesheet" href="{% url_for_static '/base/datapreview/css/datapreview.css' %}" />
-  #}
-{% endblock %}
 
 {% block scripts %}
   {{ super() }}
-  {#
-  <script src="{% url_for_static '/base/datapreview/vendor/underscore/1.1.6/underscore.js' %}"></script>
-  <script src="{% url_for_static '/base/datapreview/vendor/backbone/0.5.1/backbone.js' %}"></script>
-  <script src="{% url_for_static '/base/datapreview/vendor/mustache/0.5.0-dev/mustache.js' %}"></script>
-  <script src="{% url_for_static '/base/datapreview/vendor/bootstrap/2.0.3/bootstrap.min.js' %}"></script>
-  <script src="{% url_for_static '/base/datapreview/vendor/jquery.mustache/jquery.mustache.js' %}"></script>
-
-  <script src="{% url_for_static '/base/datapreview/vendor/slickgrid/2.0.1/jquery-ui-1.8.16.custom.min.js' %}"></script>
-  <script src="{% url_for_static '/base/datapreview/vendor/slickgrid/2.0.1/jquery.event.drag-2.0.min.js' %}"></script>
-  <script src="{% url_for_static '/base/datapreview/vendor/slickgrid/2.0.1/slick.grid.min.js' %}"></script>
-  <script src="{% url_for_static '/base/datapreview/vendor/flot/0.7/jquery.flot.js' %}"></script>
-  <script src="{% url_for_static '/base/datapreview/vendor/moment/1.6.2/moment.js' %}"></script>
-  <script src="{% url_for_static '/base/datapreview/vendor/leaflet/0.3.1/leaflet.js'%}"></script>
-
-  <script src="{% url_for_static '/base/datapreview/vendor/recline/recline.js' %}"></script>
-  <script src="{% url_for_static '/base/datapreview/datapreview.js' %}"></script>
-  #}
   {% resource 'datapreview/datapreview' %}
   <script>
     var preload_resource = {{ h.literal(c.resource_json) }};
diff --git a/ckan/templates/snippets/recline-extra-footer.html b/ckan/templates/snippets/recline-extra-footer.html
deleted file mode 100644
index 88a44ecdc6a..00000000000
--- a/ckan/templates/snippets/recline-extra-footer.html
+++ /dev/null
@@ -1,19 +0,0 @@
-<html
-  xmlns="http://www.w3.org/1999/xhtml"
-  xmlns:i18n="http://genshi.edgewall.org/i18n"
-  xmlns:py="http://genshi.edgewall.org/"
-  xmlns:xi="http://www.w3.org/2001/XInclude"
-  py:strip=""
-  >
-  <script type="text/javascript" src="${h.url_for_static('/scripts/vendor/mustache/0.5.0-dev/mustache.js')}"></script>
-
-  <script type="text/javascript" src="${h.url_for_static('/scripts/vendor/slickgrid/2.0.1/jquery-ui-1.8.16.custom.min.js')}"></script>
-  <script type="text/javascript" src="${h.url_for_static('/scripts/vendor/slickgrid/2.0.1/jquery.event.drag-2.0.min.js')}"></script>
-  <script type="text/javascript" src="${h.url_for_static('/scripts/vendor/slickgrid/2.0.1/slick.grid.min.js')}"></script>
-  <script type="text/javascript" src="${h.url_for_static('/scripts/vendor/flot/0.7/jquery.flot.js')}"></script>
-  <script type="text/javascript" src="${h.url_for_static('/scripts/vendor/moment/1.6.2/moment.js')}"></script>
-  <script type="text/javascript" src="${h.url_for_static('/scripts/vendor/leaflet/0.3.1/leaflet.js')}"></script>
-
-  <script src="${h.url_for_static('/scripts/vendor/recline/recline.js')}"></script>
-
-</html>
diff --git a/ckan/templates/snippets/recline-extra-header.html b/ckan/templates/snippets/recline-extra-header.html
deleted file mode 100644
index 0bd8333a666..00000000000
--- a/ckan/templates/snippets/recline-extra-header.html
+++ /dev/null
@@ -1,46 +0,0 @@
-<html
-  xmlns="http://www.w3.org/1999/xhtml"
-  xmlns:i18n="http://genshi.edgewall.org/i18n"
-  xmlns:py="http://genshi.edgewall.org/"
-  xmlns:xi="http://www.w3.org/2001/XInclude"
-  py:strip=""
-  >
-
-  <link rel="stylesheet" href="${h.url_for_static('/scripts/vendor/slickgrid/2.0.1/slick.grid.css')}" />
-  <link rel="stylesheet" href="${h.url_for_static('/scripts/vendor/leaflet/0.3.1/leaflet.css')}" />
-  <!--[if lte IE 8]>
-  <link rel="stylesheet" href="${h.url_for_static('/scripts/vendor/leaflet/0.3.1/leaflet.ie.css')}" />
-  <![endif]-->
-  <link rel="stylesheet" href="${h.url_for_static('/scripts/vendor/recline/css/recline.css')}" />
-
-  <style type="text/css">
-    .recline-query-editor form, .recline-query-editor .text-query {
-      height: 28px;
-    }
-
-    .recline-query-editor .pagination ul {
-      margin: 0;
-      padding: 0;
-    }
-
-    .recline-slickgrid {
-      height: 550px;
-    }
-
-    .slick-header-column {
-      border-top: 0 !important;
-      border-bottom: 0 !important;
-    }
-
-    /* needed for Chrome but not FF */
-    .header .recline-query-editor .add-on {
-      margin-left: -27px;
-    }
-
-    /* needed for FF but not chrome */
-    .header .recline-query-editor .input-prepend {
-      vertical-align: top;
-    }
-  </style>
-
-</html>

From a5683cb48c12c3103d979dd4fdb5125f2f2c4a8f Mon Sep 17 00:00:00 2001
From: tobes <toby.junk@gmail.com>
Date: Wed, 19 Sep 2012 10:34:10 +0100
Subject: [PATCH 03/37] Fanstatic fixes add extra comment re script tag abuse

---
 ckan/templates/base.html | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ckan/templates/base.html b/ckan/templates/base.html
index 9073de45052..87e7a8efe13 100644
--- a/ckan/templates/base.html
+++ b/ckan/templates/base.html
@@ -103,6 +103,8 @@
     {#
     The scripts block allows you to add additonal scripts to the page. Use the
     super() function to load the default scripts before/after your own.
+    NOTE: Scripts should be loaded by the {% resource %} tag except
+    in very special circumstances DO NOT USE THIS mETHOD FOR ADDING SCRIPTS.
 
     Example:
 

From c973d36f8a920d3de5f3c568e90524bdc57e39e0 Mon Sep 17 00:00:00 2001
From: tobes <toby.junk@gmail.com>
Date: Wed, 19 Sep 2012 10:36:00 +0100
Subject: [PATCH 04/37] Fanstatic fixes fix case in comment

---
 ckan/templates/base.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckan/templates/base.html b/ckan/templates/base.html
index 87e7a8efe13..cfc63b5fc67 100644
--- a/ckan/templates/base.html
+++ b/ckan/templates/base.html
@@ -104,7 +104,7 @@
     The scripts block allows you to add additonal scripts to the page. Use the
     super() function to load the default scripts before/after your own.
     NOTE: Scripts should be loaded by the {% resource %} tag except
-    in very special circumstances DO NOT USE THIS mETHOD FOR ADDING SCRIPTS.
+    in very special circumstances DO NOT USE THIS METHOD FOR ADDING SCRIPTS.
 
     Example:
 

From 122cf345101e25ac660b724788d40a27b1c02ced Mon Sep 17 00:00:00 2001
From: tobes <toby.junk@gmail.com>
Date: Mon, 17 Sep 2012 18:05:10 +0100
Subject: [PATCH 05/37] Fix html for internal tracking

---
 ckan/templates_legacy/snippets/internal-tracking.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ckan/templates_legacy/snippets/internal-tracking.html b/ckan/templates_legacy/snippets/internal-tracking.html
index 1239850854f..6979935caa4 100644
--- a/ckan/templates_legacy/snippets/internal-tracking.html
+++ b/ckan/templates_legacy/snippets/internal-tracking.html
@@ -3,6 +3,7 @@
       xmlns:i18n="http://genshi.edgewall.org/i18n"
       xmlns:py="http://genshi.edgewall.org/"
       xmlns:xi="http://www.w3.org/2001/XInclude"
+      py:strip="True"
       >
   <script type="text/javascript">
     $(function (){

From 4b1aab062227fe8d899e6705b544c29a8c838e72 Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Tue, 4 Sep 2012 16:24:08 +0100
Subject: [PATCH 06/37] Added some small CSS helper classes for layout

---
 ckan/public/base/less/ckan.less | 38 +++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/ckan/public/base/less/ckan.less b/ckan/public/base/less/ckan.less
index 4ca1f3c3ba1..16f182b9f81 100644
--- a/ckan/public/base/less/ckan.less
+++ b/ckan/public/base/less/ckan.less
@@ -94,4 +94,42 @@ iframe {
   margin-bottom: 16px;
 }
 
+.m-top {
+  margin-top: @gutterSmallX;
+}
+.m-left {
+  margin-left: @gutterSmallX;
+}
+.m-right {
+  margin-right: @gutterSmallX;
+}
+.m-bottom {
+  margin-bottom: @gutterSmallX;
+}
+.no-margin {
+  margin: 0;
+}
+.reduced-margin {
+  margin: 3px 5px;
+}
+
+.p-top {
+  padding-top: @gutterSmallX;
+}
+.p-left {
+  padding-left: @gutterSmallX;
+}
+.p-right {
+  padding-right: @gutterSmallX;
+}
+.p-bottom {
+  padding-bottom: @gutterSmallX;
+}
+.no-padding {
+  padding: 0;
+}
+.reduced-padding {
+  padding: 3px 5px;
+}
+
 @import "iehacks.less";

From 95e76829864724a5227aeba75edf717891c7e6a2 Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Tue, 4 Sep 2012 16:34:00 +0100
Subject: [PATCH 07/37] Added tabs into the /ckan-admin header (also small
 tweaks to thecss of the page

---
 ckan/public/base/less/forms.less          |  3 +
 ckan/templates/admin/authz.html           | 79 ++++++++++++++---------
 ckan/templates/admin/index.html           |  3 +-
 ckan/templates/admin/snippets/header.html | 10 +++
 4 files changed, 61 insertions(+), 34 deletions(-)
 create mode 100644 ckan/templates/admin/snippets/header.html

diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less
index 04f72e68cfe..688d98add68 100644
--- a/ckan/public/base/less/forms.less
+++ b/ckan/public/base/less/forms.less
@@ -40,6 +40,9 @@ input[type=checkbox] {
   top: 7px;
   padding: 0;
   margin: 0;
+  &.checkbox-onown {
+    top: 0;
+  }
 }
 
 select {
diff --git a/ckan/templates/admin/authz.html b/ckan/templates/admin/authz.html
index 6348e8e0d46..361fcbc760c 100644
--- a/ckan/templates/admin/authz.html
+++ b/ckan/templates/admin/authz.html
@@ -6,9 +6,15 @@
 {% endblock %}
 
 {% macro authz_form_table(id, roles, users, user_role_dict) %}
-  <table class="table table-bordered table-striped table-condensed">
+  <table class="table table-bordered table-striped">
+    <colgroup>
+      <col width="50%">
+      {% for role in roles %}
+      <col width="15%">
+      {% endfor %}
+    </colgroup>
     <tr>
-      <th> User </th>
+      <th>{{ _('User') }}</th>
       {% for role in roles %}
       <th> {{ role }} </th>
       {% endfor %}
@@ -22,13 +28,9 @@
             <td>
               <input type="hidden" name="{{ h.literal( '%s$%s' % (user,role)) }}" value="submitted"/>
               {% if user_role_dict[(user,role)] %}
-                <input type="checkbox"
-                name="{{ h.literal( '%s$%s' % (user,role)) }}"
-                   checked='checked'/>
+                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user,role)) }}" checked='checked' class="checkbox-onown"/>
               {% else %}
-                <input type="checkbox"
-                name="{{ h.literal( '%s$%s' % (user,role)) }}"
-                       />
+                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user,role)) }}" class="checkbox-onown"/>
               {% endif %}
             </td>
         {% endfor %}
@@ -39,21 +41,27 @@
 
 
 {% macro authz_add_table(roles) %}
-  <table class="table table-bordered table-striped table-condensed">
+  <table class="table table-bordered table-striped">
+    <colgroup>
+      <col width="50%">
+      {% for role in roles %}
+      <col width="15%">
+      {% endfor %}
+    </colgroup>
     <tr>
-      <th>User</th>
+      <th>{{ _('User') }}</th>
       {% for role in roles %}
         <th> {{ role }} </th>
       {% endfor %}
     </tr>
     <tr>
-      <td>
-        <input type="textbox" name="new_user_name" class="autocomplete-user" />
+      <td class="control-full">
+        <input type="text" name="new_user_name" class="autocomplete-user no-margin reduced-padding" />
       </td>
       {% for role in roles %}
         <td>
           <input type="hidden" name="{{ h.literal( '%s' % role) }}" value="submitted"/>
-          <input type="checkbox" name="{{ h.literal( '%s' % role) }}"/>
+          <input type="checkbox" name="{{ h.literal( '%s' % role) }}" class="checkbox-onown"/>
         </td>
       {% endfor %}
     </tr>
@@ -61,9 +69,15 @@
 {% endmacro %}
 
 {% macro authz_form_group_table(id, roles, users, user_role_dict) %}
-  <table class="table table-bordered table-striped table-condensed">
+  <table class="table table-bordered table-striped">
+    <colgroup>
+      <col width="50%">
+      {% for role in roles %}
+      <col width="15%">
+      {% endfor %}
+    </colgroup>
     <tr>
-      <th>User Group</th>
+      <th>{{ _('User Group') }}</th>
       {% for role in roles %}
       <th> {{ role }} </th>
       {% endfor %}
@@ -77,13 +91,9 @@
           <td>
             <input type="hidden" name="${ h.literal( '%s$%s' % (user,role)) }" value="submitted"/>
             {% if user_role_dict[(user,role)] %}
-                <input type="checkbox"
-                name="{{ h.literal( '%s$%s' % (user,role)) }}"
-                   checked='checked'/>
+                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user,role)) }}" checked='checked' class="checkbox-onown"/>
                    {% else %}
-                <input type="checkbox"
-                name="{{ h.literal( '%s$%s' % (user,role)) }}"
-                       />
+                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user,role)) }}" />
               {% endif %}
           </td>
       {% endfor %}
@@ -94,7 +104,13 @@
 
 
 {% macro authz_add_group_table(roles) %}
-  <table class="table table-bordered table-striped table-condensed">
+  <table class="table table-bordered table-striped">
+    <colgroup>
+      <col width="50%">
+      {% for role in roles %}
+      <col width="15%">
+      {% endfor %}
+    </colgroup>
     <tr>
       <th>User Group</th>
       {% for role in roles %}
@@ -102,13 +118,13 @@
       {% endfor %}
     </tr>
     <tr>
-      <td>
-        <input type="textbox" name="new_user_name" class="autocomplete-authzgroup" />
+      <td class="control-full">
+        <input type="text" name="new_user_name" class="autocomplete-authzgroup no-margin reduced-padding" />
       </td>
       {% for role in roles %}
         <td>
           <input type="hidden" name="{{ h.literal( '%s' % role) }}" value="submitted"/>
-          <input type="checkbox" name="{{ h.literal( '%s' % role) }}"/>
+          <input type="checkbox" name="{{ h.literal( '%s' % role) }}" class="checkbox-onown"/>
         </td>
       {% endfor %}
     </tr>
@@ -118,20 +134,19 @@
 {% block primary %}
   <article class="module">
     <div class="module-content">
-      <h1 class="page-heading">{{ _('Administration - Authorization') }}</h1>
-      <h3>Update Existing Roles</h3>
+      {% snippet 'admin/snippets/header.html', action=c.action %}
+
+      <h3 class="m-bottom">{{ _('Update Existing Roles') }}</h3>
       <form id="theform" method="POST">
         {{ authz_form_table('theform', c.roles, c.users, c.user_role_dict) }}
-        <button type="submit" name="save" class="btn btn-primary">
-          Save Changes
-        </button>
+        <button type="submit" name="save" class="btn btn-primary">{{ _('Save Changes') }}</button>
         <div class="clear"></div>
       </form>
 
-      <h3>Add Roles for Any User</h3>
+      <h3 class="m-bottom">{{ _('Add Roles for Any User') }}</h3>
       <form id="addform" method="POST">
         {{ authz_add_table(c.roles) }}
-        <button type="submit" name="add" class="btn btn-primary">Add Role</button>
+        <button type="submit" name="add" class="btn btn-primary">{{ _('Add Role') }}</button>
         <div class="clear"></div>
       </form>
 
diff --git a/ckan/templates/admin/index.html b/ckan/templates/admin/index.html
index ce6ed54ce16..578c570bd9e 100644
--- a/ckan/templates/admin/index.html
+++ b/ckan/templates/admin/index.html
@@ -8,8 +8,7 @@
 {% block primary %}
   <article class="module">
     <div class="module-content">
-      <h1 class="page-heading">{{ _('Current Sysadmins') }}</h1>
-      <p>{% set url=h.url_for(controller='admin', action='authz') %}{% trans %}You can change sysadmins on the <a href="{{ url }}">authorization page</a>.{% endtrans %}</p>
+      {% snippet 'admin/snippets/header.html', action=c.action %}
       <ul>
         {% for user in c.sysadmins %}
           <li py:for="user in c.sysadmins">
diff --git a/ckan/templates/admin/snippets/header.html b/ckan/templates/admin/snippets/header.html
new file mode 100644
index 00000000000..aaf569ddda7
--- /dev/null
+++ b/ckan/templates/admin/snippets/header.html
@@ -0,0 +1,10 @@
+<h1 class="page-heading">{{ _('Administration') }}</h1>
+
+<ul class="nav nav-tabs">
+  <li{% if action == 'index' %} class="active"{% endif %}>
+    <a href="{{ h.url_for(controller='admin', action='index') }}">{{ _('Sysadmins') }}</a>
+  </li>
+  <li{% if action == 'authz' %} class="active"{% endif %}>
+  	<a href="{{ h.url_for(controller='admin', action='authz') }}">{{ _('User Roles') }}</a>
+  </li>
+</ul>

From 568dafb851d83f1cff804960a30c30553f59fd13 Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Thu, 6 Sep 2012 12:15:10 +0100
Subject: [PATCH 08/37] Added config tab to ckan-admin page

---
 ckan/templates/admin/config.html          | 2 +-
 ckan/templates/admin/snippets/header.html | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ckan/templates/admin/config.html b/ckan/templates/admin/config.html
index aee2fc37197..fa1bd6d366b 100644
--- a/ckan/templates/admin/config.html
+++ b/ckan/templates/admin/config.html
@@ -7,7 +7,7 @@
 {% block primary %}
   <article class="module">
     <div class="module-content">
-      <h1 class="page-heading">{{ _('Configuration Settings') }}</h1>
+      {% snippet 'admin/snippets/header.html', action=c.action %}
 
       <form method='post' action="" class="form-horizontal">
           {{ autoform.generate(form_items, data, errors) }}
diff --git a/ckan/templates/admin/snippets/header.html b/ckan/templates/admin/snippets/header.html
index aaf569ddda7..dfac402b7c0 100644
--- a/ckan/templates/admin/snippets/header.html
+++ b/ckan/templates/admin/snippets/header.html
@@ -7,4 +7,7 @@ <h1 class="page-heading">{{ _('Administration') }}</h1>
   <li{% if action == 'authz' %} class="active"{% endif %}>
   	<a href="{{ h.url_for(controller='admin', action='authz') }}">{{ _('User Roles') }}</a>
   </li>
+  <li{% if action == 'config' %} class="active"{% endif %}>
+    <a href="{{ h.url_for(controller='admin', action='config') }}">{{ _('Config settings') }}</a>
+  </li>
 </ul>

From a6bdf0816c49a7754affa43528a940153da3b70c Mon Sep 17 00:00:00 2001
From: tobes <toby.junk@gmail.com>
Date: Wed, 19 Sep 2012 13:19:54 +0100
Subject: [PATCH 09/37] Minor formatting

---
 ckan/templates/admin/authz.html | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ckan/templates/admin/authz.html b/ckan/templates/admin/authz.html
index 361fcbc760c..cecb18a9f75 100644
--- a/ckan/templates/admin/authz.html
+++ b/ckan/templates/admin/authz.html
@@ -26,11 +26,11 @@
         </td>
           {% for role in roles %}
             <td>
-              <input type="hidden" name="{{ h.literal( '%s$%s' % (user,role)) }}" value="submitted"/>
+              <input type="hidden" name="{{ h.literal( '%s$%s' % (user, role)) }}" value="submitted"/>
               {% if user_role_dict[(user,role)] %}
-                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user,role)) }}" checked='checked' class="checkbox-onown"/>
+                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user, role)) }}" checked='checked' class="checkbox-onown"/>
               {% else %}
-                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user,role)) }}" class="checkbox-onown"/>
+                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user, role)) }}" class="checkbox-onown"/>
               {% endif %}
             </td>
         {% endfor %}
@@ -89,11 +89,11 @@
         </td>
       {% for role in roles %}
           <td>
-            <input type="hidden" name="${ h.literal( '%s$%s' % (user,role)) }" value="submitted"/>
+            <input type="hidden" name="${ h.literal( '%s$%s' % (user, role)) }" value="submitted"/>
             {% if user_role_dict[(user,role)] %}
-                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user,role)) }}" checked='checked' class="checkbox-onown"/>
+                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user, role)) }}" checked='checked' class="checkbox-onown"/>
                    {% else %}
-                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user,role)) }}" />
+                <input type="checkbox" name="{{ h.literal( '%s$%s' % (user, role)) }}" />
               {% endif %}
           </td>
       {% endfor %}

From e38468ef655c0edf07700cecedffc65e6e1d6064 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 14 Sep 2012 15:41:48 +0100
Subject: [PATCH 10/37] format paster command

---
 ckanext/datastore/commands.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 58dc344ee71..40842f83083 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -33,6 +33,7 @@
    GRANT SELECT ON TABLES TO "{readonlyuser}";
 '''
 
+
 class SetupDatastoreCommand(CkanCommand):
     '''Perform commands to set up the datastore.
     Make sure that the datastore urls are set properly before you run these commands.
@@ -52,9 +53,9 @@ class SetupDatastoreCommand(CkanCommand):
     summary = __doc__.split('\n')[0]
     usage = __doc__
 
-    def __init__(self,name):
+    def __init__(self, name):
 
-        super(SetupDatastoreCommand,self).__init__(name)
+        super(SetupDatastoreCommand, self).__init__(name)
 
     def command(self):
         '''

From 6ea18bb3d6a6dfa76d91fe665457f0c431b6e940 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 14 Sep 2012 17:00:50 +0100
Subject: [PATCH 11/37] enable datastore to run without ckan

---
 ckanext/datastore/db.py           | 78 +++++++++++++++++--------------
 ckanext/datastore/logic/action.py |  4 +-
 2 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index 51fda65d5a1..f2b4de917f9 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -1,16 +1,26 @@
 import json
 import datetime
 import shlex
+import os
 import logging
+import pprint
 import sqlalchemy
 from sqlalchemy.exc import ProgrammingError, IntegrityError
-from sqlalchemy import text
 import psycopg2.extras
-from paste.deploy.converters import asbool, aslist
-import ckan.plugins as p
 
 log = logging.getLogger(__name__)
 
+if not os.environ.get('DATASTORE_LOAD'):
+    from paste.deploy.converters import asbool, aslist
+    from ckan.plugins import toolkit
+    ValidationError = toolkit.ValidationError
+else:
+    log.warn("Running datastore without CKAN")
+
+    class ValidationError(Exception):
+        def __init__(self, error_dict):
+            pprint.pprint(error_dict)
+
 _pg_types = {}
 _type_names = set()
 _engines = {}
@@ -75,7 +85,7 @@ def _validate_int(i, field_name):
     try:
         int(i)
     except ValueError:
-        raise p.toolkit.ValidationError({
+        raise ValidationError({
             'field_name': ['{0} is not an integer'.format(i)]
         })
 
@@ -208,11 +218,11 @@ def check_fields(context, fields):
     'Check if field types are valid.'
     for field in fields:
         if field.get('type') and not field['type'] in _type_names:
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'fields': ['{0} is not a valid field type'.format(field['type'])]
             })
         elif not _is_valid_field_name(field['id']):
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'fields': ['{0} is not a valid field name'.format(field['id'])]
             })
 
@@ -248,7 +258,7 @@ def create_table(context, data_dict):
     for field in supplied_fields:
         if 'type' not in field:
             if not records or field['id'] not in records[0]:
-                raise p.toolkit.ValidationError({
+                raise ValidationError({
                     'fields': ['{0} type not guessable'.format(field['id'])]
                 })
             field['type'] = _guess_type(records[0][field['id']])
@@ -256,7 +266,7 @@ def create_table(context, data_dict):
     if records:
         # check record for sanity
         if not isinstance(records[0], dict):
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'records': ['The first row is not a json object']
             })
         supplied_field_ids = records[0].keys()
@@ -281,7 +291,7 @@ def create_table(context, data_dict):
 
 def _get_aliases(context, data_dict):
     res_id = data_dict['resource_id']
-    alias_sql = text(u'SELECT name FROM "_table_metadata" WHERE alias_of = :id')
+    alias_sql = sqlalchemy.text(u'SELECT name FROM "_table_metadata" WHERE alias_of = :id')
     results = context['connection'].execute(alias_sql, id=res_id).fetchall()
     return [x[0] for x in results]
 
@@ -327,7 +337,7 @@ def create_indexes(context, data_dict):
             fields = _get_list(index)
             for field in fields:
                 if field not in field_ids:
-                    raise p.toolkit.ValidationError({
+                    raise ValidationError({
                         'index': [('The field {0} is not a valid column name.').format(
                             index)]
                     })
@@ -346,7 +356,7 @@ def create_indexes(context, data_dict):
         # create unique index
         for field in primary_key:
             if field not in field_ids:
-                raise p.toolkit.ValidationError({
+                raise ValidationError({
                     'primary_key': [('The field {0} is not a valid column name.').format(
                         field)]
                 })
@@ -398,7 +408,7 @@ def alter_table(context, data_dict):
         # extension of current fields
         if num < len(current_fields):
             if field['id'] != current_fields[num]['id']:
-                raise p.toolkit.ValidationError({
+                raise ValidationError({
                     'fields': [('Supplied field "{0}" not '
                               'present or in wrong order').format(field['id'])]
                 })
@@ -407,7 +417,7 @@ def alter_table(context, data_dict):
 
         if 'type' not in field:
             if not records or field['id'] not in records[0]:
-                raise p.toolkit.ValidationError({
+                raise ValidationError({
                     'fields': ['{0} type not guessable'.format(field['id'])]
                 })
             field['type'] = _guess_type(records[0][field['id']])
@@ -416,7 +426,7 @@ def alter_table(context, data_dict):
     if records:
         # check record for sanity
         if not isinstance(records[0], dict):
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'records': ['The first row is not a json object']
             })
         supplied_field_ids = records[0].keys()
@@ -448,7 +458,7 @@ def upsert_data(context, data_dict):
     method = data_dict.get('method', UPSERT)
 
     if method not in _methods:
-        raise p.toolkit.ValidationError({
+        raise ValidationError({
             'method': [u'{0} is not defined'.format(method)]
         })
 
@@ -484,7 +494,7 @@ def upsert_data(context, data_dict):
     elif method in [UPDATE, UPSERT]:
         unique_keys = _get_unique_key(context, data_dict)
         if len(unique_keys) < 1:
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'table': [u'table does not have a unique key defined']
             })
 
@@ -493,7 +503,7 @@ def upsert_data(context, data_dict):
             missing_fields = [field for field in unique_keys
                     if field not in record]
             if missing_fields:
-                raise p.toolkit.ValidationError({
+                raise ValidationError({
                     'key': [u'fields "{0}" are missing but needed as key'.format(
                         ', '.join(missing_fields))]
                 })
@@ -506,7 +516,7 @@ def upsert_data(context, data_dict):
             non_existing_filed_names = [field for field in used_field_names
                 if field not in field_names]
             if non_existing_filed_names:
-                raise p.toolkit.ValidationError({
+                raise ValidationError({
                     'fields': [u'fields "{0}" do not exist'.format(
                         ', '.join(missing_fields))]
                 })
@@ -528,7 +538,7 @@ def upsert_data(context, data_dict):
 
                 # validate that exactly one row has been updated
                 if results.rowcount != 1:
-                    raise p.toolkit.ValidationError({
+                    raise ValidationError({
                         'key': [u'key "{0}" not found'.format(unique_values)]
                     })
 
@@ -576,14 +586,14 @@ def _get_unique_key(context, data_dict):
 def _validate_record(record, num, field_names):
     # check record for sanity
     if not isinstance(record, dict):
-        raise p.toolkit.ValidationError({
+        raise ValidationError({
             'records': [u'row {0} is not a json object'.format(num)]
         })
     ## check for extra fields in data
     extra_keys = set(record.keys()) - set(field_names)
 
     if extra_keys:
-        raise p.toolkit.ValidationError({
+        raise ValidationError({
             'records': [u'row {0} has extra keys "{1}"'.format(
                 num + 1,
                 ', '.join(list(extra_keys))
@@ -607,7 +617,7 @@ def _where(field_ids, data_dict):
     filters = data_dict.get('filters', {})
 
     if not isinstance(filters, dict):
-        raise p.toolkit.ValidationError({
+        raise ValidationError({
             'filters': ['Not a json object']}
         )
 
@@ -616,7 +626,7 @@ def _where(field_ids, data_dict):
 
     for field, value in filters.iteritems():
         if field not in field_ids:
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'filters': ['field "{0}" not in table'.format(field)]}
             )
         where_clauses.append(u'"{0}" = %s'.format(field))
@@ -666,18 +676,18 @@ def _sort(context, data_dict, field_ids):
         elif len(clause_parts) == 2:
             field, sort = clause_parts
         else:
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'sort': ['not valid syntax for sort clause']
             })
         field, sort = unicode(field, 'utf-8'), unicode(sort, 'utf-8')
 
         if field not in field_ids:
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'sort': [u'field {0} not it table'.format(
                     unicode(field, 'utf-8'))]
             })
         if sort.lower() not in ('asc', 'desc'):
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'sort': ['sorting can only be asc or desc']
             })
         clause_parsed.append(u'"{0}" {1}'.format(
@@ -714,7 +724,7 @@ def search_data(context, data_dict):
 
         for field in field_ids:
             if not field in all_field_ids:
-                raise p.toolkit.ValidationError({
+                raise ValidationError({
                     'fields': [u'field "{0}" not in table'.format(field)]}
                 )
     else:
@@ -823,7 +833,7 @@ def create(context, data_dict):
     except IntegrityError, e:
         if ('duplicate key value violates unique constraint' in str(e)
                 or 'could not create unique index' in str(e)):
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'constraints': ['Cannot insert records because of uniqueness constraint'],
                 'info': {
                     'details': str(e)
@@ -833,7 +843,7 @@ def create(context, data_dict):
             raise
     except Exception, e:
         if 'due to statement timeout' in str(e):
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'query': ['Query took too long']
             })
         raise
@@ -872,7 +882,7 @@ def delete(context, data_dict):
              data_dict['resource_id']
         ).fetchone()
         if not result:
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'resource_id': [u'table for resource {0} does not exist'.format(
                     data_dict['resource_id'])]
             })
@@ -908,14 +918,14 @@ def search(context, data_dict):
              u"(SELECT 1 FROM pg_views where viewname = '{0}')".format(id)
         ).fetchone()
         if not result:
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'resource_id': [u'table for resource {0} does not exist'.format(
                     data_dict['resource_id'])]
             })
         return search_data(context, data_dict)
     except Exception, e:
         if 'due to statement timeout' in str(e):
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'query': ['Search took too long']
             })
         raise
@@ -938,7 +948,7 @@ def search_sql(context, data_dict):
         return format_results(context, results, data_dict)
 
     except ProgrammingError, e:
-        raise p.toolkit.ValidationError({
+        raise ValidationError({
          'query': [str(e)],
          'info': {
             'statement': [e.statement],
@@ -948,7 +958,7 @@ def search_sql(context, data_dict):
         })
     except Exception, e:
         if 'due to statement timeout' in str(e):
-            raise p.toolkit.ValidationError({
+            raise ValidationError({
                 'query': ['Search took too long']
             })
         raise
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 54b40b02a83..734d163d4c1 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -3,7 +3,7 @@
 import ckan.logic as logic
 import ckan.plugins as p
 import ckanext.datastore.db as db
-from sqlalchemy import text
+import sqlalchemy
 
 log = logging.getLogger(__name__)
 _get_or_bust = logic.get_or_bust
@@ -162,7 +162,7 @@ def datastore_search(context, data_dict):
 
     data_dict['connection_url'] = pylons.config['ckan.datastore_read_url']
 
-    resources_sql = text(u'SELECT 1 FROM "_table_metadata" WHERE name = :id')
+    resources_sql = sqlalchemy.text(u'SELECT 1 FROM "_table_metadata" WHERE name = :id')
     results = db._get_engine(None, data_dict).execute(resources_sql, id=res_id)
     res_exists = results.rowcount > 0
 

From 295410fa9afc3a2f802b9566be9aa83c233c8a48 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Mon, 17 Sep 2012 16:40:02 +0100
Subject: [PATCH 12/37] Fix problems with % in column names and table names.

---
 ckanext/datastore/db.py                   | 17 ++++++----
 ckanext/datastore/tests/test_datastore.py | 39 +++++++++++++++--------
 2 files changed, 36 insertions(+), 20 deletions(-)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index f2b4de917f9..bebda375346 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -78,7 +78,10 @@ def _is_valid_field_name(name):
     return True
 
 
-_is_valid_table_name = _is_valid_field_name
+def _is_valid_table_name(name):
+    if '%' in name:
+        return False
+    return _is_valid_field_name(name)
 
 
 def _validate_int(i, field_name):
@@ -278,7 +281,7 @@ def create_table(context, data_dict):
                 })
 
     fields = datastore_fields + supplied_fields + extra_fields
-    sql_fields = u", ".join([u'"{0}" {1}'.format(f['id'], f['type'])
+    sql_fields = u", ".join([u'"{0}" {1}'.format(f['id'].replace('%', '%%'), f['type'])
                             for f in fields])
 
     sql_string = u'CREATE TABLE "{0}" ({1});'.format(
@@ -341,7 +344,7 @@ def create_indexes(context, data_dict):
                         'index': [('The field {0} is not a valid column name.').format(
                             index)]
                     })
-            fields_string = u','.join(['"%s"' % field for field in fields])
+            fields_string = u','.join(['"%s"' % field.replace('%', '%%') for field in fields])
             sql_index_strings.append(sql_index_string.format(
                 res_id=data_dict['resource_id'], unique='',
                 fields=fields_string))
@@ -363,7 +366,7 @@ def create_indexes(context, data_dict):
         if primary_key:
             sql_index_strings.append(sql_index_string.format(
                 res_id=data_dict['resource_id'], unique='unique',
-                fields=u','.join(['"%s"' % field for field in primary_key])))
+                fields=u','.join(['"%s"' % field.replace('%', '%%') for field in primary_key])))
 
     map(context['connection'].execute, sql_index_strings)
 
@@ -388,7 +391,7 @@ def _drop_indexes(context, data_dict, unique=False):
     indexes_to_drop = context['connection'].execute(
         sql_get_index_string, data_dict['resource_id']).fetchall()
     for index in indexes_to_drop:
-        context['connection'].execute(sql_drop_index.format(index[0]))
+        context['connection'].execute(sql_drop_index.format(index[0]).replace('%', '%%'))
 
 
 def alter_table(context, data_dict):
@@ -440,7 +443,7 @@ def alter_table(context, data_dict):
     for field in new_fields:
         sql = 'ALTER TABLE "{0}" ADD "{1}" {2}'.format(
             data_dict['resource_id'],
-            field['id'],
+            field['id'].replace('%', '%%'),
             field['type'])
         context['connection'].execute(sql)
 
@@ -465,7 +468,7 @@ def upsert_data(context, data_dict):
     fields = _get_fields(context, data_dict)
     field_names = _pluck('id', fields)
     records = data_dict['records']
-    sql_columns = ", ".join(['"%s"' % name for name in field_names]
+    sql_columns = ", ".join(['"%s"' % name.replace('%', '%%') for name in field_names]
                             + ['"_full_text"'])
 
     if method == INSERT:
diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py
index 50026881d79..346a7c44280 100644
--- a/ckanext/datastore/tests/test_datastore.py
+++ b/ckanext/datastore/tests/test_datastore.py
@@ -80,7 +80,20 @@ def test_create_invalid_alias_name(self):
         resource = model.Package.get('annakarenina').resources[0]
         data = {
             'resource_id': resource.id,
-            'aliases': 'foo"bar',
+            'aliases': u'foo"bar',
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+        data = {
+            'resource_id': resource.id,
+            'aliases': u'fo%25bar',
             'fields': [{'id': 'book', 'type': 'text'},
                        {'id': 'author', 'type': 'text'}]
         }
@@ -215,13 +228,13 @@ def test_create_basic(self):
         data = {
             'resource_id': resource.id,
             'aliases': aliases,
-            'fields': [{'id': 'book', 'type': 'text'},
+            'fields': [{'id': 'boo%k', 'type': 'text'},
                        {'id': 'author', 'type': 'json'}],
-            'indexes': [['book', 'author'], 'book'],
+            'indexes': [['boo%k', 'author'], 'boo%k'],
             'records': [
-                        {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']},
-                        {'book': 'annakarenina', 'author': ['tolstoy', 'putin']},
-                        {'book': 'warandpeace'}]  # treat author as null
+                        {'boo%k': 'crime', 'author': ['tolstoy', 'dostoevsky']},
+                        {'boo%k': 'annakarenina', 'author': ['tolstoy', 'putin']},
+                        {'boo%k': 'warandpeace'}]  # treat author as null
         }
         ### Firstly test to see if resource things it has datastore table
         postparams = '%s=1' % json.dumps({'id': resource.id})
@@ -248,7 +261,7 @@ def test_create_basic(self):
 
         assert results.rowcount == 3
         for i, row in enumerate(results):
-            assert data['records'][i].get('book') == row['book']
+            assert data['records'][i].get('boo%k') == row['boo%k']
             assert data['records'][i].get('author') == (
                 json.loads(row['author'][0]) if row['author'] else None)
 
@@ -288,7 +301,7 @@ def test_create_basic(self):
         #######  insert again simple
         data2 = {
             'resource_id': resource.id,
-            'records': [{'book': 'hagji murat', 'author': ['tolstoy']}]
+            'records': [{'boo%k': 'hagji murat', 'author': ['tolstoy']}]
         }
 
         postparams = '%s=1' % json.dumps(data2)
@@ -306,7 +319,7 @@ def test_create_basic(self):
 
         all_data = data['records'] + data2['records']
         for i, row in enumerate(results):
-            assert all_data[i].get('book') == row['book']
+            assert all_data[i].get('boo%k') == row['boo%k']
             assert all_data[i].get('author') == (
                 json.loads(row['author'][0]) if row['author'] else None)
 
@@ -319,7 +332,7 @@ def test_create_basic(self):
         #######  insert again extra field
         data3 = {
             'resource_id': resource.id,
-            'records': [{'book': 'crime and punsihment',
+            'records': [{'boo%k': 'crime and punsihment',
                          'author': ['dostoevsky'], 'rating': 'good'}],
             'indexes': ['rating']
         }
@@ -339,7 +352,7 @@ def test_create_basic(self):
 
         all_data = data['records'] + data2['records'] + data3['records']
         for i, row in enumerate(results):
-            assert all_data[i].get('book') == row['book'], (i, all_data[i].get('book'), row['book'])
+            assert all_data[i].get('boo%k') == row['boo%k'], (i, all_data[i].get('boo%k'), row['boo%k'])
             assert all_data[i].get('author') == (json.loads(row['author'][0]) if row['author'] else None)
 
         results = c.execute('''select * from "{0}" where _full_text @@ to_tsquery('dostoevsky') '''.format(resource.id))
@@ -350,8 +363,8 @@ def test_create_basic(self):
         #######  insert again which will fail because of unique book name
         data4 = {
             'resource_id': resource.id,
-            'records': [{'book': 'warandpeace'}],
-            'primary_key': 'book'
+            'records': [{'boo%k': 'warandpeace'}],
+            'primary_key': 'boo%k'
         }
 
         postparams = '%s=1' % json.dumps(data4)

From a61c497c0484b47f7d205d0cbbfb811b2eb99c98 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Mon, 17 Sep 2012 16:43:06 +0100
Subject: [PATCH 13/37] test % in records

---
 ckanext/datastore/tests/test_datastore.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py
index 346a7c44280..263baa17610 100644
--- a/ckanext/datastore/tests/test_datastore.py
+++ b/ckanext/datastore/tests/test_datastore.py
@@ -666,6 +666,21 @@ def test_upsert_basic(self):
         assert records[3].author == 'tolkien'
         model.Session.remove()
 
+        # test % in records
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'upsert',
+            'records': [{'author': 'tol % kien', u'b\xfck': 'the % hobbit'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
     def test_upsert_missing_key(self):
         data = {
             'resource_id': self.data['resource_id'],

From 4b252aba55ac9bea2232d8f36b2094b53b7ad334 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Mon, 17 Sep 2012 17:58:02 +0100
Subject: [PATCH 14/37] from .. import .. -> import .. as ..

---
 ckanext/datastore/db.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index bebda375346..8fcb0e9a2c0 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -11,8 +11,8 @@
 log = logging.getLogger(__name__)
 
 if not os.environ.get('DATASTORE_LOAD'):
-    from paste.deploy.converters import asbool, aslist
-    from ckan.plugins import toolkit
+    import paste.deploy.converters as converters
+    import ckan.plugins.toolkit as toolkit
     ValidationError = toolkit.ValidationError
 else:
     log.warn("Running datastore without CKAN")
@@ -54,7 +54,7 @@ def _get_list(input, strip=True):
     if input == '':
         return []
 
-    l = aslist(input, ',', True)
+    l = converters.aslist(input, ',', True)
     if strip:
         return [_strip(x) for x in l]
     else:
@@ -64,7 +64,7 @@ def _get_list(input, strip=True):
 def _get_bool(input, default=False):
     if input in [None, '']:
         return default
-    return asbool(input)
+    return converters.asbool(input)
 
 
 def _is_valid_field_name(name):

From 94c27f9352e1de450e9249ddde18e457da7b1828 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Mon, 17 Sep 2012 22:43:18 +0100
Subject: [PATCH 15/37] Fix index creation on nested fields in postgres 9.2.

---
 ckanext/datastore/db.py                   | 51 ++++++++++++-----------
 ckanext/datastore/tests/test_datastore.py |  2 +-
 2 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index 8fcb0e9a2c0..116d60c0310 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -331,42 +331,45 @@ def create_indexes(context, data_dict):
     sql_index_string_method = sql_index_skeletton + u' USING {method}({fields})'
     sql_index_string = sql_index_skeletton + u' ({fields})'
     sql_index_strings = []
-    field_ids = _pluck('id', _get_fields(context, data_dict))
+
+    fields = _get_fields(context, data_dict)
+    field_ids = _pluck('id', fields)
+    json_fields = [x['id'] for x in fields if x['type'] == 'nested']
 
     if indexes != None:
         _drop_indexes(context, data_dict, False)
 
-        for index in indexes:
-            fields = _get_list(index)
-            for field in fields:
-                if field not in field_ids:
-                    raise ValidationError({
-                        'index': [('The field {0} is not a valid column name.').format(
-                            index)]
-                    })
-            fields_string = u','.join(['"%s"' % field.replace('%', '%%') for field in fields])
-            sql_index_strings.append(sql_index_string.format(
-                res_id=data_dict['resource_id'], unique='',
-                fields=fields_string))
-
         # create index for faster full text search (indexes: gin or gist)
         sql_index_strings.append(sql_index_string_method.format(
-                res_id=data_dict['resource_id'], unique='',
-                method='gist', fields='_full_text'))
+            res_id=data_dict['resource_id'], unique='',
+            method='gist', fields='_full_text'))
+    else:
+        indexes = []
+
     if primary_key != None:
         _drop_indexes(context, data_dict, True)
+        indexes.append(primary_key)
 
-        # create unique index
-        for field in primary_key:
+    for index in indexes:
+        if not index:
+            continue
+
+        index_fields = _get_list(index)
+        for field in index_fields:
             if field not in field_ids:
                 raise ValidationError({
-                    'primary_key': [('The field {0} is not a valid column name.').format(
-                        field)]
+                    'index': [('The field {0} is not a valid column name.').format(
+                        index)]
                 })
-        if primary_key:
-            sql_index_strings.append(sql_index_string.format(
-                res_id=data_dict['resource_id'], unique='unique',
-                fields=u','.join(['"%s"' % field.replace('%', '%%') for field in primary_key])))
+        fields_string = u', '.join([
+                '(("{0}").json::text)'.format(field.replace('%', '%%'))
+            if field in json_fields else
+                 '"%s"' % field.replace('%', '%%')
+            for field in index_fields])
+        sql_index_strings.append(sql_index_string.format(
+                res_id=data_dict['resource_id'],
+                unique='unique' if index == primary_key else '',
+                fields=fields_string))
 
     map(context['connection'].execute, sql_index_strings)
 
diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py
index 263baa17610..44b68166e28 100644
--- a/ckanext/datastore/tests/test_datastore.py
+++ b/ckanext/datastore/tests/test_datastore.py
@@ -230,7 +230,7 @@ def test_create_basic(self):
             'aliases': aliases,
             'fields': [{'id': 'boo%k', 'type': 'text'},
                        {'id': 'author', 'type': 'json'}],
-            'indexes': [['boo%k', 'author'], 'boo%k'],
+            'indexes': [['boo%k', 'author'], 'author'],
             'records': [
                         {'boo%k': 'crime', 'author': ['tolstoy', 'dostoevsky']},
                         {'boo%k': 'annakarenina', 'author': ['tolstoy', 'putin']},

From a586adf856c947a96fdadfe225270d82e3bc0d3d Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 19 Sep 2012 13:48:19 +0100
Subject: [PATCH 16/37] renamed datastore urls and removed
 ckan.datastore.enabled = 1

---
 ckan/config/deployment.ini_tmpl   |  9 +++++----
 ckanext/datastore/commands.py     |  4 ++--
 ckanext/datastore/logic/action.py | 10 +++++-----
 ckanext/datastore/plugin.py       | 18 +++++++++---------
 doc/datastore.rst                 |  6 +++---
 5 files changed, 24 insertions(+), 23 deletions(-)

diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl
index fda532ba54e..04df5972377 100644
--- a/ckan/config/deployment.ini_tmpl
+++ b/ckan/config/deployment.ini_tmpl
@@ -44,6 +44,11 @@ sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckantest
 #sqlalchemy.url = sqlite:///
 #sqlalchemy.url = sqlite:///%(here)s/somedb.db
 
+## Datastore
+## Uncommment to set the datastore urls
+#ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/ckantest
+#ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/ckantest
+
 # repoze.who config
 who.config_file = %(here)s/who.ini
 who.log_level = warning
@@ -226,10 +231,6 @@ ckan.feeds.author_link =
 #ofs.aws_access_key_id = ....
 #ofs.aws_secret_access_key = ....
 
-## Webstore
-## Uncommment to enable datastore
-# ckan.datastore.enabled = 1
-
 ## ===================================
 ## Extensions
 
diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 40842f83083..e078be1c6f1 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -68,8 +68,8 @@ def command(self):
         cmd = self.args[0]
         self._load_config()
 
-        self.urlparts_w = self._get_db_config('ckan.datastore_write_url')
-        self.urlparts_r = self._get_db_config('ckan.datastore_read_url')
+        self.urlparts_w = self._get_db_config('ckan.datastore.write_url')
+        self.urlparts_r = self._get_db_config('ckan.datastore.read_url')
         self.urlparts_c = self._get_db_config('sqlalchemy.url')
 
         assert self.urlparts_w['db_name'] == self.urlparts_r['db_name'], "write and read db should be the same"
diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 734d163d4c1..6a5da311f40 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -39,7 +39,7 @@ def datastore_create(context, data_dict):
 
     p.toolkit.check_access('datastore_create', context, data_dict)
 
-    data_dict['connection_url'] = pylons.config['ckan.datastore_write_url']
+    data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
 
     # validate aliases
     aliases = db._get_list(data_dict.get('aliases', []))
@@ -84,7 +84,7 @@ def datastore_upsert(context, data_dict):
 
     p.toolkit.check_access('datastore_upsert', context, data_dict)
 
-    data_dict['connection_url'] = pylons.config['ckan.datastore_write_url']
+    data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
 
     result = db.upsert(context, data_dict)
     result.pop('id')
@@ -114,7 +114,7 @@ def datastore_delete(context, data_dict):
 
     p.toolkit.check_access('datastore_delete', context, data_dict)
 
-    data_dict['connection_url'] = pylons.config['ckan.datastore_write_url']
+    data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
 
     result = db.delete(context, data_dict)
     result.pop('id')
@@ -160,7 +160,7 @@ def datastore_search(context, data_dict):
     '''
     res_id = _get_or_bust(data_dict, 'resource_id')
 
-    data_dict['connection_url'] = pylons.config['ckan.datastore_read_url']
+    data_dict['connection_url'] = pylons.config['ckan.datastore.read_url']
 
     resources_sql = sqlalchemy.text(u'SELECT 1 FROM "_table_metadata" WHERE name = :id')
     results = db._get_engine(None, data_dict).execute(resources_sql, id=res_id)
@@ -203,7 +203,7 @@ def datastore_search_sql(context, data_dict):
 
     p.toolkit.check_access('datastore_search', context, data_dict)
 
-    data_dict['connection_url'] = pylons.config['ckan.datastore_read_url']
+    data_dict['connection_url'] = pylons.config['ckan.datastore.read_url']
 
     result = db.search_sql(context, data_dict)
     result.pop('id', None)
diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
index a0a71bbc401..ce78381367f 100644
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -25,13 +25,13 @@ class DatastorePlugin(p.SingletonPlugin):
 
     def configure(self, config):
         self.config = config
-        # check for ckan.datastore_write_url and ckan.datastore_read_url
-        if (not 'ckan.datastore_write_url' in config):
-            error_msg = 'ckan.datastore_write_url not found in config'
+        # check for ckan.datastore.write_url and ckan.datastore.read_url
+        if (not 'ckan.datastore.write_url' in config):
+            error_msg = 'ckan.datastore.write_url not found in config'
             raise DatastoreException(error_msg)
 
-        if (not 'ckan.datastore_read_url' in config):
-            error_msg = 'ckan.datastore_read_url not found in config'
+        if (not 'ckan.datastore.read_url' in config):
+            error_msg = 'ckan.datastore.read_url not found in config'
             raise DatastoreException(error_msg)
 
         # Check whether we are running one of the paster commands which means
@@ -43,8 +43,8 @@ def configure(self, config):
             return
 
         self.ckan_url = self.config['sqlalchemy.url']
-        self.write_url = self.config['ckan.datastore_write_url']
-        self.read_url = self.config['ckan.datastore_read_url']
+        self.write_url = self.config['ckan.datastore.write_url']
+        self.read_url = self.config['ckan.datastore.read_url']
 
         if not self._is_read_only_database():
             # Make sure that the right permissions are set
@@ -69,7 +69,7 @@ def configure(self, config):
         def new_resource_show(context, data_dict):
             engine = db._get_engine(
                 context,
-                {'connection_url': config['ckan.datastore_read_url']}
+                {'connection_url': config['ckan.datastore.read_url']}
             )
             new_data_dict = resource_show(context, data_dict)
             try:
@@ -185,7 +185,7 @@ def _create_alias_table(self):
         '''
         create_alias_table_sql = u'CREATE OR REPLACE VIEW "_table_metadata" AS {0}'.format(mapping_sql)
         connection = db._get_engine(None,
-            {'connection_url': pylons.config['ckan.datastore_write_url']}).connect()
+            {'connection_url': pylons.config['ckan.datastore.write_url']}).connect()
         connection.execute(create_alias_table_sql)
 
     def get_actions(self):
diff --git a/doc/datastore.rst b/doc/datastore.rst
index c946b3d113b..3b10ea17b57 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -44,10 +44,10 @@ In your config file ensure that the datastore extension is enabled::
 
  ckan.plugins = datastore
 
-Also ensure that the ``ckan.datastore_write_url`` and ``datastore_read_url`` variables are set::
+Also ensure that the ``ckan.datastore.write_url`` and ``datastore.read_url`` variables are set::
 
- ckan.datastore_write_url = postgresql://ckanuser:pass@localhost/datastore
- ckan.datastore_read_url = postgresql://readonlyuser:pass@localhost/datastore
+ ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/datastore
+ ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/datastore
 
 A few things have to be kept in mind
 

From e97b5c70e9325a1095778fef8e0d27c84e881b30 Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Thu, 20 Sep 2012 18:02:43 +0200
Subject: [PATCH 17/37] [#2937] Add extras_as_string to group history()

---
 ckan/controllers/group.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index c40979bbabb..5a1dcdb7aa8 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -439,7 +439,8 @@ def history(self, id):
 
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author,
-                   'schema': self._form_to_db_schema()}
+                   'schema': self._form_to_db_schema(),
+                   'extras_as_string': True}
         data_dict = {'id': id}
         try:
             c.group_dict = get_action('group_show')(context, data_dict)

From bc389d35ecd741cb880486301e1e64124935f5aa Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 25 Sep 2012 13:23:55 +0100
Subject: [PATCH 18/37] Updated documentation on query languages

---
 doc/datastore.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/datastore.rst b/doc/datastore.rst
index 3b10ea17b57..ae12e037fdd 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -181,8 +181,9 @@ The datastore supports querying with the datastore_search and datastore_search_s
 ..                              datastore_search         datastore_search_sql
 ..                                                       SQL                    HTSQL
 ==============================  =======================  =====================  ======================
-**Status**                      Stable                   Stable                 In development
+**Status**                      Stable                   Stable                 Will be available as plugin
 **Ease of use**                 Easy                     Complex                Medium
+**Flexibility**                 Low                      High                   Medium
 **Query language**              Custom (JSON)            SQL                    HTSQL
 **Connect multiple resources**  No                       Yes                    Yes
 **Use aliases**                 Yes                      Yes                    Yes

From f842804e814b817977b18070a05791e0b7ed6bc1 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 26 Sep 2012 10:06:19 +0100
Subject: [PATCH 19/37] Set the password for the read only user

---
 ckanext/datastore/commands.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index e078be1c6f1..7fd10516d6d 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -4,7 +4,7 @@
 import logging
 log = logging.getLogger(__name__)
 
-read_only_user_sql = '''
+read_only_user_sql = """
 -- revoke permissions for the new user
 REVOKE CREATE ON SCHEMA public FROM PUBLIC;
 REVOKE USAGE ON SCHEMA public FROM PUBLIC;
@@ -16,7 +16,7 @@
 GRANT USAGE ON SCHEMA public TO "{writeuser}";
 
 -- create new read only user
-CREATE USER "{readonlyuser}" NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN;
+CREATE USER "{readonlyuser}" WITH PASSWORD '{password}' NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN;
 
 -- take connect permissions from main db
 REVOKE CONNECT ON DATABASE "{maindb}" FROM "{readonlyuser}";
@@ -31,7 +31,7 @@
 -- grant access to new tables and views by default
 ALTER DEFAULT PRIVILEGES FOR USER "{ckanuser}" IN SCHEMA public
    GRANT SELECT ON TABLES TO "{readonlyuser}";
-'''
+"""
 
 
 class SetupDatastoreCommand(CkanCommand):
@@ -115,11 +115,11 @@ def _run_sql(self, sql, as_sql_user, database='postgres'):
         if self.verbose:
             print "Executing: \n#####\n", sql, "\n####\nOn database:", database
         if not self.simulate:
-            self._run_cmd("psql --username='{username}' --dbname='{database}' -c '{sql}' -W".format(
+            self._run_cmd("psql --username='{username}' --dbname='{database}' -c \"{sql}\" -W".format(
                 username=as_sql_user,
                 database=database,
-                sql=sql
-            )) 
+                sql=sql.replace('"', '\\"')
+            ))
 
     def create_db(self):
         sql = "create database {0}".format(self.urlparts_w['db_name'])
@@ -131,8 +131,8 @@ def create_read_only_user(self):
             datastore=self.urlparts_w['db_name'],
             ckanuser=self.urlparts_c['db_user'],
             readonlyuser=self.urlparts_r['db_user'],
+            password=self.urlparts_r['db_pass'],
             writeuser=self.urlparts_w['db_user'])
         self._run_sql(sql,
                       as_sql_user=self.sql_superuser,
                       database=self.urlparts_w['db_name'])
-

From aafed9c7d078aed3a04bbb06071e81084de48be9 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 26 Sep 2012 10:10:14 +0100
Subject: [PATCH 20/37] enable create user without password (empty = none)

---
 ckanext/datastore/commands.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 7fd10516d6d..724dcaac8d0 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -16,7 +16,7 @@
 GRANT USAGE ON SCHEMA public TO "{writeuser}";
 
 -- create new read only user
-CREATE USER "{readonlyuser}" WITH PASSWORD '{password}' NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN;
+CREATE USER "{readonlyuser}" {with_password} NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN;
 
 -- take connect permissions from main db
 REVOKE CONNECT ON DATABASE "{maindb}" FROM "{readonlyuser}";
@@ -126,12 +126,13 @@ def create_db(self):
         self._run_sql(sql, as_sql_user=self.sql_superuser)
 
     def create_read_only_user(self):
+        password = self.urlparts_r['db_pass']
         sql = read_only_user_sql.format(
             maindb=self.urlparts_c['db_name'],
             datastore=self.urlparts_w['db_name'],
             ckanuser=self.urlparts_c['db_user'],
             readonlyuser=self.urlparts_r['db_user'],
-            password=self.urlparts_r['db_pass'],
+            with_password="WITH PASSWORD '{0}'".format(password) if password else "",
             writeuser=self.urlparts_w['db_user'])
         self._run_sql(sql,
                       as_sql_user=self.sql_superuser,

From cad3ec089e73db162728c6498fe5536e7e9d2019 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 26 Sep 2012 10:30:10 +0100
Subject: [PATCH 21/37] Change quoting and use more descriptive names

---
 ckanext/datastore/commands.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 724dcaac8d0..9fbb504a55b 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -4,7 +4,7 @@
 import logging
 log = logging.getLogger(__name__)
 
-read_only_user_sql = """
+read_only_user_sql = '''
 -- revoke permissions for the new user
 REVOKE CREATE ON SCHEMA public FROM PUBLIC;
 REVOKE USAGE ON SCHEMA public FROM PUBLIC;
@@ -31,7 +31,7 @@
 -- grant access to new tables and views by default
 ALTER DEFAULT PRIVILEGES FOR USER "{ckanuser}" IN SCHEMA public
    GRANT SELECT ON TABLES TO "{readonlyuser}";
-"""
+'''
 
 
 class SetupDatastoreCommand(CkanCommand):
@@ -68,11 +68,11 @@ def command(self):
         cmd = self.args[0]
         self._load_config()
 
-        self.urlparts_w = self._get_db_config('ckan.datastore.write_url')
-        self.urlparts_r = self._get_db_config('ckan.datastore.read_url')
-        self.urlparts_c = self._get_db_config('sqlalchemy.url')
+        self.db_write_url_parts = self._get_db_config('ckan.datastore.write_url')
+        self.db_read_url_parts = self._get_db_config('ckan.datastore.read_url')
+        self.db_ckan_url_parts = self._get_db_config('sqlalchemy.url')
 
-        assert self.urlparts_w['db_name'] == self.urlparts_r['db_name'], "write and read db should be the same"
+        assert self.db_write_url_parts['db_name'] == self.db_read_url_parts['db_name'], "write and read db should be the same"
 
         if cmd == 'create-db':
             if len(self.args) != 2:
@@ -122,18 +122,18 @@ def _run_sql(self, sql, as_sql_user, database='postgres'):
             ))
 
     def create_db(self):
-        sql = "create database {0}".format(self.urlparts_w['db_name'])
+        sql = "create database {0}".format(self.db_write_url_parts['db_name'])
         self._run_sql(sql, as_sql_user=self.sql_superuser)
 
     def create_read_only_user(self):
-        password = self.urlparts_r['db_pass']
+        password = self.db_read_url_parts['db_pass']
         sql = read_only_user_sql.format(
-            maindb=self.urlparts_c['db_name'],
-            datastore=self.urlparts_w['db_name'],
-            ckanuser=self.urlparts_c['db_user'],
-            readonlyuser=self.urlparts_r['db_user'],
+            maindb=self.db_ckan_url_parts['db_name'],
+            datastore=self.db_write_url_parts['db_name'],
+            ckanuser=self.db_ckan_url_parts['db_user'],
+            readonlyuser=self.db_read_url_parts['db_user'],
             with_password="WITH PASSWORD '{0}'".format(password) if password else "",
-            writeuser=self.urlparts_w['db_user'])
+            writeuser=self.db_write_url_parts['db_user'])
         self._run_sql(sql,
                       as_sql_user=self.sql_superuser,
-                      database=self.urlparts_w['db_name'])
+                      database=self.db_write_url_parts['db_name'])

From ab6e970475b39a9eef88dc3031cc822c391973c4 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 26 Sep 2012 12:00:24 +0100
Subject: [PATCH 22/37] Check passwords for '

---
 ckanext/datastore/commands.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 9fbb504a55b..27da26d7e0f 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -127,6 +127,7 @@ def create_db(self):
 
     def create_read_only_user(self):
         password = self.db_read_url_parts['db_pass']
+        self.validate_password(password)
         sql = read_only_user_sql.format(
             maindb=self.db_ckan_url_parts['db_name'],
             datastore=self.db_write_url_parts['db_name'],
@@ -137,3 +138,7 @@ def create_read_only_user(self):
         self._run_sql(sql,
                       as_sql_user=self.sql_superuser,
                       database=self.db_write_url_parts['db_name'])
+
+    def validate_password(self, password):
+        if "'" in password:
+            raise ValueError("Passwords cannot contain '")

From ca014d3fed7fa5bae15bd14bd7b4331cd3f6ae3b Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 26 Sep 2012 13:26:11 +0100
Subject: [PATCH 23/37] pipe sql into psql instead of passing it as parameter,
 refactor parsing of database urls

---
 ckan/lib/cli.py               | 38 +++++++++++++++++++++++------
 ckanext/datastore/commands.py | 46 +++++++++++++++++------------------
 2 files changed, 52 insertions(+), 32 deletions(-)

diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py
index c20296b3517..8a77de79ae2 100644
--- a/ckan/lib/cli.py
+++ b/ckan/lib/cli.py
@@ -13,6 +13,35 @@
 #   i.e. do the imports in methods, after _load_config is called.
 #   Otherwise loggers get disabled.
 
+
+def parse_db_config(config_key='sqlalchemy.url'):
+    ''' Takes a config key for a database connection url and parses it into
+    a dictionary. Expects a url like:
+
+    'postgres://tester:pass@localhost/ckantest3'
+    '''
+    from pylons import config
+    url = config[config_key]
+    regex = [
+        '^\s*(?P<db_type>\w*)',
+        '://',
+        '(?P<db_user>[^:]*)',
+        ':?',
+        '(?P<db_pass>[^@]*)',
+        '@',
+        '(?P<db_host>[^/:]*)',
+        ':?',
+        '(?P<db_port>[^/]*)',
+        '/',
+        '(?P<db_name>[\w.-]*)'
+    ]
+    db_details_match = re.match(''.join(regex), url)
+    if not db_details_match:
+        raise Exception('Could not extract db details from url: %r' % url)
+    db_details = db_details_match.groupdict()
+    return db_details
+
+
 class MockTranslator(object):
     def gettext(self, value):
         return value
@@ -134,14 +163,7 @@ def command(self):
             sys.exit(1)
 
     def _get_db_config(self):
-        from pylons import config
-        url = config['sqlalchemy.url']
-        # e.g. 'postgres://tester:pass@localhost/ckantest3'
-        db_details_match = re.match('^\s*(?P<db_type>\w*)://(?P<db_user>[^:]*):?(?P<db_pass>[^@]*)@(?P<db_host>[^/:]*):?(?P<db_port>[^/]*)/(?P<db_name>[\w.-]*)', url)
-        if not db_details_match:
-            raise Exception('Could not extract db details from url: %r' % url)
-        db_details = db_details_match.groupdict()
-        return db_details
+        return parse_db_config()
 
     def _get_postgres_cmd(self, command):
         self.db_details = self._get_db_config()
diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 27da26d7e0f..3e2311f19d4 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -1,9 +1,12 @@
 import re
-from ckan.lib.cli import CkanCommand
-
+import sys
 import logging
+
+import ckan.lib.cli as cli
+
 log = logging.getLogger(__name__)
 
+
 read_only_user_sql = '''
 -- revoke permissions for the new user
 REVOKE CREATE ON SCHEMA public FROM PUBLIC;
@@ -34,7 +37,7 @@
 '''
 
 
-class SetupDatastoreCommand(CkanCommand):
+class SetupDatastoreCommand(cli.CkanCommand):
     '''Perform commands to set up the datastore.
     Make sure that the datastore urls are set properly before you run these commands.
 
@@ -68,9 +71,9 @@ def command(self):
         cmd = self.args[0]
         self._load_config()
 
-        self.db_write_url_parts = self._get_db_config('ckan.datastore.write_url')
-        self.db_read_url_parts = self._get_db_config('ckan.datastore.read_url')
-        self.db_ckan_url_parts = self._get_db_config('sqlalchemy.url')
+        self.db_write_url_parts = cli.parse_db_config('ckan.datastore.write_url')
+        self.db_read_url_parts = cli.parse_db_config('ckan.datastore.read_url')
+        self.db_ckan_url_parts = cli.parse_db_config('sqlalchemy.url')
 
         assert self.db_write_url_parts['db_name'] == self.db_read_url_parts['db_name'], "write and read db should be the same"
 
@@ -95,31 +98,26 @@ def command(self):
             log.error('Command "%s" not recognized' % (cmd,))
             return
 
-    def _get_db_config(self, name):
-        from pylons import config
-        url = config[name]
-        # e.g. 'postgres://tester:pass@localhost/ckantest3'
-        db_details_match = re.match('^\s*(?P<db_type>\w*)://(?P<db_user>[^:]*):?(?P<db_pass>[^@]*)@(?P<db_host>[^/:]*):?(?P<db_port>[^/]*)/(?P<db_name>[\w.-]*)', url)
-        if not db_details_match:
-            raise Exception('Could not extract db details from url: %r' % url)
-        db_details = db_details_match.groupdict()
-        return db_details
-
-    def _run_cmd(self, command_line):
+    def _run_cmd(self, command_line, inputstring=''):
         import subprocess
-        retcode = subprocess.call(command_line, shell=True)
-        if retcode != 0:
-            raise SystemError('Command exited with errorcode: %i' % retcode)
+        p = subprocess.Popen(
+            command_line, shell=True,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+        stdout_value, stderr_value = p.communicate(input=inputstring)
+        if stderr_value:
+            print '\nAn error occured: {0}'.format(stderr_value)
+            sys.exit(1)
 
     def _run_sql(self, sql, as_sql_user, database='postgres'):
         if self.verbose:
             print "Executing: \n#####\n", sql, "\n####\nOn database:", database
         if not self.simulate:
-            self._run_cmd("psql --username='{username}' --dbname='{database}' -c \"{sql}\" -W".format(
+            self._run_cmd("psql --username='{username}' --dbname='{database}' -W".format(
                 username=as_sql_user,
-                database=database,
-                sql=sql.replace('"', '\\"')
-            ))
+                database=database
+            ), inputstring=sql)
 
     def create_db(self):
         sql = "create database {0}".format(self.db_write_url_parts['db_name'])

From ef98309e86301f5875ab4da940927f0490c4e525 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 26 Sep 2012 13:27:03 +0100
Subject: [PATCH 24/37] made pluck function more readable

---
 ckanext/datastore/db.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index 116d60c0310..6d48b01081d 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -34,7 +34,6 @@ def __init__(self, error_dict):
                 '%d-%m-%Y',
                 '%m-%d-%Y',
                 ]
-_pluck = lambda field, arr: [x[field] for x in arr]
 INSERT = 'insert'
 UPSERT = 'upsert'
 UPDATE = 'update'
@@ -47,6 +46,10 @@ def _strip(input):
     return input
 
 
+def _pluck(field, arr):
+    return [x[field] for x in arr]
+
+
 def _get_list(input, strip=True):
     """Transforms a string or list to a list"""
     if input == None:

From a1d07930c3d5e3c139e10541902558cee9c66c11 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 26 Sep 2012 13:28:11 +0100
Subject: [PATCH 25/37] updated documentation on datastore

---
 doc/datastore.rst | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/doc/datastore.rst b/doc/datastore.rst
index ae12e037fdd..d9ce3aa58c6 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -51,19 +51,17 @@ Also ensure that the ``ckan.datastore.write_url`` and ``datastore.read_url`` var
 
 A few things have to be kept in mind
 
-* The datastore cannot be on the CKAN database
+* The datastore cannot be on the CKAN database (except for testing)
 * The write user (i.e. ``ckanuser``) and read-only user (i.e. ``readonlyuser``) cannot be the same
 
 To create a new database and a read-only user, use the provided paster commands after you have set the right database URLs.::
 
- paster --plugin=ckan datastore create-db
- paster --plugin=ckan datastore create-read-only-user
+ paster --plugin=ckan datastore create-db <sql-user-user>
+ paster --plugin=ckan datastore create-read-only-user <sql-user-user>
 
-To test you can create a new datastore, so on linux command line do::
+To test the setup you can create a new datastore, so on linux command line do::
 
- curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}"
-   -d '{"resource_id": "{RESOURCE-ID}", "fields": [ {"id": "a"}, {"id": "b"} ],
-    "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ]}'
+ curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" -d '{"resource_id": "{RESOURCE-ID}", "fields": [ {"id": "a"}, {"id": "b"} ], "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ]}'
 
 
 DataStorer: Automatically Add Data to the DataStore
@@ -167,6 +165,18 @@ The datastore_search_sql API endpoint allows a user to search data at a resource
  }
 
 
+datastore_search_htsql
+~~~~~~~~~~~~~~~~~~~~
+
+.. note:: HTSQL is not in the core datastore and has to be installed as a plugin.
+
+The datastore_search_htsql API endpoint allows a user to search data at a resource using the `HTSQL <http://htsql.org/doc/>`_ query expression language. The JSON for searching must be in the following form::
+
+ {
+    htsql: # a htsql query statement.
+ }
+
+
 Table aliases
 -------------
 
@@ -178,7 +188,7 @@ Comparison of different querying methods
 The datastore supports querying with the datastore_search and datastore_search_sql API endpoint. They are similar but support different features. The following list gives an overview on the different methods.
 
 ==============================  =======================  =====================  ======================
-..                              datastore_search         datastore_search_sql
+..                              datastore_search         datastore_search_sql   datastore_search_htsql
 ..                                                       SQL                    HTSQL
 ==============================  =======================  =====================  ======================
 **Status**                      Stable                   Stable                 Will be available as plugin

From 3748cacc7e609c15049da1f1505bd9bbf3bf9d7d Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 26 Sep 2012 13:28:49 +0100
Subject: [PATCH 26/37] started updating docs on using the data api

---
 doc/using-data-api.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst
index ac5b2cc97e6..85a6173e6ff 100644
--- a/doc/using-data-api.rst
+++ b/doc/using-data-api.rst
@@ -17,8 +17,11 @@ Quickstart
 There are several endpoints into the DataStore API, they are:
 
 * datastore_create: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_create``
-* datastore_search: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search``
 * datastore_delete: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_delete``
+* datastore_delete: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_upsert``
+* datastore_search: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search``
+* datastore_search: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_sql``
+* datastore_search: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search``
 
 Before going into detail about the API and the examples, it is useful to create a resource first so that you can store data against it in the Datastore. This can be done either through the CKAN Graphical User Interface.
 

From 069cbc0425011a500d8263784ccb04e20cea998f Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 26 Sep 2012 15:39:52 +0100
Subject: [PATCH 27/37] fix paster command docs

---
 ckanext/datastore/commands.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 3e2311f19d4..424c7b8681e 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -43,7 +43,7 @@ class SetupDatastoreCommand(cli.CkanCommand):
 
     Usage::
 
-        paster datastore create-db <sql-user-user>
+        paster datastore create-db <sql-super-user>
         paster datastore create-read-only-user <sql-super-user>
 
     Where:

From 040e25095552323d8a6acef35e9a43a28c15b3ef Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 27 Sep 2012 09:53:37 +0100
Subject: [PATCH 28/37] Remove warning about action api

---
 doc/apiv3.rst | 2 --
 1 file changed, 2 deletions(-)

diff --git a/doc/apiv3.rst b/doc/apiv3.rst
index f9b5a580952..92f45e5b9e1 100644
--- a/doc/apiv3.rst
+++ b/doc/apiv3.rst
@@ -21,8 +21,6 @@ If you don't specify the version number then you will default to version 1 of th
 Action API
 ~~~~~~~~~~
 
-.. warning:: The Action API is in beta in this release. Please feed back comments and problems. There is a known issue that incorrect parameters cause unhelpful errors with status 500.
-
 Overview
 --------
 

From 82cd9f08a5c233f4f214c0bd03bf678bb87bdad9 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 27 Sep 2012 10:03:31 +0100
Subject: [PATCH 29/37] allow database names with -

---
 ckanext/datastore/commands.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 424c7b8681e..8c685a46a76 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -120,7 +120,7 @@ def _run_sql(self, sql, as_sql_user, database='postgres'):
             ), inputstring=sql)
 
     def create_db(self):
-        sql = "create database {0}".format(self.db_write_url_parts['db_name'])
+        sql = 'create database "{0}"'.format(self.db_write_url_parts['db_name'])
         self._run_sql(sql, as_sql_user=self.sql_superuser)
 
     def create_read_only_user(self):

From 52e21aaf7c7bea199c648a451b4499c8feb50f9c Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 27 Sep 2012 10:05:35 +0100
Subject: [PATCH 30/37] Fix an actual bug that prevented aliases from getting
 deleted when alias: [] has been provided

---
 ckanext/datastore/db.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index 6d48b01081d..23e8a5aeef1 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -303,8 +303,8 @@ def _get_aliases(context, data_dict):
 
 
 def create_alias(context, data_dict):
-    aliases = _get_list(data_dict.get('aliases', None))
-    if aliases:
+    aliases = _get_list(data_dict.get('aliases'))
+    if aliases != None:
         # delete previous aliases
         previous_aliases = _get_aliases(context, data_dict)
         for alias in previous_aliases:
@@ -312,9 +312,9 @@ def create_alias(context, data_dict):
             context['connection'].execute(sql_alias_drop_string)
 
         for alias in aliases:
-            sql_alias_string = u'CREATE VIEW"{alias}" AS SELECT * FROM "{main}"'.format(
-                main=data_dict['resource_id'],
-                alias=alias
+            sql_alias_string = u'CREATE VIEW "{alias}" AS SELECT * FROM "{main}"'.format(
+                alias=alias,
+                main=data_dict['resource_id']
             )
             context['connection'].execute(sql_alias_string)
 

From b09de16bade5f5be7eb73318800d5180972f216c Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 27 Sep 2012 10:24:47 +0100
Subject: [PATCH 31/37] Fix tests in DatastoreCreate to run on separate dbs

---
 ckanext/datastore/tests/test_datastore.py | 40 ++++++++++++++++-------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py
index 44b68166e28..71dd9b51918 100644
--- a/ckanext/datastore/tests/test_datastore.py
+++ b/ckanext/datastore/tests/test_datastore.py
@@ -14,6 +14,16 @@ def extract(d, keys):
     return dict((k, d[k]) for k in keys if k in d)
 
 
+def clear_db(Session):
+    drop_tables = u'''select 'drop table "' || tablename || '" cascade;'
+                    from pg_tables where schemaname = 'public' '''
+    c = Session.connection()
+    results = c.execute(drop_tables)
+    for result in results:
+        c.execute(result[0])
+    Session.commit()
+
+
 class TestTypeGetters(unittest.TestCase):
     def test_list(self):
         assert db._get_list(None) == None
@@ -52,10 +62,18 @@ def setup_class(cls):
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        orm = sqlalchemy.orm
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
 
     @classmethod
     def teardown_class(cls):
         model.repo.rebuild_db()
+        clear_db(cls.Session)
 
     def test_create_requires_auth(self):
         resource = model.Package.get('annakarenina').resources[0]
@@ -256,7 +274,7 @@ def test_create_basic(self):
         assert res['fields'] == data['fields'], res['fields']
         assert res['records'] == data['records']
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(resource.id))
 
         assert results.rowcount == 3
@@ -274,11 +292,9 @@ def test_create_basic(self):
             select * from "{0}" where _full_text @@ to_tsquery('tolstoy')
             '''.format(resource.id))
         assert results.rowcount == 2
-        model.Session.remove()
 
         # check aliases for resource
         for alias in aliases:
-            c = model.Session.connection()
 
             results = [row for row in c.execute(u'select * from "{0}"'.format(resource.id))]
             results_alias = [row for row in c.execute(u'select * from "{0}"'.format(alias))]
@@ -312,7 +328,6 @@ def test_create_basic(self):
 
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
         results = c.execute('select * from "{0}"'.format(resource.id))
 
         assert results.rowcount == 4
@@ -323,11 +338,12 @@ def test_create_basic(self):
             assert all_data[i].get('author') == (
                 json.loads(row['author'][0]) if row['author'] else None)
 
+        c = self.Session.connection()
         results = c.execute('''
             select * from "{0}" where _full_text @@ 'tolstoy'
             '''.format(resource.id))
+        self.Session.remove()
         assert results.rowcount == 3
-        model.Session.remove()
 
         #######  insert again extra field
         data3 = {
@@ -345,7 +361,7 @@ def test_create_basic(self):
 
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(resource.id))
 
         assert results.rowcount == 5
@@ -356,10 +372,9 @@ def test_create_basic(self):
             assert all_data[i].get('author') == (json.loads(row['author'][0]) if row['author'] else None)
 
         results = c.execute('''select * from "{0}" where _full_text @@ to_tsquery('dostoevsky') '''.format(resource.id))
+        self.Session.remove()
         assert results.rowcount == 2
 
-        model.Session.remove()
-
         #######  insert again which will fail because of unique book name
         data4 = {
             'resource_id': resource.id,
@@ -393,7 +408,7 @@ def test_create_basic(self):
         assert res_dict['success'] is True, res_dict
 
         # new aliases should replace old aliases
-        c = model.Session.connection()
+        c = self.Session.connection()
         for alias in aliases:
             sql = (u"select * from _table_metadata "
                 "where alias_of='{0}' and name='{1}'").format(resource.id, alias)
@@ -404,6 +419,7 @@ def test_create_basic(self):
             "where alias_of='{0}' and name='{1}'").format(resource.id, 'another_alias')
         results = c.execute(sql)
         assert results.rowcount == 1
+        self.Session.remove()
 
     def test_guess_types(self):
         resource = model.Package.get('annakarenina').resources[1]
@@ -424,7 +440,7 @@ def test_guess_types(self):
                             extra_environ=auth)
         res_dict = json.loads(res.body)
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('''select * from "{0}" '''.format(resource.id))
 
         types = [db._pg_types[field[1]] for field in results.cursor.description]
@@ -436,7 +452,7 @@ def test_guess_types(self):
             assert data['records'][i].get('book') == row['book']
             assert data['records'][i].get('author') == (
                 json.loads(row['author'][0]) if row['author'] else None)
-        model.Session.remove()
+        self.Session.remove()
 
         ### extend types
 
@@ -461,7 +477,7 @@ def test_guess_types(self):
                             extra_environ=auth)
         res_dict = json.loads(res.body)
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('''select * from "{0}" '''.format(resource.id))
 
         types = [db._pg_types[field[1]] for field in results.cursor.description]

From 4a2e6c7eaa340370deac8991ff9e08d96fa8febe Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 27 Sep 2012 11:34:04 +0100
Subject: [PATCH 32/37] Timeout for upsert

---
 ckanext/datastore/db.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index 23e8a5aeef1..76d7c2c2758 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -870,12 +870,25 @@ def upsert(context, data_dict):
     '''
     engine = _get_engine(context, data_dict)
     context['connection'] = engine.connect()
+    timeout = context.get('query_timeout', 60000)
 
-    # check if table already existes
-    trans = context['connection'].begin()
-    upsert_data(context, data_dict)
-    trans.commit()
-    return _unrename_json_field(data_dict)
+    try:
+        # check if table already existes
+        trans = context['connection'].begin()
+        context['connection'].execute(
+            u'SET LOCAL statement_timeout TO {0}'.format(timeout))
+        upsert_data(context, data_dict)
+        trans.commit()
+        return _unrename_json_field(data_dict)
+    except Exception, e:
+        trans.rollback()
+        if 'due to statement timeout' in str(e):
+            raise ValidationError({
+                'query': ['Query took too long']
+            })
+        raise
+    finally:
+        context['connection'].close()
 
 
 def delete(context, data_dict):
@@ -920,7 +933,7 @@ def search(context, data_dict):
     try:
         # check if table exists
         context['connection'].execute(
-            u'set local statement_timeout to {0}'.format(timeout))
+            u'SET LOCAL statement_timeout TO {0}'.format(timeout))
         id = data_dict['resource_id']
         result = context['connection'].execute(
             u"(SELECT 1 FROM pg_tables where tablename = '{0}') union"

From a414ac0954041dae844dae7433b39579e216dfbb Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 27 Sep 2012 11:34:25 +0100
Subject: [PATCH 33/37] Transaction rollback in create

---
 ckanext/datastore/db.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index 76d7c2c2758..10434a6e8dd 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -851,6 +851,7 @@ def create(context, data_dict):
         else:
             raise
     except Exception, e:
+        trans.rollback()
         if 'due to statement timeout' in str(e):
             raise ValidationError({
                 'query': ['Query took too long']

From c3eb3597ec324cfd1756d5b5708e79c1bf895b5c Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 27 Sep 2012 12:04:07 +0100
Subject: [PATCH 34/37] Make all tests passing in setup with one or with two
 separate dbs.

---
 ckanext/datastore/tests/test_datastore.py | 138 +++++++++++++++-------
 1 file changed, 97 insertions(+), 41 deletions(-)

diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py
index 71dd9b51918..963d9f9dc26 100644
--- a/ckanext/datastore/tests/test_datastore.py
+++ b/ckanext/datastore/tests/test_datastore.py
@@ -1,14 +1,17 @@
 import unittest
 import json
+import pprint
+import datetime
+
 import sqlalchemy
+import sqlalchemy.orm as orm
+
 import ckan.plugins as p
 import ckan.lib.create_test_data as ctd
 import ckan.model as model
 import ckan.tests as tests
+import ckan.lib.cli as cli
 import ckanext.datastore.db as db
-import pprint
-import datetime
-
 
 def extract(d, keys):
     return dict((k, d[k]) for k in keys if k in d)
@@ -22,6 +25,21 @@ def clear_db(Session):
     for result in results:
         c.execute(result[0])
     Session.commit()
+    Session.remove()
+
+
+def rebuild_all_dbs(Session):
+    ''' If the tests are running on the same db, we have to make sure that
+    the ckan tables are recrated.
+    '''
+    db_read_url_parts = cli.parse_db_config('ckan.datastore.read_url')
+    db_ckan_url_parts = cli.parse_db_config('sqlalchemy.url')
+    same_db = db_read_url_parts['db_name'] == db_ckan_url_parts['db_name']
+
+    if same_db:
+        model.repo.tables_created_and_initialised = False
+    clear_db(Session)
+    model.repo.rebuild_db()
 
 
 class TestTypeGetters(unittest.TestCase):
@@ -59,6 +77,7 @@ class TestDatastoreCreate(tests.WsgiAppCase):
 
     @classmethod
     def setup_class(cls):
+        p.load('datastore')
         ctd.CreateTestData.create()
         cls.sysadmin_user = model.User.get('testsysadmin')
         cls.normal_user = model.User.get('annafan')
@@ -67,13 +86,11 @@ def setup_class(cls):
                 None,
                 {'connection_url': pylons.config['ckan.datastore.write_url']}
             )
-        orm = sqlalchemy.orm
         cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
 
     @classmethod
     def teardown_class(cls):
-        model.repo.rebuild_db()
-        clear_db(cls.Session)
+        rebuild_all_dbs(cls.Session)
 
     def test_create_requires_auth(self):
         resource = model.Package.get('annakarenina').resources[0]
@@ -292,8 +309,10 @@ def test_create_basic(self):
             select * from "{0}" where _full_text @@ to_tsquery('tolstoy')
             '''.format(resource.id))
         assert results.rowcount == 2
+        self.Session.remove()
 
         # check aliases for resource
+        c = self.Session.connection()
         for alias in aliases:
 
             results = [row for row in c.execute(u'select * from "{0}"'.format(resource.id))]
@@ -305,6 +324,7 @@ def test_create_basic(self):
                 "where alias_of='{0}' and name='{1}'").format(resource.id, alias)
             results = c.execute(sql)
             assert results.rowcount == 1
+        self.Session.remove()
 
         # check to test to see if resource now has a datastore table
         postparams = '%s=1' % json.dumps({'id': resource.id})
@@ -328,7 +348,9 @@ def test_create_basic(self):
 
         assert res_dict['success'] is True
 
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(resource.id))
+        self.Session.remove()
 
         assert results.rowcount == 4
 
@@ -479,6 +501,7 @@ def test_guess_types(self):
 
         c = self.Session.connection()
         results = c.execute('''select * from "{0}" '''.format(resource.id))
+        self.Session.remove()
 
         types = [db._pg_types[field[1]] for field in results.cursor.description]
 
@@ -551,9 +574,16 @@ def setup_class(cls):
         res_dict = json.loads(res.body)
         assert res_dict['success'] is True
 
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
     @classmethod
     def teardown_class(cls):
-        model.repo.rebuild_db()
+        rebuild_all_dbs(cls.Session)
 
     def test_upsert_requires_auth(self):
         data = {
@@ -574,10 +604,10 @@ def test_upsert_empty_fails(self):
         assert res_dict['success'] is False
 
     def test_upsert_basic(self):
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select 1 from "{0}"'.format(self.data['resource_id']))
         assert results.rowcount == 2
-        model.Session.remove()
+        self.Session.remove()
 
         hhguide = u"hitchhikers guide to the galaxy"
 
@@ -595,19 +625,19 @@ def test_upsert_basic(self):
 
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
         assert results.rowcount == 3
 
         records = results.fetchall()
         assert records[2][u'b\xfck'] == hhguide
         assert records[2].author == 'adams'
-        model.Session.remove()
+        self.Session.remove()
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute("select * from \"{0}\" where author='{1}'".format(self.data['resource_id'], 'adams'))
         assert results.rowcount == 1
-        model.Session.remove()
+        self.Session.remove()
 
         # upsert only the publish date
         data = {
@@ -624,7 +654,7 @@ def test_upsert_basic(self):
 
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
         assert results.rowcount == 3
 
@@ -632,7 +662,7 @@ def test_upsert_basic(self):
         assert records[2][u'b\xfck'] == hhguide
         assert records[2].author == 'adams'
         assert records[2].published == datetime.datetime(1979, 1, 1)
-        model.Session.remove()
+        self.Session.remove()
 
         # delete publish date
         data = {
@@ -649,7 +679,7 @@ def test_upsert_basic(self):
 
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
         assert results.rowcount == 3
 
@@ -657,7 +687,7 @@ def test_upsert_basic(self):
         assert records[2][u'b\xfck'] == hhguide
         assert records[2].author == 'adams'
         assert records[2].published == None
-        model.Session.remove()
+        self.Session.remove()
 
         data = {
             'resource_id': self.data['resource_id'],
@@ -673,14 +703,14 @@ def test_upsert_basic(self):
 
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
         assert results.rowcount == 4
 
         records = results.fetchall()
         assert records[3][u'b\xfck'] == 'the hobbit'
         assert records[3].author == 'tolkien'
-        model.Session.remove()
+        self.Session.remove()
 
         # test % in records
         data = {
@@ -759,9 +789,16 @@ def setup_class(cls):
         res_dict = json.loads(res.body)
         assert res_dict['success'] is True
 
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
     @classmethod
     def teardown_class(cls):
-        model.repo.rebuild_db()
+        rebuild_all_dbs(cls.Session)
 
     def test_insert_basic(self):
         data = {
@@ -778,8 +815,9 @@ def test_insert_basic(self):
 
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        self.Session.remove()
 
         assert results.rowcount == 3
 
@@ -831,15 +869,22 @@ def setup_class(cls):
         res_dict = json.loads(res.body)
         assert res_dict['success'] is True
 
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
     @classmethod
     def teardown_class(cls):
-        model.repo.rebuild_db()
+        rebuild_all_dbs(cls.Session)
 
     def test_update_basic(self):
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select 1 from "{0}"'.format(self.data['resource_id']))
         assert results.rowcount == 3
-        model.Session.remove()
+        self.Session.remove()
 
         hhguide = u"hitchhikers guide to the galaxy"
 
@@ -857,19 +902,19 @@ def test_update_basic(self):
 
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
         assert results.rowcount == 3
 
         records = results.fetchall()
         assert records[2][u'b\xfck'] == hhguide
         assert records[2].author == 'adams'
-        model.Session.remove()
+        self.Session.remove()
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute("select * from \"{0}\" where author='{1}'".format(self.data['resource_id'], 'adams'))
         assert results.rowcount == 1
-        model.Session.remove()
+        self.Session.remove()
 
         # update only the publish date
         data = {
@@ -886,15 +931,15 @@ def test_update_basic(self):
 
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        self.Session.remove()
         assert results.rowcount == 3
 
         records = results.fetchall()
         assert records[2][u'b\xfck'] == hhguide
         assert records[2].author == 'adams'
         assert records[2].published == datetime.datetime(1979, 1, 1)
-        model.Session.remove()
 
         # delete publish date
         data = {
@@ -911,15 +956,15 @@ def test_update_basic(self):
 
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        self.Session.remove()
         assert results.rowcount == 3
 
         records = results.fetchall()
         assert records[2][u'b\xfck'] == hhguide
         assert records[2].author == 'adams'
         assert records[2].published == None
-        model.Session.remove()
 
     def test_update_missing_key(self):
         data = {
@@ -970,6 +1015,7 @@ def test_update_non_existing_field(self):
 class TestDatastoreDelete(tests.WsgiAppCase):
     sysadmin_user = None
     normal_user = None
+    Session = None
 
     @classmethod
     def setup_class(cls):
@@ -987,9 +1033,19 @@ def setup_class(cls):
                         {'book': 'warandpeace', 'author': 'tolstoy'}]
         }
 
+        #model.repo.rebuild_db()
+        #model.repo.clean_db()
+
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
     @classmethod
     def teardown_class(cls):
-        model.repo.rebuild_db()
+        rebuild_all_dbs(cls.Session)
 
     def _create(self):
         postparams = '%s=1' % json.dumps(self.data)
@@ -1015,7 +1071,7 @@ def test_delete_basic(self):
         self._create()
         self._delete()
         resource_id = self.data['resource_id']
-        c = model.Session.connection()
+        c = self.Session.connection()
 
         # alias should be deleted
         results = c.execute("select 1 from pg_views where viewname = '{0}'".format(self.data['aliases']))
@@ -1030,7 +1086,7 @@ def test_delete_basic(self):
             expected_msg = 'relation "{0}" does not exist'.format(resource_id)
             assert expected_msg in str(e)
 
-        model.Session.remove()
+        self.Session.remove()
 
     def test_delete_invalid_resource_id(self):
         postparams = '%s=1' % json.dumps({'resource_id': 'bad'})
@@ -1054,12 +1110,12 @@ def test_delete_filters(self):
         res_dict = json.loads(res.body)
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         result = c.execute('select * from "{0}";'.format(resource_id))
         results = [r for r in result]
         assert len(results) == 1
         assert results[0].book == 'annakarenina'
-        model.Session.remove()
+        self.Session.remove()
 
         # shouldn't delete anything
         data = {'resource_id': resource_id,
@@ -1071,12 +1127,12 @@ def test_delete_filters(self):
         res_dict = json.loads(res.body)
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         result = c.execute('select * from "{0}";'.format(resource_id))
         results = [r for r in result]
         assert len(results) == 1
         assert results[0].book == 'annakarenina'
-        model.Session.remove()
+        self.Session.remove()
 
         # delete the 'annakarenina' row
         data = {'resource_id': resource_id,
@@ -1088,11 +1144,11 @@ def test_delete_filters(self):
         res_dict = json.loads(res.body)
         assert res_dict['success'] is True
 
-        c = model.Session.connection()
+        c = self.Session.connection()
         result = c.execute('select * from "{0}";'.format(resource_id))
         results = [r for r in result]
         assert len(results) == 0
-        model.Session.remove()
+        self.Session.remove()
 
         self._delete()
 

From 075d65c4a3620bbffce1705ad1caca6f4983c983 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 27 Sep 2012 13:01:39 +0100
Subject: [PATCH 35/37] Updated, extended and unified documentation of api
 usage

---
 doc/using-data-api.rst | 153 ++++++++++++++++++++---------------------
 1 file changed, 75 insertions(+), 78 deletions(-)

diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst
index 85a6173e6ff..3ab9e0c4748 100644
--- a/doc/using-data-api.rst
+++ b/doc/using-data-api.rst
@@ -20,28 +20,28 @@ There are several endpoints into the DataStore API, they are:
 * datastore_delete: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_delete``
 * datastore_delete: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_upsert``
 * datastore_search: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search``
-* datastore_search: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_sql``
-* datastore_search: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search``
+* datastore_search_sql: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_sql``
+* datastore_search_htsql: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_htsql``
 
-Before going into detail about the API and the examples, it is useful to create a resource first so that you can store data against it in the Datastore. This can be done either through the CKAN Graphical User Interface.
+Before going into detail about the API and the examples, it is useful to create a resource first so that you can store data against it in the Datastore. This can be done either through the CKAN Graphical User Interface or the API.
 
 Using the data API, we need to send JSON with this structure::
 
-  { 
-    id: Uuid, 
-    name: Name-String, 
-    title: String, 
-    version: String, 
-    url: String, 
-    resources: [ { url: String, format: String, description: String, hash: String }, ...], 
-    author: String, 
-    author_email: String, 
-    maintainer: String, 
-    maintainer_email: String, 
-    license_id: String, 
-    tags: Tag-List, 
-    notes: String, 
-    extras: { Name-String: String, ... } 
+  {
+    id: Uuid,
+    name: Name-String,
+    title: String,
+    version: String,
+    url: String,
+    resources: [ { url: String, format: String, description: String, hash: String }, ...],
+    author: String,
+    author_email: String,
+    maintainer: String,
+    maintainer_email: String,
+    license_id: String,
+    tags: Tag-List,
+    notes: String,
+    extras: { Name-String: String, ... }
  }
 
 To the following endpoint:
@@ -59,39 +59,36 @@ Some of the following commands require obtaining an :ref:`API Key <get-api-key>`
 cURL (or Browser)
 ~~~~~~~~~~~~~~~~~
 
-The following examples utilize the cURL_ command line utility. If you prefer,
-you you can just open the relevant urls in your browser::
+The following examples utilize the cURL_ command line utility. If you prefer, you you can just open the relevant urls in your browser::
 
   # This creates a datastore
-  curl -X POST {ENDPOINT:datastore_create} -H "Authorization: {YOUR-API-KEY}" -d "
-    {\"resource_id\": \"{RESOURCE-ID}\", \"fields\": [ {\"id\": \"a\"}, {\"id\": \"b\"} ], 
-    \"records\": [ { \"a\": 1, \"b\": \"xyz\"}, {\"a\": 2, \"b\": \"zzz\"} ]}"
+  curl -X POST <ENDPOINT:datastore_create> -H "Authorization: <YOUR-API-KEY>" -d '{"resource_id": "<RESOURCE-ID>", "fields": [ {"id": "a"}, {"id": "b"} ], "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ]}'
+
 
   #This queries a datastore
-  curl {ENDPOINT:datastore_search}?resource_id={RESOURCE-ID} -H "Authorization: {YOUR-API-KEY}"
+  curl "<ENDPOINT:datastore_search>?resource_id=<RESOURCE-ID>" -H "Authorization: <YOUR-API-KEY>"
 
 .. _cURL: http://curl.haxx.se/
 
 Javascript
 ~~~~~~~~~~
 
-Coming soon...
+A simple ajax (JSONP) request to the data API using jQuery::
 
-..
-    A simple ajax (JSONP) request to the data API using jQuery::
+  $.ajax({
+    url: '<ENDPOINT:datastore_search>',
+    data: {'resource_id': '<RESOURCE-ID>'},
+    dataType: 'jsonp',
+    success: function(data) {
+      alert('Total results found: ' + data.result.total)
+    }
+  });
 
-      var data = {
-        size: 5 // get 5 results
-        q: 'title:jones' // query on the title field for 'jones'
-      };
-      $.ajax({
-        url: {{endpoint}}/_search,
-        dataType: 'jsonp',
-        success: function(data) {
-          alert('Total results found: ' + data.hits.total)
-        }
-      });
+The Data API supports CORs so you can also write to it (this requires the json2_ library for ``JSON.stringify``)::
 
+  Coming soon...
+
+..
     The Data API supports CORs so you can also write to it (this requires the json2_ library for ``JSON.stringify``)::
 
       var data = {
@@ -114,59 +111,57 @@ Python
 
 A Python URLLib2 datastore_create and datastore_search would look like::
 
- #! /usr/bin/env python
- import urllib
- import urllib2
- import json
+  #! /usr/bin/env python
+  import urllib
+  import urllib2
+  import json
 
- auth_key = '{YOUR-AUTH-KEY}'
+  auth_key = '<YOUR-API-KEY>'
 
- # In python using urllib2 for datastore_create it is...
+  # In python using urllib2 for datastore_create it is...
 
- url = "http://127.0.0.1:5000/api/3/action/"
-
- datastore_structure = {
-                        'resource_id': '{RESOURCE-ID}', 
-                        'fields': [ {"id": "a"}, {"id": "b"} ], 
-                        "records": [ { "a": 12, "b": "abc"}, {"a": 2, "b": "zzz"} ]
-                      }
- headers = {'content-type': 'application/json', 'Authorization': auth_key}
+  url = "<API-ENDPOINT>"
 
+  datastore_structure = {
+                  'resource_id': '<RESOURCE-ID>',
+                  'fields': [{"id": "a"}, {"id": "b"}],
+                  "records": [{"a": 12, "b": "abc"}, {"a": 2, "b": "zzz"}]
+                }
+  headers = {'content-type': 'application/json', 'Authorization': auth_key}
 
+  req = urllib2.Request(url + 'datastore_create', data=json.dumps(datastore_structure), headers=headers)
+  response = urllib2.urlopen(req)
 
- req = urllib2.Request(url + 'datastore_create', data=json.dumps(datastore_structure), headers=headers)
- response = urllib2.urlopen(req)
 
+  # in python for datastore_search using urllib2....
 
- # in python for datastore_search using urllib2....
+  datastore_structure = {
+      'resource_id': '<RESOURCE-ID>'
+    }
 
- datastore_structure = {
-                        'resource_id': '{RESOURCE-ID}'
-                      }
+  url_values = urllib.urlencode(datastore_structure)
+  req = urllib2.Request(url + 'datastore_search?' + url_values, headers=headers)
+  response = urllib2.urlopen(req)
 
- url_values = urllib.urlencode(datastore_structure)
- req = urllib2.Request(url + 'datastore_search?' + url_values, headers=headers)
- response = urllib2.urlopen(req)
+  print response.read()
 
- print response.read()
-
- print "done\n"
+  print "done\n"
 
 
 Using the Python Requests_ library we can create a datastore like this::
 
  #! /usr/bin/env python
- 
+
  import requests
- import json 
- 
- auth_key = '<your-api-key>' 
- 
- url = "http://127.0.0.1:5000/api/3/action/" # An example "action" endpoint
- 
+ import json
+
+ auth_key = '<YOUR-API-KEY>'
+
+ url = '<API-ENDPOINT>'
+
  datastore_structure = {
-                         'resource_id': '<existing-resource-id>', 
-                         'fields': [ {"id": "a"}, {"id": "b"} ], 
+                         'resource_id': '<RESOURCE-ID>',
+                         'fields': [ {"id": "a"}, {"id": "b"} ],
                          "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ]
                        }
  headers = {'content-type': 'application/json', 'Authorization': auth_key}
@@ -174,13 +169,13 @@ Using the Python Requests_ library we can create a datastore like this::
  print "done, and now for a quick search\n"
 
  datastore_structure = {
-                         'resource_id': '<existing-resource-id>'
+                         'resource_id': '<RESOURCE-ID>'
                        }
  headers = {'content-type': 'application/json', 'Authorization': auth_key}
- r = requests.post(url + 'datastore_search', data=json.dumps(datastore_structure), headers=headers) 
- 
+ r = requests.post(url + 'datastore_search', data=json.dumps(datastore_structure), headers=headers)
+
  print r.text
- 
+
  print "done\n"
 
 
@@ -189,5 +184,7 @@ Using the Python Requests_ library we can create a datastore like this::
 PHP
 ~~~~~~
 
-Coming soon...
+::
+
+  Coming soon...
 

From a88d042162132ee5499233654eeeee06c7855148 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 27 Sep 2012 13:19:29 +0100
Subject: [PATCH 36/37] Make limit and offset integers

---
 ckanext/datastore/db.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index 10434a6e8dd..677e48f37d3 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -749,6 +749,11 @@ def search_data(context, data_dict):
     _validate_int(limit, 'limit')
     _validate_int(offset, 'offset')
 
+    if 'limit' in data_dict:
+        data_dict['limit'] = int(limit)
+    if 'offset' in data_dict:
+        data_dict['offset'] = int(offset)
+
     sort = _sort(context, data_dict, field_ids)
 
     sql_string = u'''SELECT {select}, count(*) over() AS "_full_count" {rank}

From 0e9ff2d7b1b29f48382e71b35e1af8792f0b1267 Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Thu, 27 Sep 2012 19:31:50 +0200
Subject: [PATCH 37/37] Fix broken print statements in wsgi_ckanclient.py

These were causing some tests to fail for me. I don't understand why
they aren't failing for anyone else, or for me in the past, but this fix
seems harmless.
---
 ckan/tests/wsgi_ckanclient.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/ckan/tests/wsgi_ckanclient.py b/ckan/tests/wsgi_ckanclient.py
index 89a7341b41f..cf7e88331b1 100644
--- a/ckan/tests/wsgi_ckanclient.py
+++ b/ckan/tests/wsgi_ckanclient.py
@@ -26,7 +26,7 @@ def __init__(self, app, **kwargs):
 
     def open_url(self, location, data=None, headers={}, method=None):
         if self.is_verbose:
-            self._print("ckanclient: Opening %s" % location)
+            print "ckanclient: Opening %s" % location
         self.last_location = location
 
         if data != None:
@@ -55,31 +55,31 @@ def open_url(self, location, data=None, headers={}, method=None):
                 raise ClientError('No Paste interface for method \'%s\': %s' % \
                                   (method, location))
         except paste.fixture.AppError, inst:
-            self._print("ckanclient: error: %s" % inst)
+            print "ckanclient: error: %s" % inst
             self.last_http_error = inst
             self.last_status = 500
             self.last_message = repr(inst.args)
         else:
             if res.status not in (200, 201):
-                self._print("ckanclient: Received HTTP error code from CKAN resource.")
-                self._print("ckanclient: location: %s" % location)
-                self._print("ckanclient: response code: %s" % res.status)
-                self._print("ckanclient: request headers: %s" % headers)
-                self._print("ckanclient: request data: %s" % data)
-                self._print("ckanclient: error: %s" % res)
+                print "ckanclient: Received HTTP error code from CKAN resource."
+                print "ckanclient: location: %s" % location
+                print "ckanclient: response code: %s" % res.status
+                print "ckanclient: request headers: %s" % headers
+                print "ckanclient: request data: %s" % data
+                print "ckanclient: error: %s" % res
                 self.last_http_error = res
                 self.last_status = res.status
                 self.last_message = res.body
             else:
-                self._print("ckanclient: OK opening CKAN resource: %s" % location)
+                print "ckanclient: OK opening CKAN resource: %s" % location
                 self.last_status = res.status
-                self._print('ckanclient: last status %s' % self.last_status)
+                print 'ckanclient: last status %s' % self.last_status
                 self.last_body = res.body
-                self._print('ckanclient: last body %s' % self.last_body)
+                print 'ckanclient: last body %s' % self.last_body
                 self.last_headers = dict(res.headers)
-                self._print('ckanclient: last headers %s' % self.last_headers)
+                print 'ckanclient: last headers %s' % self.last_headers
                 content_type = self.last_headers['Content-Type']
-                self._print('ckanclient: content type: %s' % content_type)
+                print 'ckanclient: content type: %s' % content_type
                 is_json_response = False
                 if 'json' in content_type:
                     is_json_response = True
@@ -87,7 +87,7 @@ def open_url(self, location, data=None, headers={}, method=None):
                     self.last_message = self._loadstr(self.last_body)
                 else:
                     self.last_message = self.last_body
-                self._print('ckanclient: last message %s' % self.last_message)
+                print 'ckanclient: last message %s' % self.last_message
         if self.last_status not in (200, 201):
             raise CkanApiError(self.last_message)