Merge branch 'master' of github.com:okfn/ckan into 621-bug-move-extra…

…s_validation_into___before

bin/osx-postgres-mem.sh

@@ -13,7 +13,8 @@ case $1 in
 		${PGCTL} -D ${PGDATA} init
 		${PGCTL} -D ${PGDATA} start
 		sleep 2;
-		psql -c "CREATE DATABASE ckantest;" postgres
+		psql -c "CREATE DATABASE ckan_dev;" postgres
+		psql -c "CREATE DATABASE ckan_test;" postgres
 		;;
 	stop)
 		## stop postgres

bin/travis-build

@@ -16,40 +16,43 @@ sudo apt-get update -qq
 sudo apt-get install solr-jetty postgresql-$PGVERSION
 
 # Don't require a password to access DB
-sudo sed -i -e 's/ident/trust/g' /etc/postgresql/$PGVERSION/main/pg_hba.conf 
+sudo sed -i -e 's/ident/trust/g' /etc/postgresql/$PGVERSION/main/pg_hba.conf
 
 sudo service postgresql reload
 
 pip install -r pip-requirements.txt --use-mirrors
 pip install -r pip-requirements-test.txt --use-mirrors
 
-psql -c 'CREATE DATABASE ckantest;' -U postgres
-psql -c 'CREATE DATABASE datastore;' -U postgres
+psql -c 'CREATE DATABASE ckan_test;' -U postgres
+psql -c 'CREATE DATABASE ckan_test_datastore;' -U postgres
 
 python setup.py develop
 
 # Configure CKAN's configuration file
-paster make-config ckan development.ini --no-interactive
-sed -i -e 's/.*solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' development.ini
-sed -i -e 's/.*ckan\.site_id.*/ckan.site_id = travis_ci/' development.ini
-sed -i -e 's/^sqlalchemy.url.*/sqlalchemy.url = postgresql:\/\/postgres@\/ckantest/' development.ini
-sed -i -e 's/.*datastore.write_url.*/ckan.datastore.write_url = postgresql:\/\/postgres@\/datastore/' development.ini
+sed -i -e 's/.*solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' test-core.ini
+sed -i -e 's/.*ckan\.site_id.*/ckan.site_id = travis_ci/' test-core.ini
+sed -i -e 's/^sqlalchemy.url.*/sqlalchemy.url = postgresql:\/\/postgres@\/ckan_test/' test-core.ini
+sed -i -e 's/.*datastore.write_url.*/ckan.datastore.write_url = postgresql:\/\/postgres@\/ckan_test_datastore/' test-core.ini
 
 # Configure Solr
 echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
+# FIXME the solr schema cannot be hardcoded as it is dependent on the ckan version
 sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml
 sudo service jetty restart
 
-paster --plugin=ckan db init
+paster db init -c test-core.ini
 
 # If Postgres >= 9.0, we don't need to use datastore's legacy mode.
 if [ $PGVERSION != '8.4' ]
 then
   psql -c 'CREATE USER readonlyuser;' -U postgres
-  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/readonlyuser@\/datastore/' development.ini
-  paster datastore set-permissions postgres
+  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/readonlyuser@\/ckan_test_datastore/' test-core.ini
+  paster datastore set-permissions postgres -c test-core.ini
+else
+  sed -i -e 's/.*datastore.read_url.*//' test-core.ini
 fi
 
+cat test-core.ini
 
 # And finally, run the tests
 nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext

ckan/config/deployment.ini_tmpl

@@ -65,7 +65,7 @@ ckan.plugins = stats json_preview recline_preview
 # Specify the database for SQLAlchemy to use:
 # * Postgres is currently required for a production CKAN deployment
 # * Sqlite (memory or file) can be used as a quick alternative for testing
-sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckantest
+sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckan_dev
 #sqlalchemy.url = sqlite:///
 #sqlalchemy.url = sqlite:///%(here)s/somedb.db
 

ckan/controllers/api.py

@@ -532,6 +532,12 @@ def search(self, ver=None, register=None):
                         params = search.\
                             convert_legacy_parameters_to_solr(params)
                     query = search.query_for(model.Package)
+
+                    # Remove any existing fq param and set the capacity to
+                    # public
+                    if 'fq' in params:
+                        del params['fq']
+                    params['fq'] = '+capacity:public'
                     results = query.run(params)
                 return self._finish_ok(results)
             except search.SearchError, e:

ckan/controllers/group.py

@@ -197,9 +197,9 @@ def _read(self, id, limit):
         q = c.q = request.params.get('q', '')
         # Search within group
         if c.group_dict.get('is_organization'):
-            q += ' owner_org: "%s"' % c.group_dict.get('id')
+            q += ' owner_org:"%s"' % c.group_dict.get('id')
         else:
-            q += ' groups: "%s"' % c.group_dict.get('name')
+            q += ' groups:"%s"' % c.group_dict.get('name')
 
         try:
             description_formatted = ckan.misc.MarkdownFormat().to_html(
@@ -680,7 +680,7 @@ def history(self, id):
 
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author,
-                   'schema': self._form_to_db_schema()}
+                   'schema': self._db_to_form_schema()}
         data_dict = {'id': id}
         try:
             c.group_dict = self._action('group_show')(context, data_dict)

ckan/lib/search/query.py

@@ -16,9 +16,9 @@
 _open_licenses = None
 
 VALID_SOLR_PARAMETERS = set([
-    'q', 'fl', 'fq', 'rows', 'sort', 'start', 'wt', 'qf',
+    'q', 'fl', 'fq', 'rows', 'sort', 'start', 'wt', 'qf', 'bf',
     'facet', 'facet.mincount', 'facet.limit', 'facet.field',
-    'extras' # Not used by Solr, but useful for extensions
+    'extras', 'fq_list', 'tie', 'defType', 'mm'
 ])
 
 # for (solr) package searches, this specifies the fields that are searched
@@ -332,7 +332,10 @@ def run(self, query):
         # filter for package status
         if not '+state:' in fq:
             fq += " +state:active"
-        query['fq'] = fq
+        query['fq'] = [fq]
+
+        fq_list = query.get('fq_list', [])
+        query['fq'].extend(fq_list)
 
         # faceting
         query['facet'] = query.get('facet', 'true')
@@ -346,12 +349,13 @@ def run(self, query):
         query['wt'] = query.get('wt', 'json')
 
         # If the query has a colon in it then consider it a fielded search and do use dismax.
-        if ':' not in query['q']:
-            query['defType'] = 'dismax'
-            query['tie'] = '0.1'
+        defType = query.get('defType', 'dismax')
+        if ':' not in query['q'] or defType == 'edismax':
+            query['defType'] = defType
+            query['tie'] = query.get('tie', '0.1')
             # this minimum match is explained
             # http://wiki.apache.org/solr/DisMaxQParserPlugin#mm_.28Minimum_.27Should.27_Match.29
-            query['mm'] = '2<-1 5<80%'
+            query['mm'] = query.get('mm', '2<-1 5<80%')
             query['qf'] = query.get('qf', QUERY_FIELDS)
 
         conn = make_connection()

ckan/logic/action/create.py

@@ -109,15 +109,18 @@ def package_create(context, data_dict):
 
     package_type = data_dict.get('type')
     package_plugin = lib_plugins.lookup_package_plugin(package_type)
-    schema = package_plugin.create_package_schema()
+    if 'schema' in context:
+        schema = context['schema']
+    else:
+        schema = package_plugin.create_package_schema()
 
     _check_access('package_create', context, data_dict)
 
     if 'api_version' not in context:
         # check_data_dict() is deprecated. If the package_plugin has a
         # check_data_dict() we'll call it, if it doesn't have the method we'll
         # do nothing.
-        check_data_dict = getattr(package_plugin, 'check_datadict', None)
+        check_data_dict = getattr(package_plugin, 'check_data_dict', None)
         if check_data_dict:
             try:
                 check_data_dict(data_dict, schema)
@@ -173,6 +176,9 @@ def package_create(context, data_dict):
     context["id"] = pkg.id
     log.debug('Created object %s' % str(pkg.name))
 
+    # Make sure that a user provided schema is not used on package_show
+    context.pop('schema', None)
+
     return_id_only = context.get('return_id_only', False)
 
     output = context['id'] if return_id_only \

ckan/logic/action/get.py

@@ -731,7 +731,10 @@ def package_show(context, data_dict):
         item.read(pkg)
 
     package_plugin = lib_plugins.lookup_package_plugin(package_dict['type'])
-    schema = package_plugin.show_package_schema()
+    if 'schema' in context:
+        schema = context['schema']
+    else:
+        schema = package_plugin.show_package_schema()
 
     if schema and context.get('validate', True):
         package_dict, errors = _validate(package_dict, schema, context=context)

ckan/logic/action/update.py

@@ -235,7 +235,10 @@ def package_update(context, data_dict):
 
     # get the schema
     package_plugin = lib_plugins.lookup_package_plugin(pkg.type)
-    schema = package_plugin.update_package_schema()
+    if 'schema' in context:
+        schema = context['schema']
+    else:
+        schema = package_plugin.update_package_schema()
 
     if 'api_version' not in context:
         # check_data_dict() is deprecated. If the package_plugin has a
@@ -288,6 +291,9 @@ def package_update(context, data_dict):
 
     return_id_only = context.get('return_id_only', False)
 
+    # Make sure that a user provided schema is not used on package_show
+    context.pop('schema', None)
+
     # we could update the dataset so we should still be able to read it.
     context['ignore_auth'] = True
     output = data_dict['id'] if return_id_only \

ckan/model/tag.py

@@ -230,7 +230,8 @@ def __init__(self, package=None, tag=None, state=None, **kwargs):
             setattr(self, k, v)
 
     def __repr__(self):
-        return '<PackageTag package=%s tag=%s>' % (self.package.name, self.tag.name)
+        s = u'<PackageTag package=%s tag=%s>' % (self.package.name, self.tag.name)
+        return s.encode('utf8')
 
     def activity_stream_detail(self, activity_id, activity_type):
         if activity_type == 'new':

ckan/templates/group/history.html

@@ -0,0 +1,12 @@
+{% extends "group/read_base.html" %}
+
+{% block subtitle %}{{ _('History') }} - {{ c.group_dict.display_name }}{% endblock %}
+
+{% block primary_content_inner %}
+  <section class="module-content">
+    <h1 class="hide-heading">{{ _('History') }}</h1>
+    {% block group_history_revisions %}
+      {% snippet "group/snippets/history_revisions.html", group_dict=c.group_dict, group_revisions=c.group_revisions %}
+    {% endblock %}
+  </section>
+{% endblock %}

ckan/templates/group/snippets/history_revisions.html

@@ -0,0 +1,12 @@
+{% import 'macros/form.html' as form %}
+
+<form class="dataset-form dataset-resource-form form-horizontal" method="post" data-module="basic-form">
+
+  {{ form.errors(error_summary) }}
+
+  <input type="hidden" name="group_name" value="{{ group_dict.name }}"/>
+  {% snippet 'group/snippets/revisions_table.html', group_dict=group_dict, group_revisions=group_revisions %}
+
+  <button class="btn btn-primary" name="diff" value="go-resources" type="submit">{{ _('Compare') }}</button>
+
+</form>

ckan/templates/group/snippets/revisions_table.html

@@ -0,0 +1,31 @@
+{% import 'macros/form.html' as form %}
+
+<table class="table table-striped table-bordered">
+  <thead>
+    <tr>
+      <th></th>
+      <th>{{ _('Revision') }}</th>
+      <th>{{ _('Timestamp') }}</th>
+      <th>{{ _('Author') }}</th>
+      <th>{{ _('Log Message') }}</th>
+    </tr>
+  </thead>
+  <tbody>
+    {% for rev in group_revisions %}
+      <tr>
+        <td nowrap="nowrap">
+          {{ h.radio('selected1', rev.id, checked=(loop.first)) }}
+          {{ h.radio('selected2', rev.id, checked=(loop.last)) }}
+        </td>
+        <td class="dataset-label">
+          {% link_for rev.id | truncate(6), controller='revision', action='read', id=rev.id %}
+        </td>
+        <td class="dataset-details">
+          {{ h.render_datetime(rev.timestamp, with_hours=True) }}
+        </td>
+        <td class="dataset-details">{{ h.linked_user(rev.author) }}</td>
+        <td class="dataset-details">{{ rev.message }}</td>
+      </tr>
+      {% endfor %}
+  </tbody>
+</table>

ckan/templates/package/history.html

@@ -0,0 +1,12 @@
+{% extends "package/read_base.html" %}
+
+{% block subtitle %}{{ _('History') }} - {{ c.pkg_dict.title or c.pkg_dict.name }}{% endblock %}
+
+{% block primary_content_inner %}
+  <section class="module-content">
+    <h1 class="hide-heading">{{ _('History') }}</h1>
+    {% block package_history_revisions %}
+      {% snippet "package/snippets/history_revisions.html", pkg_dict=pkg, pkg_revisions=c.pkg_revisions %}
+    {% endblock %}
+  </section>
+{% endblock %}

ckan/templates/package/read_base.html

@@ -17,23 +17,6 @@
 {% endblock %}
 
 {% block primary_content %}
-  {% block package_revision_info %}
-    {% if c.pkg_revision_id %}
-      <div class="module info">
-        <p class="module-content">
-          {% set timestamp = h.render_datetime(c.pkg_revision_timestamp, with_hours=True) %}
-          {% set url = h.url(controller='package', action='read', id=pkg.name) %}
-
-          {% if c.pkg_revision_not_latest %}
-            {% trans timestamp=timestamp, url=url %}This is an old revision of this dataset, as edited at {{ timestamp }}. It may differ significantly from the <a href="{{ url }}">current revision</a>.{% endtrans %}
-          {% else %}
-            {% trans timestamp=timestamp %}This is the current revision of this dataset, as edited at {{ timestamp }}.{% endtrans %}
-          {% endif %}
-        </div>
-      </div>
-    {% endif %}
-  {% endblock %}
-
   <article class="module prose">
     {% block page_header %}
       <header class="module-content page-header">
@@ -46,6 +29,24 @@
         </ul>
       </header>
     {% endblock %}
+
+    {% block package_revision_info %}
+      {% if c.pkg_revision_id %}
+        <div class="module info alert alert-info">
+          <p class="module-content">
+            {% set timestamp = h.render_datetime(c.pkg_revision_timestamp, with_hours=True) %}
+            {% set url = h.url(controller='package', action='read', id=pkg.name) %}
+
+            {% if c.pkg_revision_not_latest %}
+              {% trans timestamp=timestamp, url=url %}This is an old revision of this dataset, as edited at {{ timestamp }}. It may differ significantly from the <a href="{{ url }}">current revision</a>.{% endtrans %}
+            {% else %}
+              {% trans timestamp=timestamp %}This is the current revision of this dataset, as edited at {{ timestamp }}.{% endtrans %}
+            {% endif %}
+          </p>
+        </div>
+      {% endif %}
+    {% endblock %}
+
     {% block primary_content_inner %}{% endblock %}
   </article>
 {% endblock %}

ckan/templates/package/search.html

@@ -57,7 +57,6 @@
                 {%- if c.translated_fields and c.translated_fields.has_key((field,value)) -%}
                   {{ c.translated_fields[(field,value)] }}
                 {%- else -%}
-                  {{ value }}
                   {{ h.list_dict_filter(search_facets_items , 'name', 'display_name', value) }}
                 {%- endif %}
                 <a href="{{ c.remove_field(field, value) }}" class="remove" title="{{ _('Remove') }}"><i class="icon-remove"></i></a>

ckan/templates/package/snippets/history_revisions.html

@@ -0,0 +1,12 @@
+{% import 'macros/form.html' as form %}
+
+<form class="dataset-form dataset-resource-form form-horizontal" method="post" data-module="basic-form">
+
+  {{ form.errors(error_summary) }}
+
+  <input type="hidden" name="pkg_name" value="{{ pkg_dict.name }}"/>
+  {% snippet 'package/snippets/revisions_table.html', pkg_dict=pkg_dict, pkg_revisions=pkg_revisions %}
+
+  <button class="btn btn-primary" name="diff" value="go-resources" type="submit">{{ _('Compare') }}</button>
+
+</form>