Merge branch 'master' of github.com:okfn/ckan into feature-1515-activ…

…ity-streams

Conflicts:
	ckan/model/meta.py

ckan/config/environment.py

@@ -81,6 +81,10 @@ def find_controller(self, controller):
                           'ckan.site_id for SOLR search-index rebuild to work.'
         config['ckan.site_id'] = ckan_host
 
+    # Check if SOLR schema is compatible
+    from ckan.lib.search import check_solr_schema_version
+    check_solr_schema_version()
+
     config['routes.map'] = make_map()
     config['pylons.app_globals'] = app_globals.Globals()
     config['pylons.h'] = ckan.lib.helpers

ckan/config/solr/CHANGELOG.txt

@@ -0,0 +1,12 @@
+CKAN SOLR schemas changelog
+===========================
+
+v1.3 - (ckan>=1.5.1)
+--------------------
+* Use the index_id (hash of dataset id + site_id) as uniqueKey (#1430)
+* Store extras (#1455)
+* Store dataset creation and modification date (#191)
+
+v1.2 - (ckan<=1.5)
+--------------------
+* Original version

ckan/config/solr/README.txt

@@ -0,0 +1,30 @@
+CKAN SOLR schemas
+=================
+
+This folder contains the latest and previous versions of the SOLR XML
+schema files used by CKAN. These can be use on the SOLR server to
+override the default SOLR schema. Please note that not all schemas are 
+backwards compatible with old CKAN versions. Check the CHANGELOG.txt file
+in this same folder to check which version of the schema should you use
+depending on the CKAN version you are using.
+
+Developers, when pushing changes to the SOLR schema:
+
+* Note that updates on the schema are only release based, i.e. all changes
+  in the schema between releases will be part of the same new version of
+  the schema.
+
+* Name the new version of the file using the following convention::
+
+    schema-<version>.xml
+
+* Update the `version` attribute of the `schema` tag in the new file::
+
+    <schema name="ckan" version="<version>">
+
+* Update the SUPPORTED_SCHEMA_VERSIONS list in `ckan/lib/search/__init__.py`
+  Consider if the changes introduced are or are not compatible with
+  previous schema versions.
+
+* Update the CHANGELOG.txt file with the new version, the CKAN version
+  required and changes made to the schema.

ckan/config/schema.xml → ckan/config/solr/schema-1.2.xml

@@ -131,10 +131,10 @@
     <field name="linked_from" type="text" indexed="true" stored="false" multiValued="true"/>
     <field name="child_of" type="text" indexed="true" stored="false" multiValued="true"/>
     <field name="parent_of" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="extras_*" type="text" indexed="true" stored="false" multiValued="true"/>
 
     <field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
 
-    <dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
     <dynamicField name="*" type="string" indexed="true"  stored="false"/>
 </fields>
 

ckan/config/solr/schema-1.3.xml

@@ -0,0 +1,166 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="ckan" version="1.3">
+
+<types>
+    <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+    <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
+    <fieldtype name="binary" class="solr.BinaryField"/>
+    <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>
+    <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0"/>
+
+    <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+        <analyzer type="index">
+            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+            <!-- in this example, we will only use synonyms at query time
+            <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
+            -->
+            <!-- Case insensitive stop word removal.
+              add enablePositionIncrements=true in both the index and query
+              analyzers to leave a 'gap' for more accurate phrase queries.
+            -->
+            <filter class="solr.StopFilterFactory"
+                ignoreCase="true"
+                words="stopwords.txt"
+                enablePositionIncrements="true"
+                />
+            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+            <filter class="solr.LowerCaseFilterFactory"/>
+            <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+        </analyzer>
+        <analyzer type="query">
+            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+            <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+            <filter class="solr.StopFilterFactory"
+                ignoreCase="true"
+                words="stopwords.txt"
+                enablePositionIncrements="true"
+                />
+            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+            <filter class="solr.LowerCaseFilterFactory"/>
+            <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+        </analyzer>
+    </fieldType>
+
+
+    <!-- A general unstemmed text field - good if one does not know the language of the field -->
+    <fieldType name="textgen" class="solr.TextField" positionIncrementGap="100">
+        <analyzer type="index">
+            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+            <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
+            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+            <filter class="solr.LowerCaseFilterFactory"/>
+        </analyzer>
+        <analyzer type="query">
+            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+            <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+            <filter class="solr.StopFilterFactory"
+                ignoreCase="true"
+                words="stopwords.txt"
+                enablePositionIncrements="true"
+                />
+            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+            <filter class="solr.LowerCaseFilterFactory"/>
+        </analyzer>
+    </fieldType>
+</types>
+
+
+<fields>
+    <field name="index_id" type="string" indexed="true" stored="true" required="true" />
+    <field name="id" type="string" indexed="true" stored="true" required="true" />
+    <field name="site_id" type="string" indexed="true" stored="true" required="true" />
+    <field name="title" type="text" indexed="true" stored="true" />
+    <field name="entity_type" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="state" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="name" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="revision_id" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="version" type="string" indexed="true" stored="true" />
+    <field name="url" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="ckan_url" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="download_url" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="notes" type="text" indexed="true" stored="true"/>
+    <field name="author" type="textgen" indexed="true" stored="true" />
+    <field name="author_email" type="textgen" indexed="true" stored="true" />
+    <field name="maintainer" type="textgen" indexed="true" stored="true" />
+    <field name="maintainer_email" type="textgen" indexed="true" stored="true" />
+    <field name="license" type="string" indexed="true" stored="true" />
+    <field name="license_id" type="string" indexed="true" stored="true" />
+    <field name="ratings_count" type="int" indexed="true" stored="false" />
+    <field name="ratings_average" type="float" indexed="true" stored="false" />
+    <field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
+
+    <field name="res_description" type="textgen" indexed="true" stored="true" multiValued="true"/>
+    <field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
+
+    <!-- catchall field, containing all other searchable text fields (implemented
+         via copyField further on in this schema  -->
+    <field name="text" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="urls" type="text" indexed="true" stored="false" multiValued="true"/>
+
+    <field name="depends_on" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="dependency_of" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="derives_from" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="has_derivation" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="links_to" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="linked_from" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="child_of" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="parent_of" type="text" indexed="true" stored="false" multiValued="true"/>
+
+    <field name="metadata_created" type="date" indexed="true" stored="true" multiValued="false"/>
+    <field name="metadata_modified" type="date" indexed="true" stored="true" multiValued="false"/>
+
+    <field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
+
+    <dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
+    <dynamicField name="*" type="string" indexed="true"  stored="false"/>
+</fields>
+
+<uniqueKey>index_id</uniqueKey>
+<defaultSearchField>text</defaultSearchField>
+<solrQueryParser defaultOperator="AND"/>
+
+<copyField source="url" dest="urls"/>
+<copyField source="ckan_url" dest="urls"/>
+<copyField source="download_url" dest="urls"/>
+<copyField source="res_url" dest="urls"/>
+<copyField source="extras_*" dest="text"/>
+<copyField source="urls" dest="text"/>
+<copyField source="name" dest="text"/>
+<copyField source="title" dest="text"/>
+<copyField source="text" dest="text"/>
+<copyField source="license" dest="text"/>
+<copyField source="notes" dest="text"/>
+<copyField source="tags" dest="text"/>
+<copyField source="groups" dest="text"/>
+<copyField source="res_description" dest="text"/>
+<copyField source="maintainer" dest="text"/>
+<copyField source="author" dest="text"/>
+
+</schema>

ckan/controllers/admin.py

@@ -16,7 +16,7 @@ class AdminController(BaseController):
     def __before__(self, action, **params):
         super(AdminController, self).__before__(action, **params)
         if not ckan.authz.Authorizer().is_sysadmin(unicode(c.user)):
-            abort(401, 'Need to be system administrator to administer')        
+            abort(401, _('Need to be system administrator to administer'))        
         c.revision_change_state_allowed = (
             c.user and
             self.authorizer.is_authorized(c.user, model.Action.CHANGE_STATE,
@@ -102,7 +102,7 @@ def action_save_form(users_or_authz_groups):
 
             # finally commit the change to the database
             model.Session.commit()
-            h.flash_success("Changes Saved")
+            h.flash_success(_("Changes Saved"))
 
         if ('save' in request.POST):
             action_save_form('users')
@@ -139,7 +139,7 @@ def action_add_form(users_or_authz_groups):
                 user_object = model.User.by_name(new_user)
                 if user_object==None:
                     # The submitted user does not exist. Bail with flash message
-                    h.flash_error('unknown user:' + str (new_user))
+                    h.flash_error(_('unknown user:') + str (new_user))
                 else:
                     # Whenever our desired state is different from our current state, change it.
                     for (r,val) in desired_roles.items():
@@ -149,14 +149,14 @@ def action_add_form(users_or_authz_groups):
                         else:
                             if (r in current_roles):
                                 model.remove_user_from_role(user_object, r, model.System())
-                    h.flash_success("User Added")
+                    h.flash_success(_("User Added"))
 
             elif users_or_authz_groups=='authz_groups':
                 current_roles = [uor.role for uor in current_uors if ( uor.authorized_group and uor.authorized_group.name == new_user )]
                 user_object = model.AuthorizationGroup.by_name(new_user)
                 if user_object==None:
                     # The submitted user does not exist. Bail with flash message
-                    h.flash_error('unknown authorization group:' + str (new_user))
+                    h.flash_error(_('unknown authorization group:') + str (new_user))
                 else:
                     # Whenever our desired state is different from our current state, change it.
                     for (r,val) in desired_roles.items():
@@ -166,7 +166,7 @@ def action_add_form(users_or_authz_groups):
                         else:
                             if (r in current_roles):
                                 model.remove_authorization_group_from_role(user_object, r, model.System())
-                    h.flash_success("Authorization Group Added")
+                    h.flash_success(_("Authorization Group Added"))
 
 
             else:
@@ -265,7 +265,7 @@ def trash(self):
                         for r in revisions:
                             affected_pkgs = set(r.packages).difference(set(c.deleted_packages))
                             if affected_pkgs:
-                                msg = _('Cannot purge package %s as ' + \
+                                msg = _('Cannot purge package %s as '
                                     'associated revision %s includes non-deleted packages %s')
                                 msg = msg % (pkg.id, r.id, [pkg.id for r in affected_pkgs])
                                 msgs.append(msg)
@@ -284,12 +284,11 @@ def trash(self):
                         # Ensure that whatever 'head' pointer is used gets moved down to the next revision
                         model.repo.purge_revision(revision, leave_record=False)
                     except Exception, inst:
-                        msg = 'Problem purging revision %s: %s' % (id,
-                                inst)
+                        msg = _('Problem purging revision %s: %s') % (id, inst)
                         msgs.append(msg)
                 h.flash_success(_('Purge complete'))
             else:
-                msgs.append('Action not implemented.')
+                msgs.append(_('Action not implemented.'))
 
             for msg in msgs:
                 h.flash_error(msg)

ckan/controllers/home.py

@@ -14,6 +14,7 @@
 from ckan.lib.base import *
 import ckan.lib.stats
 from ckan.lib.hash import get_redirect
+from ckan.lib.helpers import url_for
 
 class HomeController(BaseController):
     repo = model.repo
@@ -60,6 +61,27 @@ def index(self):
             c.package_count = 0
             c.groups = []
 
+        if c.userobj is not None:
+            msg = None
+            url = url_for(controller='user', action='edit')
+            is_google_id = \
+                c.userobj.name.startswith('https://www.google.com/accounts/o8/id')
+            if not c.userobj.email and (is_google_id and not c.userobj.fullname):
+                msg = _('Please <a href="%s">update your profile</a>'
+                    ' and add your email address and your full name. %s uses'
+                    ' your email address if you need to reset your'
+                    ' password.''') % (url, g.site_title)
+            elif not c.userobj.email:
+                msg = _('Please <a href="%s">update your profile</a>'
+                    ' and add your email address. %s uses your email address'
+                    ' if you need to reset your password.') \
+                    % (url, g.site_title)
+            elif is_google_id and not c.userobj.fullname:
+                msg = _('Please <a href="%s">update your profile</a>'
+                    ' and add your full name.') % (url)
+            if msg:
+                h.flash_notice(msg, allow_html=True)
+
         return render('home/index.html')
 
     def license(self):
@@ -84,7 +106,7 @@ def locale(self):
                 # e.g. babel.Locale.parse('no').get_display_name() returns None
                 h.flash_notice(_("Language has been set to: English"))
             except:
-                h.flash_notice("Language has been set to: English")
+                h.flash_notice(_("Language has been set to: English"))
         else:
             abort(400, _("No language given!"))
         return_to = get_redirect()