Merge branch 'master' into clean-models

Conflicts:
	ckan/model/__init__.py
	ckan/model/resource.py

ckan/ckan_nose_plugin.py

@@ -30,7 +30,6 @@ def startContext(self, ctx):
             # init_db is run at the start of every class because
             # when you use an in-memory sqlite db, it appears that
             # the db is destroyed after every test when you Session.Remove().
-            model.repo.init_db()
 
             ## This is to make sure the configuration is run again.
             ## Plugins use configure to make their own tables and they
@@ -40,6 +39,7 @@ def startContext(self, ctx):
             for plugin in PluginImplementations(IConfigurable):
                 plugin.configure(config)
 
+            model.repo.init_db()
 
     def options(self, parser, env):
         parser.add_option(

ckan/config/environment.py

@@ -30,10 +30,14 @@ class _Helpers(object):
     def __init__(self, helpers, restrict=True):
         functions = {}
         allowed = helpers.__allowed_functions__
+        # list of functions due to be depreciated
+        self.depreciated = []
 
         for helper in dir(helpers):
-            if restrict and (helper not in allowed):
-                continue
+            if helper not in allowed:
+                self.depreciated.append(helper)
+                if restrict:
+                    continue
             functions[helper] = getattr(helpers, helper)
         self.functions = functions
 
@@ -46,6 +50,8 @@ def __init__(self, helpers, restrict=True):
                     raise Exception('overwritting extra helper %s' % helper)
                 extra_helpers.append(helper)
                 functions[helper] = helpers[helper]
+        # logging
+        self.log = logging.getLogger('ckan.helpers')
 
     @classmethod
     def null_function(cls, *args, **kw):
@@ -57,10 +63,21 @@ def null_function(cls, *args, **kw):
     def __getattr__(self, name):
         ''' return the function/object requested '''
         if name in self.functions:
+            if name in self.depreciated:
+                msg = 'Template helper function `%s` is depriciated' % name
+                self.log.warn(msg)
             return self.functions[name]
         else:
-            log = logging.getLogger('ckan.helpers')
-            log.critical('Helper function `%s` could not be found (missing extension?)' % name)
+            if name in self.depreciated:
+                msg = 'Template helper function `%s` is not available ' \
+                      'as it has been depriciated.\nYou can enable it ' \
+                      'by setting ckan.restrict_template_vars = true ' \
+                      'in your .ini file.' % name
+                self.log.critical(msg)
+            else:
+                msg = 'Helper function `%s` could not be found\n ' \
+                      '(are you missing an extension?)' % name
+                self.log.critical(msg)
             return self.null_function
 
 
@@ -171,9 +188,8 @@ def template_loaded(template):
     ckan_db = os.environ.get('CKAN_DB') 
 
     if ckan_db:
-        engine = sqlalchemy.create_engine(ckan_db)
-    else:
-        engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
+        config['sqlalchemy.url'] = ckan_db
+    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
 
     if not model.meta.engine:
         model.init_model(engine)

ckan/config/middleware.py

@@ -1,8 +1,11 @@
 """Pylons middleware initialization"""
 import urllib
-import logging 
+import urllib2
+import logging
 import json
+import hashlib
 
+import sqlalchemy as sa
 from beaker.middleware import CacheMiddleware, SessionMiddleware
 from paste.cascade import Cascade
 from paste.registry import RegistryManager
@@ -14,6 +17,7 @@
 from routes.middleware import RoutesMiddleware
 from repoze.who.config import WhoConfig
 from repoze.who.middleware import PluggableAuthenticationMiddleware
+
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import IMiddleware
 from ckan.lib.i18n import get_locales
@@ -130,6 +134,8 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     if asbool(config.get('ckan.page_cache_enabled')):
         app = PageCacheMiddleware(app, config)
 
+    # Tracking add config option
+    app = TrackingMiddleware(app, config)
     return app
 
 class I18nMiddleware(object):
@@ -277,3 +283,40 @@ def _start_response(status, response_headers, exc_info=None):
             pipe.rpush(key, page_string)
             pipe.execute()
         return page
+
+
+class TrackingMiddleware(object):
+
+    def __init__(self, app, config):
+        self.app = app
+        self.engine = sa.create_engine(config.get('sqlalchemy.url'))
+
+
+    def __call__(self, environ, start_response):
+        path = environ['PATH_INFO']
+        if path == '/_tracking':
+            # do the tracking
+            # get the post data
+            payload = environ['wsgi.input'].read()
+            parts = payload.split('&')
+            data = {}
+            for part in parts:
+                k, v = part.split('=')
+                data[k] = urllib2.unquote(v).decode("utf8")
+            start_response('200 OK', [('Content-Type', 'text/html')])
+            # we want a unique anonomized key for each user so that we do
+            # not count multiple clicks from the same user.
+            key = ''.join([
+                environ['HTTP_USER_AGENT'],
+                environ['REMOTE_ADDR'],
+                environ['HTTP_ACCEPT_LANGUAGE'],
+                environ['HTTP_ACCEPT_ENCODING'],
+            ])
+            key = hashlib.md5(key).hexdigest()
+            # store key/data here
+            sql = '''INSERT INTO tracking_raw
+                     (user_key, url, tracking_type)
+                     VALUES (%s, %s, %s)'''
+            self.engine.execute(sql, key, data.get('url'), data.get('type'))
+            return []
+        return self.app(environ, start_response)

ckan/config/routing.py

@@ -57,6 +57,7 @@ def make_map():
             'resource',
             'tag',
             'group',
+            'related',
             'authorizationgroup',
             'revision',
             'licenses',
@@ -151,6 +152,11 @@ def make_map():
     ##map.connect('/package/new', controller='package_formalchemy', action='new')
     ##map.connect('/package/edit/{id}', controller='package_formalchemy', action='edit')
 
+    with SubMapper(map, controller='related') as m:
+        m.connect('related_edit', '/related/{id}/edit', action='edit')
+        m.connect('related_list', '/dataset/{id}/related', action='list')
+        m.connect('related_read', '/dataset/{id}/related/{related_id}', action='read')
+
     with SubMapper(map, controller='package') as m:
         m.connect('/dataset', action='search')
         m.connect('/dataset/{action}',
@@ -183,6 +189,7 @@ def make_map():
         m.connect('/dataset/{id}.{format}', action='read')
         m.connect('/dataset/{id}', action='read')
         m.connect('/dataset/{id}/resource/{resource_id}', action='resource_read')
+        m.connect('/dataset/{id}/resource/{resource_id}/embed', action='resource_embedded_dataviewer')
 
     # group
     map.redirect('/groups', '/group')
@@ -210,7 +217,6 @@ def make_map():
     register_package_plugins(map)
     register_group_plugins(map)
 
-
     # authz group
     map.redirect('/authorizationgroups', '/authorizationgroup')
     map.redirect('/authorizationgroups/{url:.*}', '/authorizationgroup/{url}')

ckan/config/solr/schema-1.4.xml

@@ -0,0 +1,175 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="ckan" version="1.4">
+
+<types>
+    <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+    <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
+    <fieldtype name="binary" class="solr.BinaryField"/>
+    <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>
+    <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0"/>
+
+    <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+        <analyzer type="index">
+            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+            <!-- in this example, we will only use synonyms at query time
+            <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
+            -->
+            <!-- Case insensitive stop word removal.
+              add enablePositionIncrements=true in both the index and query
+              analyzers to leave a 'gap' for more accurate phrase queries.
+            -->
+            <filter class="solr.StopFilterFactory"
+                ignoreCase="true"
+                words="stopwords.txt"
+                enablePositionIncrements="true"
+                />
+            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+            <filter class="solr.LowerCaseFilterFactory"/>
+            <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+        </analyzer>
+        <analyzer type="query">
+            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+            <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+            <filter class="solr.StopFilterFactory"
+                ignoreCase="true"
+                words="stopwords.txt"
+                enablePositionIncrements="true"
+                />
+            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+            <filter class="solr.LowerCaseFilterFactory"/>
+            <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+        </analyzer>
+    </fieldType>
+
+
+    <!-- A general unstemmed text field - good if one does not know the language of the field -->
+    <fieldType name="textgen" class="solr.TextField" positionIncrementGap="100">
+        <analyzer type="index">
+            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+            <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
+            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+            <filter class="solr.LowerCaseFilterFactory"/>
+        </analyzer>
+        <analyzer type="query">
+            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+            <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+            <filter class="solr.StopFilterFactory"
+                ignoreCase="true"
+                words="stopwords.txt"
+                enablePositionIncrements="true"
+                />
+            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+            <filter class="solr.LowerCaseFilterFactory"/>
+        </analyzer>
+    </fieldType>
+</types>
+
+
+<fields>
+    <field name="index_id" type="string" indexed="true" stored="true" required="true" />
+    <field name="id" type="string" indexed="true" stored="true" required="true" />
+    <field name="site_id" type="string" indexed="true" stored="true" required="true" />
+    <field name="title" type="text" indexed="true" stored="true" />
+    <field name="entity_type" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="state" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="name" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="revision_id" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="version" type="string" indexed="true" stored="true" />
+    <field name="url" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="ckan_url" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="download_url" type="string" indexed="true" stored="true" omitNorms="true" />
+    <field name="notes" type="text" indexed="true" stored="true"/>
+    <field name="author" type="textgen" indexed="true" stored="true" />
+    <field name="author_email" type="textgen" indexed="true" stored="true" />
+    <field name="maintainer" type="textgen" indexed="true" stored="true" />
+    <field name="maintainer_email" type="textgen" indexed="true" stored="true" />
+    <field name="license" type="string" indexed="true" stored="true" />
+    <field name="license_id" type="string" indexed="true" stored="true" />
+    <field name="ratings_count" type="int" indexed="true" stored="false" />
+    <field name="ratings_average" type="float" indexed="true" stored="false" />
+    <field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
+
+    <field name="res_description" type="textgen" indexed="true" stored="true" multiValued="true"/>
+    <field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
+
+    <!-- catchall field, containing all other searchable text fields (implemented
+         via copyField further on in this schema  -->
+    <field name="text" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="urls" type="text" indexed="true" stored="false" multiValued="true"/>
+
+    <field name="depends_on" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="dependency_of" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="derives_from" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="has_derivation" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="links_to" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="linked_from" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="child_of" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="parent_of" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="views_total" type="int" indexed="true" stored="false"/>
+    <field name="views_recent" type="int" indexed="true" stored="false"/>
+    <field name="recources_accessed_total" type="int" indexed="true" stored="false"/>
+    <field name="recources_accessed_recent" type="int" indexed="true" stored="false"/>
+
+    <field name="metadata_created" type="date" indexed="true" stored="true" multiValued="false"/>
+    <field name="metadata_modified" type="date" indexed="true" stored="true" multiValued="false"/>
+
+    <field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
+
+    <!-- Copy the title field into titleString, and treat as a string
+         (rather than text type).  This allows us to sort on the titleString -->
+    <field name="titleString" type="string" indexed="true" stored="false" />
+    <copyField source="title" dest="titleString"/>
+
+    <dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
+    <dynamicField name="*" type="string" indexed="true"  stored="false"/>
+</fields>
+
+<uniqueKey>index_id</uniqueKey>
+<defaultSearchField>text</defaultSearchField>
+<solrQueryParser defaultOperator="AND"/>
+
+<copyField source="url" dest="urls"/>
+<copyField source="ckan_url" dest="urls"/>
+<copyField source="download_url" dest="urls"/>
+<copyField source="res_url" dest="urls"/>
+<copyField source="extras_*" dest="text"/>
+<copyField source="urls" dest="text"/>
+<copyField source="name" dest="text"/>
+<copyField source="title" dest="text"/>
+<copyField source="text" dest="text"/>
+<copyField source="license" dest="text"/>
+<copyField source="notes" dest="text"/>
+<copyField source="tags" dest="text"/>
+<copyField source="groups" dest="text"/>
+<copyField source="res_description" dest="text"/>
+<copyField source="maintainer" dest="text"/>
+<copyField source="author" dest="text"/>
+
+</schema>