[controllers,lib,templates][l] faceted browsing for packages

ckan/config/routing.py

@@ -220,9 +220,10 @@ def make_map():
 
     map.redirect("/packages", "/package")
     map.redirect("/packages/{url:.*}", "/package/{url}")
-    map.connect('/package/', controller='package', action='index')
+    map.connect('/package', controller='package', action='search')
+    map.connect('/package/', controller='package', action='search')
     map.connect('/package/search', controller='package', action='search')
-    map.connect('/package/list', controller='package', action='list')
+    map.connect('/package/list', controller='package', action='search')
     map.connect('/package/new', controller='package', action='new')
     map.connect('/package/new_title_to_slug', controller='package', action='new_title_to_slug')
     map.connect('/package/autocomplete', controller='package', action='autocomplete')

ckan/controllers/package.py

@@ -1,5 +1,6 @@
 import logging
 import urlparse
+import urllib
 
 from sqlalchemy.orm import eagerload_all
 from sqlalchemy import or_
@@ -23,67 +24,65 @@ class PackageController(BaseController):
     authorizer = ckan.authz.Authorizer()
     extensions = ExtensionPoint(IPackageController)
 
-    def index(self):
-        query = ckan.authz.Authorizer().authorized_query(c.user, model.Package)
-        c.package_count = query.count()
-        return render('package/index.html')
-
-    @proxy_cache()
-    def list(self):
-        query = ckan.authz.Authorizer().authorized_query(c.user, model.Package)
-        query = query.options(eagerload_all('package_tags.tag'))
-        query = query.options(eagerload_all('package_resources_all'))
-        c.page = h.AlphaPage(
-            collection=query,
-            page=request.params.get('page', 'A'),
-            alpha_attribute='title',
-            other_text=_('Other'),
-        )
-        return render('package/list.html')
-
     def search(self):        
-        c.q = request.params.get('q') # unicode format (decoded from utf8)
+        q = c.q = request.params.get('q') # unicode format (decoded from utf8)
         c.open_only = request.params.get('open_only')
         c.downloadable_only = request.params.get('downloadable_only')
-        if c.q:
-            c.query_error = False
-            page = int(request.params.get('page', 1))
-            limit = 20
-            query = query_for(model.Package)
-            try:
-                query.run(query=c.q,
-                          limit=limit,
-                          offset=(page-1)*limit,
-                          return_objects=True,
-                          filter_by_openness=c.open_only,
-                          filter_by_downloadable=c.downloadable_only,
-                          username=c.user)
-
-                c.page = h.Page(
-                    collection=query.results,
-                    page=page,
-                    item_count=query.count,
-                    items_per_page=limit
-                )
-                c.page.items = query.results
-            except SearchError, se:
-                c.query_error = True
-                c.page = h.Page(collection=[])
-
-            # tag search
-            c.tag_limit = 25
-            query = query_for('tag', backend='sql')
-            try:
-                query.run(query=c.q,
-                          return_objects=True,
-                          limit=c.tag_limit,
-                          username=c.user)
-                c.tags = query.results
-                c.tags_count = query.count
-            except SearchError, se:
-                c.tags = []
-                c.tags_count = 0
+        if c.q is None or len(c.q.strip()) == 0:
+            q = '*:*'
+        c.query_error = False
+        page = int(request.params.get('page', 1))
+        limit = 20
+        query = query_for(model.Package)
 
+        def drill_down_url(**by):
+            url = h.url_for(controller='package', action='search')
+            param = request.params.items()
+            for k, v in by.items():
+                if not (k, v) in param:
+                    param.append((k, v.encode('utf-8')))
+            return url + '?' + urllib.urlencode(param)
+
+        c.drill_down_url = drill_down_url 
+
+        def remove_field(key, value):
+            url = h.url_for(controller='package', action='search')
+            param = request.params.items()
+            param.remove((key, value))
+            return url + '?' + urllib.urlencode(
+                [(k, v.encode('utf-8')) for k, v in param])
+
+        c.remove_field = remove_field
+
+        try:
+            c.fields = []
+            for (param, value) in request.params.items():
+                if not param in ['q', 'open_only', 'downloadable_only', 'page']:
+                    c.fields.append((param, value))
+
+            query.run(query=q,
+                      fields=c.fields,
+                      facet_by=g.facets,
+                      limit=limit,
+                      offset=(page-1)*limit,
+                      return_objects=True,
+                      filter_by_openness=c.open_only,
+                      filter_by_downloadable=c.downloadable_only,
+                      username=c.user)
+
+            c.page = h.Page(
+                collection=query.results,
+                page=page,
+                item_count=query.count,
+                items_per_page=limit
+            )
+            c.facets = query.facets
+            c.page.items = query.results
+        except SearchError, se:
+            c.query_error = True
+            c.facets = {}
+            c.page = h.Page(collection=[])
+
         return render('package/search.html')
 
     def _pkg_cache_key(self, pkg):

ckan/lib/app_globals.py

@@ -19,6 +19,8 @@ def __init__(self):
         self.site_logo = config.get('ckan.site_logo', '/images/ckan_logo_fullname_long.png')
         self.site_url = config.get('ckan.site_url', 'http://www.ckan.net')
 
+        self.facets = config.get('search.facets', 'groups tags res_format license').split()
+
         # has been setup in load_environment():
         self.site_id = config.get('ckan.site_id')
-
+

ckan/lib/cli.py

@@ -276,6 +276,7 @@ class SearchIndexCommand(CkanCommand):
 
     def command(self):
         self._load_config()
+        from ckan.lib.search import rebuild
 
         if not self.args:
             # default to run

ckan/lib/helpers.py

@@ -42,6 +42,22 @@ def subnav_link(c, text, action, **kwargs):
         class_=('active' if c.action == action else '')
     )
 
+def facet_items(c, name, limit=10):
+    from pylons import request
+    if not c.facets or not c.facets.get(name): 
+        return []
+    facets = []
+    for k, v in c.facets.get(name).items():
+        if not len(k.strip()):
+            continue
+        if not (name, k) in request.params.items():
+            facets.append((k, v))
+    return sorted(facets, key=lambda (k, v): v, reverse=True)[:limit]
+
+def facet_title(name):
+    from pylons import config 
+    return config.get('search.facets.%s.title' % name, name.capitalize())
+
 def am_authorized(c, action, domain_object=None):
     from ckan.authz import Authorizer
     if domain_object is None:
@@ -63,12 +79,19 @@ def linked_user(user):
                        url_for(controller='user', action='read', id=_name))
     return user
 
+def group_name_to_title(name):
+    from ckan import model
+    group = model.Group.by_name(name)
+    if group is not None:
+        return group.title
+    return name
+
 def markdown_extract(text):
     if (text is None) or (text == ''):
         return ''
     html = fromstring(markdown(text))
     plain = html.xpath("string()")
-    return unicode(truncate(plain, length=270, indicator='...', whole_word=True))
+    return unicode(truncate(plain, length=190, indicator='...', whole_word=True))
 
 def icon_url(name):
     return '/images/icons/%s.png' % name

ckan/lib/search/__init__.py

@@ -46,6 +46,8 @@ def rebuild():
     package_index.clear()
     for pkg in model.Session.query(model.Package).all():
         package_index.insert_entity(pkg)
+    model.Session.commit()
+
 
 def query_for(_type, backend=None):
     """ Query for entities of a specified type (name, class, instance). """

ckan/lib/search/common.py

@@ -101,13 +101,15 @@ def _format_results(self):
                 attr_name = self.options.ref_entity_with_attr
                 self.results = [getattr(entity, attr_name) for entity in self.results]
 
-    def run(self, query=None, terms=[], fields={}, options=None, **kwargs):
+    def run(self, query=None, terms=[], fields={}, facet_by=[], options=None, **kwargs):
         if options is None:
             options = QueryOptions(**kwargs) 
         else:
             options.update(kwargs)
         self.options = options
         self.options.validate()
+        self.facet_by = facet_by
+        self.facets = dict()
         self.query = QueryParser(query, terms, fields)
         self.query.validate()
         self._run()

ckan/lib/search/solr_.py

@@ -56,11 +56,16 @@ def _run(self):
 
         # show only results from this CKAN instance:
         fq = fq + " +site_id:\"%s\"" % config.get('ckan.site_id')
-        
+
         conn = self.backend.make_connection()
         try:
             data = conn.query(self.query.query,
                               fq=fq, 
+                              # make sure data.facet_counts is set:
+                              facet='true',                               
+                              facet_limit=50,
+                              facet_field=self.facet_by,
+                              facet_mincount=1,
                               start=self.options.offset, 
                               rows=self.options.limit,
                               fields='id,score', 
@@ -78,6 +83,7 @@ def _run(self):
         result_ids = [(r.get('id')) for r in data.results]
         q = authz.Authorizer().authorized_query(self.options.username, model.Package)
         q = q.filter(model.Package.id.in_(result_ids))
+        self.facets = data.facet_counts.get('facet_fields', {})
         self.results = q.all()
 
 

ckan/lib/search/sql.py

@@ -267,9 +267,8 @@ def update_dict(self, pkg_dict):
     def remove_dict(self, pkg_dict):
         if not 'id' in pkg_dict or not 'name' in pkg_dict:
             return 
-        sql = "DELETE FROM package_search WHERE package_id=%%s"
-        params.append(pkg_dict['id'])
-        self._run_sql(sql, params)
+        sql = "DELETE FROM package_search WHERE package_id=%s"
+        self._run_sql(sql, [pkg_dict.get('id')])
         log.debug("Delete entry %s from index" % pkg_dict.get('id'))
 
 

ckan/public/css/ckan.css

@@ -1,3 +1,12 @@
+body, input, textarea, .page-title span, .pingback a.url {
+    font-family: "Lucida Grande", Lucida, Verdana, sans-serif;
+}
+
+#content, #content input, #content textarea {
+    font-size: 14px;
+    line-height: 20px;
+}
+
 a { 
   color: #b00; 
   text-decoration: none;
@@ -223,6 +232,75 @@ a:hover {
   margin-bottom: 10px;
 }
 
+/* ==========================
+ * Facets
+ */
+
+.facet-box {
+
+}
+
+.facet-box h2 {
+    color: #000;
+    font-size: 1.2em;
+    font-weight: bold;
+    margin-top: 1em;
+}
+
+.facet-options {
+    margin-top: 0.5em;
+}
+
+.facet-options li {
+    padding-top: 0.2em;
+    font-size: 1.2em; 
+    color: #000; 
+}
+
+.register-link {
+    padding-top: 10px;
+}
+
+.register-link a {
+    color: white; 
+    background: #199150;
+    font-weight: bold;
+    padding: 5px;
+    width: 100%;
+    font-size: 1.3em;
+    -moz-border-radius: 2px;
+    -webkit-border-radius: 2px;
+    border-radius: 2px;
+
+}
+
+.package-search-filters {
+    margin-top: 15px;
+}
+
+.search-field {
+    display: inline-block;
+    margin-right: 5px;
+    margin-bottom: 10px;
+    padding: 1px 1px 3px 2px;
+    font-size: 14px;
+    background-color: #eee;
+    line-height: 16px;
+    -moz-box-shadow: 1px 1px 3px #bbb;
+    -webkit-box-shadow: 1px 1px 3px #bbb;
+    box-shadow: 1px 1px 3px #bbb;
+
+}
+
+.search-field-name::after {
+    content: ":";
+}
+
+.search-field-value {
+    font-weight: bold;
+}
+
+
 /* ===========================
  * Footer
  */

ckan/public/css/ckan.master.css

@@ -60,7 +60,19 @@ h1, h2, h3, h4, h5, h6 {
 /* ================ */
 
 input.search {
-  width: 30em;
+  width: 99%;
+  padding: 5px;
+  font-size: 1.1em;
+  margin: 0px;
+  -webkit-appearance: textfield;
+}
+
+.package-search input.button {
+    display: inline-block;
+    float: right;
+    margin-top: 5px;
+    margin-right: 10px !important;
+    margin-bottom: 1px !important;
 }
 
 /* ========================================= */
@@ -206,6 +218,7 @@ dl.icons dd.tiny {
 
 ul.packages {
 	padding-left: 0;
+        margin: 0 0 18px 0;
 }
 
 .packages .header {
@@ -218,7 +231,7 @@ ul.packages {
 
 .packages li {
 	list-style: none;
-	padding: 0.4em 0 0.4em 0.5em;
+	padding: 0.4em 0 0.4em 0.0em;
 	border-left: 0.5em solid #fff;
 	border-bottom: 1px solid #ececec;
 	overflow: hidden;