diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 1bed08494eb..151f2905078 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -220,9 +220,10 @@ def make_map(): map.redirect("/packages", "/package") map.redirect("/packages/{url:.*}", "/package/{url}") - map.connect('/package/', controller='package', action='index') + map.connect('/package', controller='package', action='search') + map.connect('/package/', controller='package', action='search') map.connect('/package/search', controller='package', action='search') - map.connect('/package/list', controller='package', action='list') + map.connect('/package/list', controller='package', action='search') map.connect('/package/new', controller='package', action='new') map.connect('/package/new_title_to_slug', controller='package', action='new_title_to_slug') map.connect('/package/autocomplete', controller='package', action='autocomplete') diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 45aa4feafe4..49d663f227f 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -1,5 +1,6 @@ import logging import urlparse +import urllib from sqlalchemy.orm import eagerload_all from sqlalchemy import or_ @@ -23,67 +24,65 @@ class PackageController(BaseController): authorizer = ckan.authz.Authorizer() extensions = ExtensionPoint(IPackageController) - def index(self): - query = ckan.authz.Authorizer().authorized_query(c.user, model.Package) - c.package_count = query.count() - return render('package/index.html') - - @proxy_cache() - def list(self): - query = ckan.authz.Authorizer().authorized_query(c.user, model.Package) - query = query.options(eagerload_all('package_tags.tag')) - query = query.options(eagerload_all('package_resources_all')) - c.page = h.AlphaPage( - collection=query, - page=request.params.get('page', 'A'), - alpha_attribute='title', - other_text=_('Other'), - ) - return render('package/list.html') - def search(self): - c.q = request.params.get('q') # unicode format (decoded from utf8) + q = c.q = request.params.get('q') # unicode format (decoded from utf8) c.open_only = request.params.get('open_only') c.downloadable_only = request.params.get('downloadable_only') - if c.q: - c.query_error = False - page = int(request.params.get('page', 1)) - limit = 20 - query = query_for(model.Package) - try: - query.run(query=c.q, - limit=limit, - offset=(page-1)*limit, - return_objects=True, - filter_by_openness=c.open_only, - filter_by_downloadable=c.downloadable_only, - username=c.user) - - c.page = h.Page( - collection=query.results, - page=page, - item_count=query.count, - items_per_page=limit - ) - c.page.items = query.results - except SearchError, se: - c.query_error = True - c.page = h.Page(collection=[]) - - # tag search - c.tag_limit = 25 - query = query_for('tag', backend='sql') - try: - query.run(query=c.q, - return_objects=True, - limit=c.tag_limit, - username=c.user) - c.tags = query.results - c.tags_count = query.count - except SearchError, se: - c.tags = [] - c.tags_count = 0 + if c.q is None or len(c.q.strip()) == 0: + q = '*:*' + c.query_error = False + page = int(request.params.get('page', 1)) + limit = 20 + query = query_for(model.Package) + def drill_down_url(**by): + url = h.url_for(controller='package', action='search') + param = request.params.items() + for k, v in by.items(): + if not (k, v) in param: + param.append((k, v.encode('utf-8'))) + return url + '?' + urllib.urlencode(param) + + c.drill_down_url = drill_down_url + + def remove_field(key, value): + url = h.url_for(controller='package', action='search') + param = request.params.items() + param.remove((key, value)) + return url + '?' + urllib.urlencode( + [(k, v.encode('utf-8')) for k, v in param]) + + c.remove_field = remove_field + + try: + c.fields = [] + for (param, value) in request.params.items(): + if not param in ['q', 'open_only', 'downloadable_only', 'page']: + c.fields.append((param, value)) + + query.run(query=q, + fields=c.fields, + facet_by=g.facets, + limit=limit, + offset=(page-1)*limit, + return_objects=True, + filter_by_openness=c.open_only, + filter_by_downloadable=c.downloadable_only, + username=c.user) + + c.page = h.Page( + collection=query.results, + page=page, + item_count=query.count, + items_per_page=limit + ) + c.facets = query.facets + c.page.items = query.results + except SearchError, se: + c.query_error = True + c.facets = {} + c.page = h.Page(collection=[]) + return render('package/search.html') def _pkg_cache_key(self, pkg): diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index b21414b67f9..f767e3a7b52 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -19,6 +19,8 @@ def __init__(self): self.site_logo = config.get('ckan.site_logo', '/images/ckan_logo_fullname_long.png') self.site_url = config.get('ckan.site_url', 'http://www.ckan.net') + self.facets = config.get('search.facets', 'groups tags res_format license').split() + # has been setup in load_environment(): self.site_id = config.get('ckan.site_id') - \ No newline at end of file + diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index d3e1823b2e4..b983537cd93 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -276,6 +276,7 @@ class SearchIndexCommand(CkanCommand): def command(self): self._load_config() + from ckan.lib.search import rebuild if not self.args: # default to run diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 0a7b6de6442..c7c7cdfafca 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -42,6 +42,22 @@ def subnav_link(c, text, action, **kwargs): class_=('active' if c.action == action else '') ) +def facet_items(c, name, limit=10): + from pylons import request + if not c.facets or not c.facets.get(name): + return [] + facets = [] + for k, v in c.facets.get(name).items(): + if not len(k.strip()): + continue + if not (name, k) in request.params.items(): + facets.append((k, v)) + return sorted(facets, key=lambda (k, v): v, reverse=True)[:limit] + +def facet_title(name): + from pylons import config + return config.get('search.facets.%s.title' % name, name.capitalize()) + def am_authorized(c, action, domain_object=None): from ckan.authz import Authorizer if domain_object is None: @@ -63,12 +79,19 @@ def linked_user(user): url_for(controller='user', action='read', id=_name)) return user +def group_name_to_title(name): + from ckan import model + group = model.Group.by_name(name) + if group is not None: + return group.title + return name + def markdown_extract(text): if (text is None) or (text == ''): return '' html = fromstring(markdown(text)) plain = html.xpath("string()") - return unicode(truncate(plain, length=270, indicator='...', whole_word=True)) + return unicode(truncate(plain, length=190, indicator='...', whole_word=True)) def icon_url(name): return '/images/icons/%s.png' % name diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index 0ab576ee7b9..db11f248a10 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -46,6 +46,8 @@ def rebuild(): package_index.clear() for pkg in model.Session.query(model.Package).all(): package_index.insert_entity(pkg) + model.Session.commit() + def query_for(_type, backend=None): """ Query for entities of a specified type (name, class, instance). """ diff --git a/ckan/lib/search/common.py b/ckan/lib/search/common.py index 6d97c1f928f..1e03d3d62e2 100644 --- a/ckan/lib/search/common.py +++ b/ckan/lib/search/common.py @@ -101,13 +101,15 @@ def _format_results(self): attr_name = self.options.ref_entity_with_attr self.results = [getattr(entity, attr_name) for entity in self.results] - def run(self, query=None, terms=[], fields={}, options=None, **kwargs): + def run(self, query=None, terms=[], fields={}, facet_by=[], options=None, **kwargs): if options is None: options = QueryOptions(**kwargs) else: options.update(kwargs) self.options = options self.options.validate() + self.facet_by = facet_by + self.facets = dict() self.query = QueryParser(query, terms, fields) self.query.validate() self._run() diff --git a/ckan/lib/search/solr_.py b/ckan/lib/search/solr_.py index 2b32f0bd640..64673d3eaf4 100644 --- a/ckan/lib/search/solr_.py +++ b/ckan/lib/search/solr_.py @@ -56,11 +56,16 @@ def _run(self): # show only results from this CKAN instance: fq = fq + " +site_id:\"%s\"" % config.get('ckan.site_id') - + conn = self.backend.make_connection() try: data = conn.query(self.query.query, fq=fq, + # make sure data.facet_counts is set: + facet='true', + facet_limit=50, + facet_field=self.facet_by, + facet_mincount=1, start=self.options.offset, rows=self.options.limit, fields='id,score', @@ -78,6 +83,7 @@ def _run(self): result_ids = [(r.get('id')) for r in data.results] q = authz.Authorizer().authorized_query(self.options.username, model.Package) q = q.filter(model.Package.id.in_(result_ids)) + self.facets = data.facet_counts.get('facet_fields', {}) self.results = q.all() diff --git a/ckan/lib/search/sql.py b/ckan/lib/search/sql.py index 02464a3228d..a071397b4db 100644 --- a/ckan/lib/search/sql.py +++ b/ckan/lib/search/sql.py @@ -267,9 +267,8 @@ def update_dict(self, pkg_dict): def remove_dict(self, pkg_dict): if not 'id' in pkg_dict or not 'name' in pkg_dict: return - sql = "DELETE FROM package_search WHERE package_id=%%s" - params.append(pkg_dict['id']) - self._run_sql(sql, params) + sql = "DELETE FROM package_search WHERE package_id=%s" + self._run_sql(sql, [pkg_dict.get('id')]) log.debug("Delete entry %s from index" % pkg_dict.get('id')) diff --git a/ckan/public/css/ckan.css b/ckan/public/css/ckan.css index e7706fa369b..99d45ec2789 100644 --- a/ckan/public/css/ckan.css +++ b/ckan/public/css/ckan.css @@ -1,3 +1,12 @@ +body, input, textarea, .page-title span, .pingback a.url { + font-family: "Lucida Grande", Lucida, Verdana, sans-serif; +} + +#content, #content input, #content textarea { + font-size: 14px; + line-height: 20px; +} + a { color: #b00; text-decoration: none; @@ -223,6 +232,75 @@ a:hover { margin-bottom: 10px; } +/* ========================== + * Facets + */ + +.facet-box { + +} + +.facet-box h2 { + color: #000; + font-size: 1.2em; + font-weight: bold; + margin-top: 1em; +} + +.facet-options { + margin-top: 0.5em; +} + +.facet-options li { + padding-top: 0.2em; + font-size: 1.2em; + color: #000; +} + +.register-link { + padding-top: 10px; +} + +.register-link a { + color: white; + background: #199150; + font-weight: bold; + padding: 5px; + width: 100%; + font-size: 1.3em; + -moz-border-radius: 2px; + -webkit-border-radius: 2px; + border-radius: 2px; + +} + +.package-search-filters { + margin-top: 15px; +} + +.search-field { + display: inline-block; + margin-right: 5px; + margin-bottom: 10px; + padding: 1px 1px 3px 2px; + font-size: 14px; + background-color: #eee; + line-height: 16px; + -moz-box-shadow: 1px 1px 3px #bbb; + -webkit-box-shadow: 1px 1px 3px #bbb; + box-shadow: 1px 1px 3px #bbb; + +} + +.search-field-name::after { + content: ":"; +} + +.search-field-value { + font-weight: bold; +} + + /* =========================== * Footer */ diff --git a/ckan/public/css/ckan.master.css b/ckan/public/css/ckan.master.css index 75687987f82..d0578f89aae 100644 --- a/ckan/public/css/ckan.master.css +++ b/ckan/public/css/ckan.master.css @@ -60,7 +60,19 @@ h1, h2, h3, h4, h5, h6 { /* ================ */ input.search { - width: 30em; + width: 99%; + padding: 5px; + font-size: 1.1em; + margin: 0px; + -webkit-appearance: textfield; +} + +.package-search input.button { + display: inline-block; + float: right; + margin-top: 5px; + margin-right: 10px !important; + margin-bottom: 1px !important; } /* ========================================= */ @@ -206,6 +218,7 @@ dl.icons dd.tiny { ul.packages { padding-left: 0; + margin: 0 0 18px 0; } .packages .header { @@ -218,7 +231,7 @@ ul.packages { .packages li { list-style: none; - padding: 0.4em 0 0.4em 0.5em; + padding: 0.4em 0 0.4em 0.0em; border-left: 0.5em solid #fff; border-bottom: 1px solid #ececec; overflow: hidden; diff --git a/ckan/public/images/icons/unfilter.png b/ckan/public/images/icons/unfilter.png new file mode 100755 index 00000000000..bd6271b2467 Binary files /dev/null and b/ckan/public/images/icons/unfilter.png differ diff --git a/ckan/templates/layout_base.html b/ckan/templates/layout_base.html index b82f20a3e02..6980522ca5a 100644 --- a/ckan/templates/layout_base.html +++ b/ckan/templates/layout_base.html @@ -83,11 +83,11 @@

-
+
- + ${g.site_title}
diff --git a/ckan/templates/package/facets.html b/ckan/templates/package/facets.html new file mode 100644 index 00000000000..f4e927fc414 --- /dev/null +++ b/ckan/templates/package/facets.html @@ -0,0 +1,31 @@ + + + +
+

${h.facet_title(code)}

+ +
+ + + +
+
+ ${h.facet_title(field)} + ${value} + + ${h.icon('unfilter')} + +
+
+ + + diff --git a/ckan/templates/package/search.html b/ckan/templates/package/search.html index d91ad1eefda..17f14798db5 100644 --- a/ckan/templates/package/search.html +++ b/ckan/templates/package/search.html @@ -5,21 +5,29 @@ xmlns:xi="http://www.w3.org/2001/XInclude" py:strip=""> + Search - Data Packages + + + +
+ Register a package +
+ + ${facet_sidebar('groups', label=h.group_name_to_title)} + ${facet_sidebar('tags')} + ${facet_sidebar('res_format')} + ${facet_sidebar('license')} + + +
-

Search Data Packages

+

Discover Data Packages

+ ${field_list()} - -
- - -

${c.tags_count} tags found (max ${c.tag_limit} shown).

- ${tag_list(c.tags)} - -

There was an error while searching. Please try another search term.

@@ -31,9 +39,8 @@

Search Data Packages

${c.page.pager(q=c.q)} ${package_list(c.page.items)} ${c.page.pager(q=c.q)} - -
+ diff --git a/ckan/templates/package/search_form.html b/ckan/templates/package/search_form.html index 643c7c1b4ac..ae3e64c938d 100644 --- a/ckan/templates/package/search_form.html +++ b/ckan/templates/package/search_form.html @@ -1,9 +1,21 @@ + + + + diff --git a/ckan/tests/functional/test_authz.py b/ckan/tests/functional/test_authz.py index 7cbfeb21514..f239b60cc1d 100644 --- a/ckan/tests/functional/test_authz.py +++ b/ckan/tests/functional/test_authz.py @@ -236,10 +236,10 @@ def test_04_user_edits(self): self._test_can('edit', self.mrloggedin, ['wx', 'wr', 'ww']) def test_list(self): - self._test_can('list', [self.testsysadmin, self.pkgadmin], ['xx', 'rx', 'wx', 'rr', 'wr', 'ww'], entities=['package']) + #self._test_can('list', [self.testsysadmin, self.pkgadmin], ['xx', 'rx', 'wx', 'rr', 'wr', 'ww'], entities=['package']) self._test_can('list', [self.testsysadmin, self.groupadmin], ['xx', 'rx', 'wx', 'rr', 'wr', 'ww'], entities=['group']) - self._test_can('list', self.mrloggedin, ['rx', 'wx', 'rr', 'wr', 'ww']) - self._test_can('list', self.visitor, ['rr', 'wr', 'ww']) + #self._test_can('list', self.mrloggedin, ['rx', 'wx', 'rr', 'wr', 'ww']) + #self._test_can('list', self.visitor, ['rr', 'wr', 'ww']) # we abandon checks for reading due to caching # self._test_cant('list', self.mrloggedin, ['xx']) # self._test_cant('list', self.visitor, ['xx', 'rx', 'wx']) @@ -259,12 +259,6 @@ def test_search_deleted(self): self._test_can('search', self.mrloggedin, ['rx', 'wx', 'rr', 'wr', 'ww'], entities=['package']) self._test_cant('search', self.mrloggedin, ['deleted', 'xx'], entities=['package']) - def test_list_deleted(self): - self._test_can('list', self.pkgadmin, ['xx', 'rx', 'wx', 'rr', 'wr', 'ww', 'deleted'], interfaces=['wui'], entities=['package']) - self._test_can('list', self.mrloggedin, ['rx', 'wx', 'rr', 'wr', 'ww'], interfaces=['wui']) - self._test_cant('list', self.mrloggedin, ['deleted'], interfaces=['wui']) - self._test_cant('list', self.mrloggedin, ['xx'], interfaces=['wui'], entities=['package']) - def test_05_author_is_new_package_admin(self): user = self.mrloggedin @@ -297,10 +291,6 @@ def test_sysadmin_can_edit_anything(self): def test_sysadmin_can_search_anything(self): self._test_can('search', self.testsysadmin, ['xx', 'rx', 'wx', 'rr', 'wr', 'ww', 'deleted'], entities=['package']) - def test_sysadmin_can_list_anything(self): - self._test_can('list', self.testsysadmin, ['xx', 'rx', 'wx', 'rr', 'wr', 'ww'], interfaces=['wui']) - self._test_can('list', self.testsysadmin, ['deleted'], interfaces=['wui'], entities=['package']) - def test_visitor_creates(self): self._test_can('create', self.visitor, ['rr'], interfaces=['wui'], entities=['package']) @@ -333,13 +323,6 @@ def setup_class(self): model.Session.remove() indexer.index() - #def test_visitor_creates(self): - #from pprint import pprint - #pprint(model.Session.query(model.User).all()) - #pprint(model.Session.query(model.RoleAction).all()) - #pprint(model.Session.query(model.UserObjectRole).filter_by(context='System').all()) - # self._test_cant('create', self.visitor, ['rrx'], entities=['package']) - def test_user_creates(self): self._test_can('create', self.mrloggedin, ['rr']) diff --git a/ckan/tests/functional/test_package.py b/ckan/tests/functional/test_package.py index b67f95b025a..ec2dbfe8b26 100644 --- a/ckan/tests/functional/test_package.py +++ b/ckan/tests/functional/test_package.py @@ -260,23 +260,10 @@ def setup_class(self): def teardown_class(self): CreateTestData.delete() - def test_index(self): - offset = url_for(controller='package') - res = self.app.get(offset) - assert 'Data Packages' in res - - def test_minornavigation(self): - offset = url_for(controller='package') - res = self.app.get(offset) - # TODO: make this a bit more rigorous! - assert 'Browse' in res, res - res = res.click('Browse') - assert 'Browse - Data Packages' in res - def test_minornavigation_2(self): - offset = url_for(controller='package') + offset = url_for(controller='package', action='search') res = self.app.get(offset) - res = res.click('Register') + res = res.click('Register a package') assert 'New - Data Packages' in res def test_read(self): @@ -341,17 +328,15 @@ def check_link(res, controller, id): assert 'decoy' not in res, res assert 'decoy"' not in res, res - def test_list(self): - offset = url_for(controller='package', action='list') - res = self.app.get(offset) - assert 'Packages' in res - name = u'annakarenina' - title = u'A Novel By Tolstoy' - assert title in res - res = res.click(title) - assert '%s - Data Packages' % title in res, res - main_div = self.main_div(res) - assert title in main_div, main_div.encode('utf8') + #res = self.app.get(offset) + #assert 'Packages' in res + #name = u'annakarenina' + #title = u'A Novel By Tolstoy' + #assert title in res + #res = res.click(title) + #assert '%s - Data Packages' % title in res, res + #main_div = self.main_div(res) + #assert title in main_div, main_div.encode('utf8') def test_search(self): offset = url_for(controller='package', action='search') @@ -418,7 +403,6 @@ def setup(self): if not self.res: self.res = self.app.get(self.offset) - @classmethod def _reset_data(self): CreateTestData.delete() @@ -1217,13 +1201,6 @@ def teardown_class(self): CreateTestData.delete() self.purge_packages([self.non_active_name]) - def test_list(self): - offset = url_for(controller='package', action='list') - res = self.app.get(offset) - assert 'Browse - Data Packages' in res - assert 'annakarenina' in res - assert self.non_active_name not in res - def test_read(self): offset = url_for(controller='package', action='read', id=self.non_active_name) res = self.app.get(offset, status=[302, 401])