From 03c1ce053359603d69d9de4e8a81238171354082 Mon Sep 17 00:00:00 2001 From: Denis Zgonjanin Date: Mon, 14 Sep 2015 14:26:20 -0400 Subject: [PATCH 1/3] Search index rebuild takes a while; let's give it a progress counter --- ckan/lib/search/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index ea28b0410f0..87b5c222be5 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -183,7 +183,12 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False, def if not refresh: package_index.clear() - for pkg_id in package_ids: + total_packages = len(package_ids) + for counter, pkg_id in enumerate(package_ids): + sys.stdout.write( + "\rIndexing dataset {0}/{1}".format(counter, total_packages) + ) + sys.stdout.flush() try: package_index.update_dict( logic.get_action('package_show')(context, From 35823d807c569626a151a7a938d7f681d13b6014 Mon Sep 17 00:00:00 2001 From: Denis Zgonjanin Date: Tue, 15 Sep 2015 10:50:17 -0400 Subject: [PATCH 2/3] add --quiet option to paster search index rebuild progress meter --- ckan/lib/cli.py | 23 ++++++++++++++--------- ckan/lib/search/__init__.py | 13 ++++++++----- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 1747eade27e..e98533cfed0 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -402,14 +402,14 @@ class SearchIndexCommand(CkanCommand): '''Creates a search index for all datasets Usage: - search-index [-i] [-o] [-r] [-e] rebuild [dataset_name] - reindex dataset_name if given, if not then rebuild - full search index (all datasets) - search-index rebuild_fast - reindex using multiprocessing using all cores. - This acts in the same way as rubuild -r [EXPERIMENTAL] - search-index check - checks for datasets not indexed - search-index show DATASET_NAME - shows index of a dataset - search-index clear [dataset_name] - clears the search index for the provided dataset or - for the whole ckan instance + search-index [-i] [-o] [-r] [-e] [-q] rebuild [dataset_name] - reindex dataset_name if given, if not then rebuild + full search index (all datasets) + search-index rebuild_fast - reindex using multiprocessing using all cores. + This acts in the same way as rubuild -r [EXPERIMENTAL] + search-index check - checks for datasets not indexed + search-index show DATASET_NAME - shows index of a dataset + search-index clear [dataset_name] - clears the search index for the provided dataset or + for the whole ckan instance ''' summary = __doc__.split('\n')[0] @@ -432,6 +432,10 @@ def __init__(self, name): action='store_true', default=False, help='Refresh current index (does not clear the existing one)') + self.parser.add_option('-q', '--quiet', dest='quiet', + action='store_true', default=False, + help='Do not output index rebuild progress') + self.parser.add_option('-e', '--commit-each', dest='commit_each', action='store_true', default=False, help= '''Perform a commit after indexing each dataset. This ensures that changes are @@ -474,7 +478,8 @@ def rebuild(self): rebuild(only_missing=self.options.only_missing, force=self.options.force, refresh=self.options.refresh, - defer_commit=(not self.options.commit_each)) + defer_commit=(not self.options.commit_each), + quiet=self.options.quiet) if not self.options.commit_each: commit() diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index 87b5c222be5..ba5e7aee81d 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -135,7 +135,8 @@ def notify(self, entity, operation): log.warn("Discarded Sync. indexing for: %s" % entity) -def rebuild(package_id=None, only_missing=False, force=False, refresh=False, defer_commit=False, package_ids=None): +def rebuild(package_id=None, only_missing=False, force=False, refresh=False, + defer_commit=False, package_ids=None, quiet=False): ''' Rebuilds the search index. @@ -185,10 +186,12 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False, def total_packages = len(package_ids) for counter, pkg_id in enumerate(package_ids): - sys.stdout.write( - "\rIndexing dataset {0}/{1}".format(counter, total_packages) - ) - sys.stdout.flush() + if not quiet: + sys.stdout.write( + "\rIndexing dataset {0}/{1}".format( + counter +1, total_packages) + ) + sys.stdout.flush() try: package_index.update_dict( logic.get_action('package_show')(context, From fd9bf893f9c0cfc77043f3b6384a62ee960612ac Mon Sep 17 00:00:00 2001 From: David Read Date: Wed, 16 Sep 2015 06:54:25 +0000 Subject: [PATCH 3/3] Fix test --- ckan/tests/legacy/lib/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/tests/legacy/lib/test_cli.py b/ckan/tests/legacy/lib/test_cli.py index 3be780ba74f..cba14b4b1f5 100644 --- a/ckan/tests/legacy/lib/test_cli.py +++ b/ckan/tests/legacy/lib/test_cli.py @@ -80,7 +80,7 @@ def test_clear_and_rebuild_index(self): # Rebuild index self.search.args = () - self.search.options = FakeOptions(only_missing=False,force=False,refresh=False,commit_each=False) + self.search.options = FakeOptions(only_missing=False, force=False, refresh=False, commit_each=False, quiet=False) self.search.rebuild() pkg_count = model.Session.query(model.Package).filter(model.Package.state==u'active').count()