Skip to content

Commit

Permalink
Set test request context for rebuild fast
Browse files Browse the repository at this point in the history
  • Loading branch information
tino097 committed Sep 25, 2018
1 parent ee03487 commit eaf03e6
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 12 deletions.
23 changes: 14 additions & 9 deletions ckan/cli/search_index.py
@@ -1,5 +1,6 @@
# encoding: utf-8

import multiprocessing as mp
import os

import click
Expand Down Expand Up @@ -84,6 +85,7 @@ def clear(dataset_name):
@click.pass_context
def rebuild_fast(ctx):
conf = ctx.obj.config
flask_app = ctx.obj.app.apps['flask_app']._wsgi_app
db_url = conf['sqlalchemy.url']
engine = sa.create_engine(db_url)
package_ids = []
Expand All @@ -94,7 +96,6 @@ def rebuild_fast(ctx):
def start(ids):
## load actual enviroment for each subprocess, so each have thier own
## sa session
self._load_config()
from ckan.lib.search import rebuild, commit
rebuild(package_ids=ids)
commit()
Expand All @@ -108,11 +109,15 @@ def chunks(l, n):
yield l[n*newn-newn:]

processes = []
for chunk in chunks(package_ids, mp.cpu_count()):
process = mp.Process(target=start, args=(chunk,))
processes.append(process)
process.daemon = True
process.start()

for process in processes:
process.join()
with flask_app.test_request_context():
try:
for chunk in chunks(package_ids, mp.cpu_count()):
process = mp.Process(target=start, args=(chunk,))
processes.append(process)
process.daemon = True
process.start()

for process in processes:
process.join()
except Exception as e:
click.echo(e.message)
3 changes: 0 additions & 3 deletions ckan/lib/cli.py
Expand Up @@ -556,9 +556,6 @@ def rebuild_fast(self):
package_ids.append(row[0])

def start(ids):
## load actual enviroment for each subprocess, so each have thier own
## sa session
self._load_config()
from ckan.lib.search import rebuild, commit
rebuild(package_ids=ids)
commit()
Expand Down

0 comments on commit eaf03e6

Please sign in to comment.