Skip to content

Commit

Permalink
configurable shards
Browse files Browse the repository at this point in the history
  • Loading branch information
Bek committed Oct 2, 2020
1 parent 4a53685 commit 3c1e3e3
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 5 deletions.
2 changes: 2 additions & 0 deletions base.ini
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ retry.attempts = 3
file_upload_bucket = snowflakes-files-dev
file_upload_profile_name = snowflakes-files-upload
elasticsearch.server = localhost:9201
elasticsearch.shards.primary = 1
elasticsearch.shards.replicate = 0
aws_ip_ranges_path = %(here)s/aws-ip-ranges.json
download_proxy = https://download.encodeproject.org/
annotations_path = %(here)s/annotations.json
Expand Down
12 changes: 7 additions & 5 deletions src/snovault/elasticsearch/create_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,13 +164,13 @@ def schema_mapping(name, schema):
}


def index_settings():
def index_settings(primary_shards=3, replicate_shards=1):
return {
'settings': {
'index.max_result_window': 99999,
'index.mapping.total_fields.limit': 5000,
'index.number_of_shards': 1,
'index.number_of_replicas': 0,
'index.number_of_shards': primary_shards,
'index.number_of_replicas': replicate_shards,
'index.max_ngram_diff': 32,
'analysis': {
'filter': {
Expand Down Expand Up @@ -484,6 +484,8 @@ def create_snovault_index_alias(es, indices):
def run(app, collections=None, dry_run=False):
index = app.registry.settings['snovault.elasticsearch.index']
registry = app.registry
primary_shards = registry.settings.get('elasticsearch.shards.primary', 3)
replicate_shards = registry.settings.get('elasticsearch.shards.replicate', 1)
if not dry_run:
es = app.registry[ELASTIC_SEARCH]
print('CREATE MAPPING RUNNING')
Expand All @@ -503,9 +505,9 @@ def run(app, collections=None, dry_run=False):
if mapping is None:
continue # Testing collections
if dry_run:
print(json.dumps(sorted_dict({index: {doc_type: mapping}}), indent=4))
print(json.dumps(sorted_dict({index: {collection_name: mapping}}), indent=4))
continue
create_elasticsearch_index(es, index, index_settings())
create_elasticsearch_index(es, index, index_settings(primary_shards, replicate_shards))
set_index_mapping(es, index, mapping)
if collection_name != 'meta':
indices.append(index)
Expand Down

0 comments on commit 3c1e3e3

Please sign in to comment.