Update ES

.gitignore

@@ -55,3 +55,5 @@ tags
 .mypy_cache/
 tmp*
 luigi/*.pid
+*.pid
+elasticsearch.yml

README.md

@@ -52,6 +52,7 @@ This section is purely informative, you can skip to [Run AWS Scripts](#run-aws-s
 * Python 3.6
 * Apache Spark 2.2.0
 * Vowpal Wabbit 8.1.1
+* Elastic Search 5.6.X (Not 6.X)
 * CUDA and Nvidia drivers if using a GPU instance
 * lz4
 * All python packages in `packer/requirements.txt`
@@ -186,10 +187,15 @@ To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLeve
 17/07/25 10:04:01 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address
 ```
 
-2. Install ElasticSearch
+2. Install ElasticSearch 5.6
+
+Install version 5.6.X, do not use 6.X. Also be sure that the directory `bin/` within the extracted files is in your
+`$PATH` as it contains the necessary binary `elasticsearch`.
 
 https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html
 
+https://www.elastic.co/downloads/past-releases/elasticsearch-5-6-8
+
 3. Install Python packages
 
 ```

cli.py

@@ -13,6 +13,7 @@
 from jinja2 import Environment, PackageLoader
 
 from qanta import qlogging
+from qanta.guesser.elasticsearch import create_es_config, start_elasticsearch, stop_elasticsearch
 from qanta.util.environment import ENVIRONMENT
 from qanta.datasets.quiz_bowl import QuestionDatabase, Question, QB_QUESTION_DB
 from qanta.guesser.abstract import AbstractGuesser
@@ -21,7 +22,7 @@
 from qanta.update_db import write_answer_map, merge_answer_mapping
 
 
-log = qlogging.get(__name__)
+log = qlogging.get('cli')
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
@@ -177,7 +178,9 @@ def generate_guesser_slurm(slurm_config_file, task, output_dir):
     enabled_guessers = list(AbstractGuesser.list_enabled_guessers())
 
     for i, gs in enumerate(enabled_guessers):
-        if gs.guesser_class in slurm_config:
+        if gs.guesser_class == 'ElasticSearchGuesser':
+            raise ValueError('ElasticSearchGuesser is not compatible with slurm')
+        elif gs.guesser_class in slurm_config:
             guesser_slurm_config = slurm_config[gs.guesser_class]
         else:
             guesser_slurm_config = None
@@ -217,5 +220,25 @@ def generate_guesser_slurm(slurm_config_file, task, output_dir):
         f.write(master_script)
 
 
+@main.command()
+@click.option('--generate-config/--no-generate-config', default=True, is_flag=True)
+@click.option('--config-dir', default='.')
+@click.option('--pid-file', default='elasticsearch.pid')
+@click.argument('command', type=click.Choice(['start', 'stop', 'configure']))
+def elasticsearch(generate_config, config_dir, pid_file, command):
+    if generate_config:
+        create_es_config(path.join(config_dir, 'elasticsearch.yml'))
+
+    if command == 'configure':
+        return
+
+    if command == 'start':
+        start_elasticsearch(config_dir, pid_file)
+    elif command == 'stop':
+        stop_elasticsearch(pid_file)
+
+
+
+
 if __name__ == '__main__':
     main()

log4j2.properties

@@ -0,0 +1,74 @@
+status = error
+
+# log action execution errors for easier debugging
+logger.action.name = org.elasticsearch.action
+logger.action.level = debug
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+
+appender.rolling.type = RollingFile
+appender.rolling.name = rolling
+appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.rolling.ref = rolling
+
+appender.deprecation_rolling.type = RollingFile
+appender.deprecation_rolling.name = deprecation_rolling
+appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
+appender.deprecation_rolling.layout.type = PatternLayout
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
+appender.deprecation_rolling.policies.type = Policies
+appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.deprecation_rolling.policies.size.size = 1GB
+appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
+appender.deprecation_rolling.strategy.max = 4
+
+logger.deprecation.name = org.elasticsearch.deprecation
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
+logger.deprecation.additivity = false
+
+appender.index_search_slowlog_rolling.type = RollingFile
+appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
+appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
+appender.index_search_slowlog_rolling.layout.type = PatternLayout
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
+appender.index_search_slowlog_rolling.policies.type = Policies
+appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_search_slowlog_rolling.policies.time.interval = 1
+appender.index_search_slowlog_rolling.policies.time.modulate = true
+
+logger.index_search_slowlog_rolling.name = index.search.slowlog
+logger.index_search_slowlog_rolling.level = trace
+logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
+logger.index_search_slowlog_rolling.additivity = false
+
+appender.index_indexing_slowlog_rolling.type = RollingFile
+appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
+appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
+appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+appender.index_indexing_slowlog_rolling.policies.type = Policies
+appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_indexing_slowlog_rolling.policies.time.interval = 1
+appender.index_indexing_slowlog_rolling.policies.time.modulate = true
+
+logger.index_indexing_slowlog.name = index.indexing.slowlog.index
+logger.index_indexing_slowlog.level = trace
+logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
+logger.index_indexing_slowlog.additivity = false

qanta-hyper.yaml

@@ -1,7 +1,10 @@
 parameters:
-  qanta.guesser.dan.DanGuesser:
-    random_seed: [0, 1, 2, 3]
+  qanta.guesser.elasticsearch.ElasticSearchGuesser:
     enabled: [true]
+    use_wiki: [true, false]
+  #qanta.guesser.dan.DanGuesser:
+  #  random_seed: [0, 1, 2, 3]
+  #  enabled: [true]
 
   #qanta.guesser.vw.VWGuesser:
   #  random_seed: [0, 1, 2, 3, 4]