manticore poc

Dockerfile

@@ -1,15 +1,14 @@
-FROM python:3.9.9-slim-buster as sphinxsearch_base
+FROM manticoresearch/manticore:5.0.2 as manticore_base
 
 RUN apt-get update && \
     apt-get install -y \
     cron \
-    default-mysql-client \
-    gettext-base \
-    gosu \
-    jq \
-    procps \
+    gettext \
+    libpq-dev \
+    manticore-converter \
+    python3-pip \
     rsync \
-    sphinxsearch \
+    jq \
     vim && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
@@ -20,31 +19,36 @@ RUN apt-get update && \
     mkfifo /tmp/stdout /tmp/stderr && \
     chmod 0666 /tmp/stdout /tmp/stderr
 
-    # set up geodata, file permissions, copy files and run container as geodata
-FROM sphinxsearch_base as sphinxsearch_geodata
+# set up geodata, file permissions, copy files and run container as geodata
+FROM manticore_base as manticore_geodata
 
-    # add geodata user, same uid/gid as the EFS owner is needed here
+# add geodata user, same uid/gid as the EFS owner is needed here
 RUN groupadd -r geodata -g 2500 && \
     useradd -u 2500 -r -g geodata -s /sbin/nologin --create-home geodata && \
     # create mountpoint folders with geodata ownership
-    install -o geodata -g geodata -d /var/lib/sphinxsearch/data/index/ && \
-    install -o geodata -p geodata -d /var/lib/sphinxsearch/data/index_efs/ && \
+
+    install -o geodata -g geodata -d /var/lib/manticore/data/index/ && \
+    install -o geodata -p geodata -d /var/lib/manticore/data/index_efs/ && \
+
     # change ownerships to geodata which will run the service or the maintenance scripts
     # and mount the efs folder
-    chown -R geodata:geodata /var/run/sphinxsearch/ && \
-    chown -R geodata:geodata /var/log/sphinxsearch/ && \
-    chown -R geodata:geodata /etc/sphinxsearch && \
+    chown -R geodata:geodata /var/run/manticore/ && \
+    chown -R geodata:geodata /var/log/manticore/ && \
+    chown -R geodata:geodata /etc/manticoresearch && \
+    chown -R geodata:geodata /var/run/mysqld && \
     # install pip3 psycopg2, python3.9 does not (yet) support python3-psycopg2 package
     gosu geodata pip3 install psycopg2-binary==2.9.2
 
-FROM sphinxsearch_geodata
+FROM manticore_geodata
 
 # copy sphinxsearch config and maintenance code
 COPY --chown=geodata:geodata scripts/docker-* scripts/index-sync-rotate.sh scripts/pg2sphinx_trigger.py scripts/checker.sh /
 COPY --chown=geodata:geodata conf /conf/
 
 USER geodata
 
+WORKDIR /
+
 # default CMD
 ENTRYPOINT [ "/docker-entry.sh" ]
 # run service with the following script if no CMDs are sent to docker run / default

Makefile

@@ -79,15 +79,15 @@ PPID := $(shell echo $$PPID)
 export DOCKER_EXEC :=  docker run \
 				--rm \
 				-t \
-				-v $(SPHINX_EFS):/var/lib/sphinxsearch/data/index/ \
+				-v $(SPHINX_EFS):/var/lib/manticore/data/index/ \
 				--env-file $(ENV_FILE) \
 				--name $(DOCKER_LOCAL_TAG)_maintenance_$(PPID)\
 				$(DOCKER_IMG_LOCAL_TAG)
 
 export DOCKER_EXEC_LOCAL :=  docker run \
 				--rm \
 				-t \
-				-v $(CURRENT_DIR)/conf/:/var/lib/sphinxsearch/data/index/ \
+				-v $(CURRENT_DIR)/conf/:/var/lib/manticore/data/index/ \
 				--env-file $(ENV_FILE) \
 				--name $(DOCKER_LOCAL_TAG)_maintenance_$(PPID) \
 				$(DOCKER_IMG_LOCAL_TAG)
@@ -222,7 +222,7 @@ endif
 
 .PHONY: check-config-local
 check-config-local: dockerbuild config
-	$(DOCKER_EXEC_LOCAL) indextool --checkconfig -c /etc/sphinxsearch/sphinx.conf | grep "config valid" || $(DOCKER_EXEC_LOCAL) indextool --checkconfig -c /etc/sphinxsearch/sphinx.conf
+	$(DOCKER_EXEC_LOCAL) indextool --checkconfig -c /etc/manticoresearch/manticore.conf | grep "config valid" || $(DOCKER_EXEC_LOCAL) indextool --checkconfig -c /etc/manticoresearch/manticore.conf
 	DOCKER_EXEC_LOCAL="$(DOCKER_EXEC_LOCAL)" ./scripts/check-config-local.sh
 
 
@@ -289,8 +289,8 @@ dockerrun: dockerbuild sphinx_efs
 		--restart=always \
 		-d \
 		-p $(SPHINX_PORT):$(SPHINX_PORT) \
-		-v $(SPHINX_EFS):/var/lib/sphinxsearch/data/index_efs/ \
-		-v ${DOCKER_INDEX_VOLUME}:/var/lib/sphinxsearch/data/index/ \
+		-v $(SPHINX_EFS):/var/lib/manticore/data/index_efs/ \
+		-v ${DOCKER_INDEX_VOLUME}:/var/lib/manticore/data/index/ \
 		--env-file $(ENV_FILE) \
 		--name $(DOCKER_LOCAL_TAG) \
 		$(DOCKER_IMG_LOCAL_TAG)
@@ -302,8 +302,8 @@ dockerrundebug: dockerbuild sphinx_efs
 		--rm \
 		-it \
 		-p $(SPHINX_PORT):$(SPHINX_PORT) \
-		-v $(SPHINX_EFS):/var/lib/sphinxsearch/data/index_efs/ \
-		-v ${DOCKER_INDEX_VOLUME}:/var/lib/sphinxsearch/data/index/ \
+		-v $(SPHINX_EFS):/var/lib/manticore/data/index_efs/ \
+		-v ${DOCKER_INDEX_VOLUME}:/var/lib/manticore/data/index/ \
 		--env-file $(ENV_FILE) \
 		--name $(DOCKER_LOCAL_TAG) \
 		$(DOCKER_IMG_LOCAL_TAG)

README.md

@@ -36,11 +36,11 @@ $ sudo -u root systemctl start sphinxsearch
 
 Object            | Path
 ------------------|-----------------------------------|
-**PID:**          | /var/run/sphinxsearch.pid         |
-**Searchd Log**   | /var/log/sphinxsearch/searchd.log |
-**Query Log:**    | /var/log/sphinxsearch/query.log   |
-**Indexes:**      | /var/lib/sphinxsearch/data/index/ |
-**Configuration:**| /etc/sphinxsearch/sphinx.conf     |
+**PID:**          | /var/run/manticore.pid         |
+**Searchd Log**   | /var/log/manticore/searchd.log |
+**Query Log:**    | /var/log/manticore/query.log   |
+**Indexes:**      | /var/lib/manticore/data/index/ |
+**Configuration:**| /etc/manticoresearch/manticore.conf     |
 
 ### Search Daemon:
 
@@ -59,21 +59,21 @@ $ sudo -u root systemctl start sphinxsearch
 #### validate config
 
 ```
-$ indextool --checkconfig -c /etc/sphinxsearch/sphinx.conf
+$ indextool --checkconfig -c /etc/manticoresearch/manticore.conf
 ```
 
 ### Rebuild / update Indexes:
 #### rebuild / build some indexes index1 index2 index3
 There will be a service restart after every index
 
 ```bash
-$ sudo -u sphinxsearch indexer --verbose --rotate --sighup-each --config /etc/sphinxsearch/sphinx.conf index1 index2 index3
+$ sudo -u sphinxsearch indexer --verbose --rotate --sighup-each --config /etc/manticoresearch/manticore.conf index1 index2 index3
 ```
 
 #### rebuild / build all indexes
 
 ```bash
-$ sudo -u sphinxsearch indexer --verbose --rotate --sighup-each --config /etc/sphinxsearch/sphinx.conf --all
+$ sudo -u sphinxsearch indexer --verbose --rotate --sighup-each --config /etc/manticoresearch/manticore.conf --all
 ```
 multithread indexer is not possible: http://sphinxsearch.com/forum/view.html?id=3936a
 
@@ -167,6 +167,6 @@ $ make deploy-int-config db=zeitreihen
 You can use this command to synchronize the remote sphinx config with the remote indices:
 * create all the missing indexes
 * remove orphaned indexes
-The sphinx configuration will not be deployed. The command can be used with integration or production: 
+The sphinx configuration will not be deployed. The command can be used with integration or production:
 * ``$ make deploy-int-clean_index``
 * ``$ make deploy-prod-clean_index``

conf/00_db.conf.part

@@ -36,20 +36,20 @@ indexer
 
 searchd
 {
-    listen = $SPHINX_PORT
-    # syslog -> logstash -> kibana
-    log = /var/log/sphinxsearch/searchd.log
-    # syslog -> logstash -> kibana
-    query_log = /var/log/sphinxsearch/query.log
-    pid_file = /var/run/sphinxsearch/searchd.pid
+    listen = 9312
+    listen = 9306:mysql41
+    listen = /var/run/mysqld/mysqld.sock:mysql41
+    listen = 127.0.0.1:9312
+    listen = 9308:http
+    log = /var/log/manticore/searchd.log
+    query_log = /var/log/manticore/query.log
+    pid_file = /var/run/manticore/searchd.pid
     # mysql listener
     listen = localhost:9306:mysql41
     preopen_indexes = 0
-    # dist_threads should be number of cpus/cores
-    dist_threads = $CPUS
-    read_timeout = 5
-    max_children = 50
+    # max_threads_per_query should be number of cpus/cores
+    max_threads_per_query = 8
+    network_timeout = 5
     binlog_path =
     watchdog = 1
-    ondisk_attrs_default = 1
-}
+}