From cfd3e4b98f9a447a609df235fce9543bef55d029 Mon Sep 17 00:00:00 2001
From: Pengrong Zhu <zhu.pengrong@foxmail.com>
Date: Fri, 29 Oct 2021 22:53:07 +0800
Subject: [PATCH] fix dockerfile (#13)

---
 docker_support/Dockerfile          | 11 +++++------
 docker_support/docker-compose.yaml |  2 ++
 qlib_server/data_updater.py        |  4 +++-
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/docker_support/Dockerfile b/docker_support/Dockerfile
index 13076fe..562c2fd 100644
--- a/docker_support/Dockerfile
+++ b/docker_support/Dockerfile
@@ -21,22 +21,21 @@ RUN apt-get --allow-releaseinfo-change update\
     && apt-get install -y g++\
     && apt-get clean\
     && rm -rf /var/lib/apt/lists/*\
-    && pip install pyqlib\
+    && pip install cython numpy -U\
+    && pip install git+https://github.com/microsoft/qlib\
     && cd $SERVER_DIR\
     && python setup.py install\
     && pip install -U python-engineio==3.11.2 python-socketio==4.4.0 flask-socketio==3.1.2\
     && /bin/sed -i "s@<QLIB_DATA>@$QLIB_DATA@g" config_template.yaml\
     && /bin/sed -i "s@<QUEUE_HOST>@$QUEUE_HOST@g" config_template.yaml\
-    && /bin/sed -i "s@<REDIS_HOST>@$REDIS_HOST@g" config_template.yaml\
-    && /bin/sed -i "s@<REDIS_PORT>@$REDIS_PORT@g" config_template.yaml\
-    && /bin/sed -i "s@<REDIS_DB>@$REDIS_DB@g" config_template.yaml\
     && /bin/sed -i "s@<QUEUE_USER>@$QUEUE_USER@g" config_template.yaml\
     && /bin/sed -i "s@<QUEUE_PASS>@$QUEUE_PASS@g" config_template.yaml\
     && /bin/sed -i "s@<TASK_QUEUE>@$TASK_QUEUE@g" config_template.yaml\
     && /bin/sed -i "s@<MESSAGE_QUEUE>@$MESSAGE_QUEUE@g" config_template.yaml\
+    && /bin/sed -i "s@<REDIS_HOST>@$REDIS_HOST@g" config_template.yaml\
+    && /bin/sed -i "s@<REDIS_PORT>@$REDIS_PORT@g" config_template.yaml\
+    && /bin/sed -i "s@<REDIS_DB>@$REDIS_DB@g" config_template.yaml\
     && /bin/sed -i "s@<FLASK_SERVER_HOST>@$FLASK_SERVER_HOST@g" config_template.yaml\
-    && /bin/sed -i "s@<TASK_QUEUE>@task_queue:$QLIB_DATA@g" config_template.yaml\
-    && /bin/sed -i "s@<MESSAGE_QUEUE>@message_queue:$QLIB_DATA@g" config_template.yaml\
     && git clone https://github.com/vishnubob/wait-for-it.git
 
 EXPOSE 9710
diff --git a/docker_support/docker-compose.yaml b/docker_support/docker-compose.yaml
index 6ceac2d..f3cb720 100644
--- a/docker_support/docker-compose.yaml
+++ b/docker_support/docker-compose.yaml
@@ -21,6 +21,8 @@ services:
         - REDIS_HOST=redis
         - QUEUE_USER=${RABBITMQ_DEFAULT_USER}
         - QUEUE_PASS=${RABBITMQ_DEFAULT_PASS}
+        - MESSAGE_QUEUE=${MESSAGE_QUEUE}
+        - TASK_QUEUE=${TASK_QUEUE}
         - QLIB_DATA=${QLIB_DATA}
         - FLASK_SERVER_HOST=${FLASK_SERVER_HOST}
         - QLIB_CODE=${QLIB_CODE}
diff --git a/qlib_server/data_updater.py b/qlib_server/data_updater.py
index c21d16e..97ea835 100644
--- a/qlib_server/data_updater.py
+++ b/qlib_server/data_updater.py
@@ -100,7 +100,9 @@ def _upate_workers(self, all_cache_path, worker_fun):
         error_info = []
         warning_info = []
         with tqdm(total=cache_length) as p_bar:
-            with ProcessPoolExecutor(max_workers=self.max_workers) as executor:
+            with ProcessPoolExecutor(
+                max_workers=1 if "dataset_cache" in worker_fun.__name__ else self.max_workers
+            ) as executor:
                 futures_map = {}
                 for cache_path in cache_path_list:
                     futures_map[executor.submit(worker_fun, cache_path)] = cache_path