fix dockerfile (#13)

microsoft · Oct 29, 2021 · cfd3e4b · cfd3e4b
1 parent 4e180a8
commit cfd3e4b
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 7 deletions.
diff --git a/docker_support/Dockerfile b/docker_support/Dockerfile
@@ -21,22 +21,21 @@ RUN apt-get --allow-releaseinfo-change update\
     && apt-get install -y g++\
     && apt-get clean\
     && rm -rf /var/lib/apt/lists/*\
-    && pip install pyqlib\
+    && pip install cython numpy -U\
+    && pip install git+https://github.com/microsoft/qlib\
     && cd $SERVER_DIR\
     && python setup.py install\
     && pip install -U python-engineio==3.11.2 python-socketio==4.4.0 flask-socketio==3.1.2\
     && /bin/sed -i "s@<QLIB_DATA>@$QLIB_DATA@g" config_template.yaml\
     && /bin/sed -i "s@<QUEUE_HOST>@$QUEUE_HOST@g" config_template.yaml\
-    && /bin/sed -i "s@<REDIS_HOST>@$REDIS_HOST@g" config_template.yaml\
-    && /bin/sed -i "s@<REDIS_PORT>@$REDIS_PORT@g" config_template.yaml\
-    && /bin/sed -i "s@<REDIS_DB>@$REDIS_DB@g" config_template.yaml\
     && /bin/sed -i "s@<QUEUE_USER>@$QUEUE_USER@g" config_template.yaml\
     && /bin/sed -i "s@<QUEUE_PASS>@$QUEUE_PASS@g" config_template.yaml\
     && /bin/sed -i "s@<TASK_QUEUE>@$TASK_QUEUE@g" config_template.yaml\
     && /bin/sed -i "s@<MESSAGE_QUEUE>@$MESSAGE_QUEUE@g" config_template.yaml\
+    && /bin/sed -i "s@<REDIS_HOST>@$REDIS_HOST@g" config_template.yaml\
+    && /bin/sed -i "s@<REDIS_PORT>@$REDIS_PORT@g" config_template.yaml\
+    && /bin/sed -i "s@<REDIS_DB>@$REDIS_DB@g" config_template.yaml\
     && /bin/sed -i "s@<FLASK_SERVER_HOST>@$FLASK_SERVER_HOST@g" config_template.yaml\
-    && /bin/sed -i "s@<TASK_QUEUE>@task_queue:$QLIB_DATA@g" config_template.yaml\
-    && /bin/sed -i "s@<MESSAGE_QUEUE>@message_queue:$QLIB_DATA@g" config_template.yaml\
     && git clone https://github.com/vishnubob/wait-for-it.git
 
 EXPOSE 9710

diff --git a/docker_support/docker-compose.yaml b/docker_support/docker-compose.yaml
@@ -21,6 +21,8 @@ services:
         - REDIS_HOST=redis
         - QUEUE_USER=${RABBITMQ_DEFAULT_USER}
         - QUEUE_PASS=${RABBITMQ_DEFAULT_PASS}
+        - MESSAGE_QUEUE=${MESSAGE_QUEUE}
+        - TASK_QUEUE=${TASK_QUEUE}
         - QLIB_DATA=${QLIB_DATA}
         - FLASK_SERVER_HOST=${FLASK_SERVER_HOST}
         - QLIB_CODE=${QLIB_CODE}

diff --git a/qlib_server/data_updater.py b/qlib_server/data_updater.py
@@ -100,7 +100,9 @@ def _upate_workers(self, all_cache_path, worker_fun):
         error_info = []
         warning_info = []
         with tqdm(total=cache_length) as p_bar:
-            with ProcessPoolExecutor(max_workers=self.max_workers) as executor:
+            with ProcessPoolExecutor(
+                max_workers=1 if "dataset_cache" in worker_fun.__name__ else self.max_workers
+            ) as executor:
                 futures_map = {}
                 for cache_path in cache_path_list:
                     futures_map[executor.submit(worker_fun, cache_path)] = cache_path