Добавлена возможность использования контейнеров

.dockerignore

@@ -0,0 +1,10 @@
+.git
+.cache
+Dockerfile
+db
+db_config
+portfolio
+docker-compose.yml
+logs
+reports
+.venv

Dockerfile

@@ -0,0 +1,52 @@
+FROM nvidia/cuda:11.0.3-cudnn8-runtime-ubuntu20.04
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get -q update \
+ && apt-get install -yq --no-install-recommends \
+    wget \
+    ca-certificates \
+    sudo \
+    locales \
+    fonts-liberation \
+    run-one \
+    libx11-xcb1 \
+    libxtst6 \
+    gnupg \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+    locale-gen
+RUN wget -qO - https://www.mongodb.org/static/pgp/server-5.0.asc | apt-key add -
+RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/5.0 multiverse" > /etc/apt/sources.list.d/mongodb-org-5.0.list
+RUN apt-get update && apt-get install -y mongodb-org-tools python3.9 python3-pip &&  apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN python3.9 -m pip install -U pip
+WORKDIR /opt/app
+COPY requirements.txt .
+RUN python3.9 -m pip install -U -r requirements.txt
+#RUN apt-get update \
+#    && apt-get install -y wget gnupg \
+#    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+#    && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+#    && apt-get update \
+#    && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \
+#      --no-install-recommends \
+#    && rm -rf /var/lib/apt/lists/*
+#RUN apt-get update && apt-get install libx11-xcb1 libxtst6
+COPY fix-permissions /usr/local/bin/fix-permissions
+RUN chmod a+rx /usr/local/bin/fix-permissions
+ARG NB_USER="user"
+ARG NB_UID="1000"
+ARG NB_GID="100"
+RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
+    sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \
+    sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \
+    useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \
+    chmod g+w /etc/passwd && \
+    fix-permissions $HOME
+USER $NB_UID
+# RUN mkdir "/home/$NB_USER"
+RUN pyppeteer-install
+USER root
+COPY . /opt/app/
+RUN python3.9 setup.py install
+USER $NB_USER
+ENTRYPOINT ["bash"]
+#ENTRYPOINT ["python3.9", "/opt/app/poptimizer/__main__.py", "evolve" ]

docker-compose.yml

@@ -0,0 +1,40 @@
+version: "3.8"
+services:
+  poptimizer:
+    build: .
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - capabilities:
+              - gpu
+    volumes:
+      - ./portfolio:/home/user/portfolio
+      - ./logs:/home/user/logs
+      - ./reports:/home/user/reports
+      - ./config:/home/user/config
+      - ./dump:/home/user/dump
+    environment:
+      MONGODB_URI: "mongodb://poptimizer:poptimizer@mongodb:27017"
+      REPORTS_PATH: "/home/user/reports"
+      PORTFOLIO_PATH: "/home/user/portfolio"
+      LOG_PATH: "/home/user/logs"
+      CONFIG_PATH: "/home/user/config"
+      DUMP_PATH: "/home/user/dump"
+    privileged: true
+    user: "${UID}:${GID}"
+    cap_add:
+      - mknod
+      - sys_admin
+    user:
+      "root"
+    restart: always
+
+  mongodb:
+    image: mongo:5.0
+    volumes:
+      - ./db:/data/db
+      - ./db_config:/data/configdb  
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: poptimizer
+      MONGO_INITDB_ROOT_PASSWORD: poptimizer

fix-permissions

@@ -0,0 +1,35 @@
+#!/bin/bash
+# set permissions on a directory
+# after any installation, if a directory needs to be (human) user-writable,
+# run this script on it.
+# It will make everything in the directory owned by the group $NB_GID
+# and writable by that group.
+# Deployments that want to set a specific user id can preserve permissions
+# by adding the `--group-add users` line to `docker run`.
+
+# uses find to avoid touching files that already have the right permissions,
+# which would cause massive image explosion
+
+# right permissions are:
+# group=$NB_GID
+# AND permissions include group rwX (directory-execute)
+# AND directories have setuid,setgid bits set
+
+set -e
+
+for d in "$@"; do
+    find "$d" \
+        ! \( \
+            -group $NB_GID \
+            -a -perm -g+rwX  \
+        \) \
+        -exec chgrp $NB_GID {} \; \
+        -exec chmod g+rwX {} \;
+    # setuid, setgid *on directories only*
+    find "$d" \
+        \( \
+            -type d \
+            -a ! -perm -6000  \
+        \) \
+        -exec chmod +6000 {} \;
+done

poptimizer/config.py

@@ -5,7 +5,7 @@
 import pandas as pd
 import torch
 import yaml
-
+import os
 from poptimizer.shared.log import get_handlers
 
 
@@ -24,21 +24,22 @@ class POptimizerError(Exception):
 _root = pathlib.Path(__file__).parents[1]
 
 # Путь к директории с отчетами
-REPORTS_PATH = _root / "reports"
+REPORTS_PATH = pathlib.Path(os.getenv("REPORTS_PATH",_root / "reports"))
 
 # Путь к директории с портфелями
-PORT_PATH = _root / "portfolio"
+PORT_PATH = pathlib.Path(os.getenv("PORTFOLIO_PATH",_root / "portfolio"))
 
 # Путь к директории с логами
-LOG_PATH = _root / "logs"
+LOG_PATH = pathlib.Path(os.getenv("LOG_PATH",_root / "logs"))
 
 # Конфигурация логгера
 logging.basicConfig(level=logging.INFO, handlers=get_handlers(LOG_PATH))
 
 
 def _load_config() -> dict[str, Union[int, float, str]]:
     cfg = {}
-    path = _root / "config" / "config.yaml"
+    path = pathlib.Path(os.getenv("CONFIG_PATH",_root / "config")) / "config.yaml"
+    print(path)
     if path.exists():
         with path.open() as file:
             cfg = yaml.safe_load(file)

poptimizer/data/adapters/odm.py

@@ -10,9 +10,9 @@
 from motor import motor_asyncio
 
 from poptimizer.shared import adapters, connections
-
+import os
 # Путь к dump с данными по дивидендам
-MONGO_DUMP: Final = pathlib.Path(__file__).parents[3] / "dump"
+MONGO_DUMP: Final = pathlib.Path(os.getenv("DUMP_PATH",pathlib.Path(__file__).parents[3] / "dump"))
 
 # Ссылки на данные по дивидендам в интернете
 DIV_DATA_URL: Final = (
@@ -70,7 +70,7 @@ async def _restore_dump(
     """Осуществляет восстановление данных по дивидендам."""
     if SOURCE_DB not in await client.list_database_names():
         logging.info("Начато восстановление данных с дивидендами")
-        mongo_restore = ["mongorestore", MONGO_DUMP]
+        mongo_restore = ["mongorestore", "--uri="+connections.MONGO_URI, MONGO_DUMP]
         process = psutil.Popen(mongo_restore)
         status = process.wait()
         logging.info(f"Восстановление данных с дивидендами завершен со статусом {status}")
@@ -83,7 +83,7 @@ async def _dump_dividends_db(client: motor_asyncio.AsyncIOMotorClient) -> None:
     div_count = await collection.find_one({"_id": ID})
     if div_count is None or n_docs != div_count[KEY]:
         logging.info(f"Backup данных с дивидендами {n_docs} документов")
-        process = psutil.Popen(["mongodump", "--out", MONGO_DUMP, "--db", SOURCE_DB])
+        process = psutil.Popen(["mongodump", "--uri="+connections.MONGO_URI, "--out", MONGO_DUMP, "--db", SOURCE_DB])
         status = process.wait()
         await collection.replace_one({"_id": ID}, {KEY: n_docs}, upsert=True)
         logging.info(f"Backup данных с дивидендами завершен со статусом {status}")

poptimizer/shared/connections.py

@@ -7,10 +7,10 @@
 import aiohttp
 import psutil
 from motor import motor_asyncio
-
+import os
 # Настройки сервера MongoDB
 _MONGO_PATH: Final = pathlib.Path(__file__).parents[2] / "db"
-_MONGO_URI: Final = "mongodb://localhost:27017"
+MONGO_URI: Final = os.getenv("MONGODB_URI","mongodb://localhost:27017")
 
 # Размер пула http-соединений - при большем размере многие сайты ругаются
 _POOL_SIZE: Final = 20
@@ -54,7 +54,7 @@ def http_session_factory(pool_size: int) -> aiohttp.ClientSession:
     atexit.register(_clean_up, session)
     return session
 
-
-start_mongo_server()
-MONGO_CLIENT: Final = motor_asyncio.AsyncIOMotorClient(_MONGO_URI, tz_aware=False)
+if not os.getenv("MONGODB_URI", False):
+    start_mongo_server()
+MONGO_CLIENT: Final = motor_asyncio.AsyncIOMotorClient(MONGO_URI, tz_aware=False)
 HTTP_SESSION: Final = http_session_factory(_POOL_SIZE)

poptimizer/store/database.py

@@ -3,10 +3,10 @@
 from typing import Any, Final
 
 import pymongo
-
+from poptimizer.shared import connections
 # Старые настройки по MongoDB
-_MONGO_URI: Final = "mongodb://localhost:27017"
-MONGO_CLIENT: Final = pymongo.MongoClient(_MONGO_URI, tz_aware=False)
+# _MONGO_URI: Final = "mongodb://localhost:27017"
+MONGO_CLIENT: Final = pymongo.MongoClient(connections.MONGO_URI, tz_aware=False)
 DB: Final = "data"
 MISC: Final = "misc"
 _ID: Final = "_id"

setup.py

@@ -0,0 +1,9 @@
+from setuptools import setup, find_packages
+setup(
+    name="poptimizer",
+    version="1.0",
+    packages=find_packages(),
+    entry_points={
+        'console_scripts': ['app=app.app:main', ]    
+    },
+)