Pull request #1: SLCAI-54 automl module

Merge in SLAISM/autoape-mlps from SLCAI-54-automl-module to main * commit '96965ee026032a5d6617a8b9459d19406f12b41f': dockerfile : base image를 python-base -> python-base-gpu로 변경 build 수정 원복 test2 resource usage checking log test 빌드 수정 로그 수정 job info log 출력 오타수정 resource 추가 resource 폴더 추가 모듈 구조 변경
seculayer · Apr 22, 2022 · 2be8df1 · 2be8df1
2 parents 28579c6 + 96965ee
commit 2be8df1
Show file tree

Hide file tree

Showing 217 changed files with 230 additions and 24,551 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,44 +1,101 @@
-FROM seculayer/python:3.7-cuda11.2 as builder
+# syntax=docker/dockerfile:1.3
+FROM registry.seculayer.com:31500/ape/python-base-gpu:py3.7 as builder
 MAINTAINER jinkim "jinkim@seculayer.com"
 
 ARG app="/opt/app"
 
-RUN mkdir -p $app
-WORKDIR $app
+RUN pip3.7 install wheel && git config --global http.sslVerify false
 
-COPY ./requirements.txt ./requirements.txt
-RUN pip3.7 install -r ./requirements.txt -t $app/lib
+# pycmmn setup
+# specific branch
+RUN --mount=type=secret,id=token git clone --depth=5 -c http.extraHeader="Authorization: Bearer $(cat /run/secrets/token)" --single-branch -b SLCAI-54-automl-module https://ssdlc-bitbucket.seculayer.com:8443/scm/slaism/autoape-pycmmn.git $app/pycmmn
+#RUN --mount=type=secret,id=token git clone --depth=5 -c http.extraHeader="Authorization: Bearer $(cat /run/secrets/token)" https://ssdlc-bitbucket.seculayer.com:8443/scm/slaism/autoape-pycmmn.git $app/pycmmn
+WORKDIR $app/pycmmn
+RUN pip3.7 install -r requirements.txt -t $app/pycmmn/lib && python3.7 setup.py bdist_wheel
 
-COPY ./mlps ./mlps
-COPY ./setup.py ./setup.py
+# dataconverter setup
+# specific branch
+RUN --mount=type=secret,id=token git clone --depth=5 -c http.extraHeader="Authorization: Bearer $(cat /run/secrets/token)" --single-branch -b SLCAI-54-automl-module https://ssdlc-bitbucket.seculayer.com:8443/scm/slaism/autoape-dataconverter.git $app/dataconverter
+#RUN --mount=type=secret,id=token git clone --depth=5 -c http.extraHeader="Authorization: Bearer $(cat /run/secrets/token)" https://ssdlc-bitbucket.seculayer.com:8443/scm/slaism/autoape-dataconverter.git $app/dataconverter
+WORKDIR $app/dataconverter
+RUN pip3.7 install -r requirements.txt -t $app/dataconverter/lib && python3.7 setup.py bdist_wheel
 
-RUN pip3.7 install wheel
-RUN python3.7 setup.py bdist_wheel
+# apeflow setup
+# specific branch
+RUN --mount=type=secret,id=token git clone --depth=5 -c http.extraHeader="Authorization: Bearer $(cat /run/secrets/token)" --single-branch -b SLCAI-54-automl-module https://ssdlc-bitbucket.seculayer.com:8443/scm/slaism/autoape-apeflow.git $app/apeflow
+#RUN --mount=type=secret,id=token git clone --depth=5 -c http.extraHeader="Authorization: Bearer $(cat /run/secrets/token)" https://ssdlc-bitbucket.seculayer.com:8443/scm/slaism/autoape-apeflow.git $app/apeflow
+WORKDIR $app/apeflow
+RUN pip3.7 install -r requirements.txt -t $app/apeflow/lib && python3.7 setup.py bdist_wheel
+
+# mlps setup
+# specific branch
+RUN --mount=type=secret,id=token git clone --depth=5 -c http.extraHeader="Authorization: Bearer $(cat /run/secrets/token)" --single-branch -b SLCAI-54-automl-module https://ssdlc-bitbucket.seculayer.com:8443/scm/slaism/autoape-mlps.git $app/mlps
+#RUN --mount=type=secret,id=token git clone --depth=5 -c http.extraHeader="Authorization: Bearer $(cat /run/secrets/token)" https://ssdlc-bitbucket.seculayer.com:8443/scm/slaism/autoape-mlps.git $app/mlps \
+
+WORKDIR $app/mlps
+RUN pip3.7 install -r $app/mlps/requirements.txt -t $app/mlps/lib && python3.7 setup.py bdist_wheel
+
+
+
+FROM registry.seculayer.com:31500/ape/python-base-gpu:py3.7 as app
 
-FROM seculayer/python:3.7-cuda11.2 as app
 ARG app="/opt/app"
+ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
 
-RUN mkdir -p /eyeCloudAI/app/ape/mlps \
-    && mkdir -p /eyeCloudAI/data/processing/ape/models
+# pycmmn install
+RUN mkdir -p /eyeCloudAI/app/ape/pycmmn
+WORKDIR /eyeCloudAI/app/ape/pycmmn
 
-RUN groupadd -g 1000 aiuser
-RUN useradd -r -u 1000 -g aiuser aiuser
-RUN chown -R aiuser:aiuser /eyeCloudAI
+COPY --from=builder "$app/pycmmn/lib" /eyeCloudAI/app/ape/pycmmn/lib
+COPY --from=builder "$app/pycmmn/dist/pycmmn-1.0.0-py3-none-any.whl" \
+        /eyeCloudAI/app/ape/pycmmn/pycmmn-1.0.0-py3-none-any.whl
 
-USER aiuser
+RUN pip3.7 install /eyeCloudAI/app/ape/pycmmn/pycmmn-1.0.0-py3-none-any.whl --no-dependencies  \
+    -t /eyeCloudAI/app/ape/pycmmn/ \
+    && rm /eyeCloudAI/app/ape/pycmmn/pycmmn-1.0.0-py3-none-any.whl
+
+# dataconverter install
+RUN mkdir -p /eyeCloudAI/app/ape/dataconverter
+WORKDIR /eyeCloudAI/app/ape/dataconverter
 
+COPY --from=builder "$app/dataconverter/lib" /eyeCloudAI/app/ape/dataconverter/lib
+COPY --from=builder "$app/dataconverter/dist/dataconverter-1.0.0-py3-none-any.whl" \
+        /eyeCloudAI/app/ape/dataconverter/dataconverter-1.0.0-py3-none-any.whl
+
+RUN pip3.7 install /eyeCloudAI/app/ape/dataconverter/dataconverter-1.0.0-py3-none-any.whl --no-dependencies  \
+    -t /eyeCloudAI/app/ape/dataconverter/ \
+    && rm /eyeCloudAI/app/ape/dataconverter/dataconverter-1.0.0-py3-none-any.whl
+
+# apeflow install
+RUN mkdir -p /eyeCloudAI/app/ape/apeflow
+WORKDIR /eyeCloudAI/app/ape/apeflow
+
+COPY --from=builder "$app/apeflow/lib" /eyeCloudAI/app/ape/apeflow/lib
+COPY --from=builder "$app/apeflow/dist/apeflow-1.0.0-py3-none-any.whl" \
+        /eyeCloudAI/app/ape/apeflow/apeflow-1.0.0-py3-none-any.whl
+
+RUN pip3.7 install /eyeCloudAI/app/ape/apeflow/apeflow-1.0.0-py3-none-any.whl --no-dependencies  \
+    -t /eyeCloudAI/app/ape/apeflow/ \
+    && rm /eyeCloudAI/app/ape/apeflow/apeflow-1.0.0-py3-none-any.whl
+
+# mlps install
+RUN mkdir -p /eyeCloudAI/app/ape/mlps
 WORKDIR /eyeCloudAI/app/ape/mlps
 
 COPY ./mlps.sh /eyeCloudAI/app/ape/mlps
+RUN chmod +x /eyeCloudAI/app/ape/mlps/mlps.sh
 
-COPY --from=builder --chown=aiuser:aiuser "$app/lib" /eyeCloudAI/app/ape/mlps/lib/
-
-COPY --from=builder --chown=aiuser:aiuser "$app/dist/mlps-3.0.0-py3-none-any.whl" /eyeCloudAI/app/ape/mlps/
+COPY --from=builder "$app/mlps/lib" /eyeCloudAI/app/ape/mlps/lib
+COPY --from=builder "$app/mlps/dist/mlps-3.0.0-py3-none-any.whl" \
+        /eyeCloudAI/app/ape/mlps/mlps-3.0.0-py3-none-any.whl
 
 RUN pip3.7 install /eyeCloudAI/app/ape/mlps/mlps-3.0.0-py3-none-any.whl --no-dependencies  \
-    -t /eyeCloudAI/app/ape/mlps \
+    -t /eyeCloudAI/app/ape/mlps/ \
     && rm /eyeCloudAI/app/ape/mlps/mlps-3.0.0-py3-none-any.whl
 
-ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
+RUN groupadd -g 1000 aiuser
+RUN useradd -r -u 1000 -g aiuser aiuser
+RUN chown -R aiuser:aiuser /eyeCloudAI
+USER aiuser
 
 CMD []
diff --git a/build-mlps.sh b/build-mlps.sh
@@ -1,9 +1,10 @@
 #!/bin/bash
 
 VERSION="3.0.0"
-
+MODULE_NAME="mlps"
 REGISTRY_URL="registry.seculayer.com:31500"
+SECRET_PATH="../access_token/token"
 
 # docker build
-DOCKER_BUILDKIT=1 docker build -t $REGISTRY_URL/ape/automl-mlps:$VERSION .
-docker push $REGISTRY_URL/ape/automl-mlps:$VERSION
+DOCKER_BUILDKIT=1 docker build --no-cache --secret id=token,src=$SECRET_PATH -t $REGISTRY_URL/ape/automl-$MODULE_NAME:$VERSION .
+docker push $REGISTRY_URL/ape/automl-$MODULE_NAME:$VERSION
diff --git a/conf/mlps-conf.xml b/conf/mlps-conf.xml
@@ -21,11 +21,6 @@
         <value>/resources</value>
         <description>None</description>
     </property>
-    <property>
-        <name>resource_cnvrtr</name>
-        <value>/cnvrtr</value>
-        <description>resource folder for data converting.</description>
-    </property>
     <property>
         <name>dir_data_root</name>
         <value>/data</value>
@@ -41,43 +36,18 @@
         <value>/jobs</value>
         <description>None</description>
     </property>
-    <property>
-        <name>dir_load_model</name>
-        <value>/load_models</value>
-        <description>None</description>
-    </property>
-    <property>
-        <name>dir_model</name>
-        <value>/models</value>
-        <description>None</description>
-    </property>
     <property>
         <name>dir_storage</name>
         <value>/storage/ape</value>
         <description>None</description>
     </property>
-    <property>
-        <name>dir_learn_feat</name>
-        <value>/da</value>
-        <description>None</description>
-    </property>
-    <property>
-        <name>dir_verify_feat</name>
-        <value>/verifys</value>
-        <description>None</description>
-    </property>
-    <property>
-        <name>dir_rtdetect_feat</name>
-        <value>/rtdetect</value>
-        <description>None</description>
-    </property>
     <property>
         <name>dir_result</name>
         <value>/results</value>
         <description>None</description>
     </property>
     <property>
-        <name>dir_ml_tmp</name>
+        <name>dir_temp</name>
         <value>/temp</value>
         <description>None</description>
     </property>
@@ -86,16 +56,6 @@
         <value>/errors</value>
         <description>None</description>
     </property>
-    <property>
-        <name>user_custom_converter_package_nm</name>
-        <value>cnvrtr</value>
-        <description>eyeCloudAI Custom converter class file path. default is [cnvrtr].</description>
-    </property>
-    <property>
-        <name>user_custom_algorithm_package_root</name>
-        <value>/eyeCloudAI/app/ape/custom</value>
-        <description>None</description>
-    </property>
     <!-- log setting -->
     <property>
         <name>log_dir</name>
@@ -112,44 +72,11 @@
         <value>INFO</value>
         <description>[INFO, DEBUG, WARN, ERROR, CRITICAL]</description>
     </property>
-    <!-- mlps.core.manager setting -->
-    <property>
-        <name>job_ext</name>
-        <value>.job</value>
-        <description>None</description>
-    </property>
-    <property>
-        <name>max_sub_process</name>
-        <value>2</value>
-        <description>Cnt of Data Loader SUB Processor</description>
-    </property>
-    <property>
-        <name>remove_temp_folder</name>
-        <value>false</value>
-        <description>[true or false]</description>
-    </property>
-    <!-- mlps.core.dataloader setting -->
-    <property>
-        <name>dataloader_ext</name>
-        <value>.done</value>
-        <description>.done</description>
-    </property>
-    <property>
-        <name>dataloader_timeout</name>
-        <value>7200</value>
-        <description>Second</description>
-    </property>
-    <!-- mlps.core.cnvrtr setting -->
-    <property>
-        <name>cvt_data</name>
-        <value>False</value>
-        <description>None</description>
-    </property>
     <!-- rest url setting -->
     <property>
         <name>mrms_svc</name>
 <!--        <value>mrms-svc</value>-->
-        <value>10.1.35.118</value>
+        <value>10.1.35.72</value>
 <!--        <value>10.1.12.234</value>-->
         <description>None</description>
     </property>
@@ -180,4 +107,9 @@
         <value>5</value>
         <description>Seconds</description>
     </property>
+    <property>
+        <name>cvt_data</name>
+        <value>false</value>
+        <description>true or false</description>
+    </property>
 </configuration>
diff --git a/deploy_mlps.sh b/deploy_mlps.sh
diff --git a/mlps.sh b/mlps.sh
@@ -7,15 +7,16 @@
 ######################################################################################
 
 APP_PATH=/eyeCloudAI/app/ape
-
-MLPS_LIB_PATH=$APP_PATH/mlps/lib
-MLPS_PATH=$APP_PATH/mlps
 ####
 export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
 export CUDA_HOME=/usr/local/cuda
-export PYTHONPATH=$MLPS_LIB_PATH:$MLPS_PATH
+export PYTHONPATH=$PYTHONPATH:$APP_PATH/mlps/lib:$APP_PATH/mlps
+export PYTHONPATH=$PYTHONPATH:$APP_PATH/pycmmn/lib:$APP_PATH/pycmmn
+export PYTHONPATH=$PYTHONPATH:$APP_PATH/apeflow/lib:$APP_PATH/apeflow
+export PYTHONPATH=$PYTHONPATH:$APP_PATH/dataconverter/lib:$APP_PATH/dataconverter
 
 KEY=${1}
 TASK_IDX=${2}
 JOB_TYPE=${3}
-/usr/local/bin/python3.7 -m mlps.MLProcessingServer ${KEY} ${TASK_IDX} ${JOB_TYPE}
+
+/usr/local/bin/python3.7 -m mlps.MLProcessingServer ${KEY} ${TASK_IDX} ${JOB_TYPE}