feat: add azkaban support

.env.template

@@ -22,7 +22,7 @@ DOCKERHUB_REPO=docker.io/lvsid # change to your own docker hub.
 
 ### Services ##############################################
 # the enable service list
-ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django")
+ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django" "azkaban")
 
 ###########################################################
 ################ Containers Customization #################

.work/config/.env.amd64

@@ -22,7 +22,7 @@ DOCKERHUB_REPO=docker.io/lvsid # change to your own docker hub.
 
 ### Services ##############################################
 # the enable service list
-ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django")
+ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django" "azkaban")
 
 ###########################################################
 ################ Containers Customization #################

.work/config/.env.arm64

@@ -22,7 +22,7 @@ DOCKERHUB_REPO=docker.io/lvsid # change to your own docker hub.
 
 ### Services ##############################################
 # the enable service list
-ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django")
+ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django" "azkaban")
 
 ###########################################################
 ################ Containers Customization #################

azkaban/.env

@@ -2,12 +2,12 @@
 
 # -------------- image config ---------------
 # official image
-IMAGE_OFFICIAL_AZKABAN_NAME=test/test
-IMAGE_OFFICIAL_AZKABAN_VERSION=4.0.0
+IMAGE_OFFICIAL_AZKABAN_NAME=centos
+IMAGE_OFFICIAL_AZKABAN_VERSION=8.4.2105
 # basic image
-IMAGE_BASIC_AZKABAN_VERSION=4.0.0
+IMAGE_BASIC_AZKABAN_VERSION=0.0.1
 # app image
 IMAGE_APP_AZKABAN_VERSION=latest
 # -------------- service config --------------
 AZKABAN_HOST_PORT=4500
-AZKABAN_CONTAINER_PORT=4500
+AZKABAN_CONTAINER_PORT=8081

azkaban/docker-compose.yml

@@ -19,4 +19,5 @@ services:
     networks:
       - backend
     # only for debug
-    # command: "tail -f /dev/null"
+    # Since it(/bin/start-solo.sh) will start a process in the background and exit directly, resulting in no program running in the foreground, you can enable a command like tail -f to ensure that it will not exit.
+    command: "tail -f /dev/null"

azkaban/hooks/after_start.sh

@@ -2,3 +2,5 @@
 
 # include sdk of sparrow.
 . .work/include/sdk.sh
+
+docker exec -d sparrow_container_${CONTAINER_NAMESPACE}_azkaban bash /home/azkaban/azkaban-solo-server/build/install/azkaban-solo-server/bin/start-solo.sh

azkaban/make_app_image/Dockerfile

@@ -1,3 +1,49 @@
 ARG FROM_PLATFORM
 ARG IMAGE_BASIC_AZKABAN_VERSION
 FROM --platform=${FROM_PLATFORM} sparrow-basic-azkaban:${IMAGE_BASIC_AZKABAN_VERSION}
+
+# change source
+# fix error: https://stackoverflow.com/questions/70963985/error-failed-to-download-metadata-for-repo-appstream-cannot-prepare-internal
+RUN cd /etc/yum.repos.d/
+RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
+RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+RUN yum install wget -y
+
+# change yum source
+RUN cd /etc/yum.repos.d/ && ls -alh && cp CentOS-Linux-BaseOS.repo CentOS-Linux-BaseOS.repo.bk && wget http://mirrors.aliyun.com/repo/Centos-8.repo
+RUN yum clean all && cd /etc/yum.repos.d/ && ls -alh && mv Centos-8.repo CentOS-Linux-BaseOS.repo && yum makecache
+
+# install git
+RUN yum update -y && yum install git -y && git --version
+
+# install java
+RUN yum -y list java* && yum install java-1.8.0-openjdk.x86_64 -y
+
+# install nodejs
+RUN curl --silent --location https://rpm.nodesource.com/setup_18.x | bash
+RUN yum install -y nodejs && node -v
+
+# change npm source
+# you don't need yum install -y npm, because npm has already installed after nodejs is installed.
+# if you yum install -y npm, may cause long-time wait and error
+RUN npm config set registry https://registry.npmmirror.com
+RUN npm config get registry
+
+# clone azkaban
+RUN cd /home && git clone https://github.com/azkaban/azkaban.git
+
+# modify azkaban
+# error: Execution failed for task ':azkaban-web-server:nodeSetup'. https://blog.csdn.net/liuandefeng/article/details/130983335 | https://www.cnblogs.com/wuzhiblog/p/13620906.html | https://github.com/azkaban/azkaban/issues/3310
+RUN sed -i 's/download = true/download = false/g' /home/azkaban/azkaban-web-server/build.gradle
+
+# build azkaban
+RUN cd /home/azkaban/ && ./gradlew build installDist -x test
+
+# run azkaban
+# Since it will start a process in the background and exit directly, resulting in no program running in the foreground, you can enable a command like tail -f to ensure that it will not exit.
+# RUN cd /home/azkaban/azkaban-solo-server/build/install/azkaban-solo-server && ./bin/start-solo.sh
+WORKDIR /home/azkaban/azkaban-solo-server/build/install/azkaban-solo-server
+
+# after start, you need start ./bin/start-solo.sh
+# There will be problems if you start it directly here. The current implementation plan is to execute the startup command in the after_start hook.
+# ./hooks/after_start.sh

sparrowtool

@@ -169,7 +169,7 @@ search_env() {
     done
 
     # search these directory
-    subdirectories=("etcd" "etcdkeeper" "go" "jupyter" "kafka" "kafkaui" "mysql" "nginx" "phpfpm" "postgres" "python" "redis" "zookeeper" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django") # when you create a new service, then append it to the subdirectories variable
+    subdirectories=("etcd" "etcdkeeper" "go" "jupyter" "kafka" "kafkaui" "mysql" "nginx" "phpfpm" "postgres" "python" "redis" "zookeeper" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django" "azkaban") # when you create a new service, then append it to the subdirectories variable
     for dir in "${subdirectories[@]}"; do
         # show the service information
         if [ "$start_port" == "" ]; then