[Core] Optimize packaging (#799)

devlive-community · Jul 7, 2024 · c6bfbe2 · c6bfbe2
2 parents bc7f2a2 + aa9f24f
commit c6bfbe2
Show file tree

Hide file tree

Showing 110 changed files with 923 additions and 960 deletions.
diff --git a/.github/workflows/bofore_checker.yml b/.github/workflows/bofore_checker.yml
@@ -44,8 +44,6 @@ jobs:
 
   before_checker_bugs:
     runs-on: ubuntu-latest
-    needs:
-      - before_checker_ui
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -54,8 +52,14 @@ jobs:
         with:
           java-version: '11'
           distribution: 'temurin'
-      - run: chmod 755 ./mvnw
-      - run: ./mvnw clean install spotbugs:spotbugs -Dcheckstyle.skip -Dgpg.skip -Dskip.yarn -DskipTests=true
+      - name: Grant execute permission for mvnw
+        run: chmod +x mvnw
+      - name: Run SpotBugs skip server
+        env:
+          MAVEN_OPTS: -Xmx1024m
+        run: ./mvnw clean install spotbugs:spotbugs -pl '!core/datacap-server' -Dcheckstyle.skip -Dgpg.skip -Dskip.yarn -DskipTests=true
+      - name: Run SpotBugs for server
+        run: ./mvnw clean install spotbugs:spotbugs -Dcheckstyle.skip -Dgpg.skip -Dskip.yarn -DskipTests=true -f core/datacap-server/pom.xml
 
   before_checker_package:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -38,7 +38,7 @@ jobs:
           distribution: 'temurin'
 
       - run: |
-          ./mvnw -T 1C clean install package -Dfindbugs.skip -Dgpg.skip -Dcheckstyle.skip -DskipTests=true -Dskip.npm -q
+          ./mvnw -T 1C clean install package -Dspotbugs.skip -Dgpg.skip -Dcheckstyle.skip -DskipTests=true -Dskip.npm -q
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v2

diff --git a/.gitignore b/.gitignore
@@ -43,7 +43,7 @@ list
 mongo_*.xml
 
 # shaded #
-/dependency-reduced-pom.xml
+**/dependency-reduced-pom.xml
 shaded/*/dependency-reduced-pom.xml
 
 # datacap #

diff --git a/client/datacap-cli/pom.xml b/client/datacap-cli/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>io.edurt.datacap</groupId>
         <artifactId>datacap</artifactId>
-        <version>2024.03.7</version>
+        <version>2024.03.8-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

diff --git a/configure/assembly/plugin.xml b/configure/assembly/plugin.xml
@@ -22,6 +22,13 @@
             <includes>
                 <include>*.jar</include>
             </includes>
+            <excludes>
+                <exclude>**/*-javadoc.jar</exclude>
+                <exclude>**/*-sources.jar</exclude>
+                <exclude>**/kotlin-*.jar</exclude>
+                <exclude>**/lombok-*.jar</exclude>
+                <exclude>**/*-test.jar</exclude>
+            </excludes>
         </fileSet>
     </fileSets>
 </assembly>
diff --git a/configure/assembly/server.xml b/configure/assembly/server.xml
@@ -98,5 +98,13 @@
                 <include>**/datacap*.jar</include>
             </includes>
         </fileSet>
+        <fileSet>
+            <directory>${project.parent.basedir}</directory>
+            <outputDirectory>.</outputDirectory>
+            <includes>
+                <include>mvnw</include>
+                <include>mvnw.cmd</include>
+            </includes>
+        </fileSet>
     </fileSets>
 </assembly>
diff --git a/configure/etc/bin/debug.sh b/configure/etc/bin/debug.sh
@@ -52,8 +52,7 @@ job_runner_debug_server() {
     printf "Server starting                        | %s\n" "$APPLICATION_NAME"
     cd "$HOME"
     get_jvm_conf
-    PLUGIN_DIR=`find plugins/* -type d | sed 's/\(.*\)/\1\/\*/' | xargs | tr ' ' ':'`
-    "$JAVA_HOME"/bin/java -cp "$HOME/lib/*:$PLUGIN_DIR" ${JVM_CONF} "$APPLICATION_NAME" \
+    "$JAVA_HOME"/bin/java -cp "$HOME/lib/*:$HOME/plugins/*:$HOME/notifys/*:$HOME/schedulers/*:$HOME/parsers/*:$HOME/fss/*:$HOME/converts/*:$HOME/executors/*" ${JVM_CONF} "$APPLICATION_NAME" \
             --spring.config.location="$HOME/configure/"
 }
 

diff --git a/configure/etc/bin/install-plugin.sh b/configure/etc/bin/install-plugin.sh
@@ -0,0 +1,73 @@
+#!/bin/sh0
+
+HOME=$(pwd)
+VERSION=2024.03.8-SNAPSHOT
+
+common_install_handler() {
+    TYPE=$1
+    if [ ! -d "${HOME}/${TYPE}s" ];
+      then
+          mkdir "${HOME}/${TYPE}s"
+          echo "Create ${TYPE}s directory"
+    fi
+
+    echo "Install datacap ${TYPE}, usage version is ${VERSION}"
+    while read line; do
+        full_line=$(echo "$line" | cut -c 1)
+        if [ "$full_line" != "-" ] && [ "$full_line" != "#" ] && [ ! -z $full_line ]
+        then
+            echo "install connector : $line"
+        		"${HOME}"/mvnw dependency:get -DgroupId=io.edurt.datacap -DartifactId="${line}" -Dversion=${VERSION} -Ddest="${HOME}/${TYPE}s"
+        fi
+    done <"${HOME}/configure/${TYPE}".conf
+}
+
+job_install_plugin() {
+  printf "========== Job install plugin start ========== \n"
+  common_install_handler "plugin"
+  printf "========== Job install plugin end ========== \n"
+}
+
+job_install_notify() {
+  printf "========== Job install notify start ========== \n"
+  common_install_handler "notify"
+  printf "========== Job install notify end ========== \n"
+}
+
+job_install_scheduler() {
+  printf "========== Job install scheduler start ========== \n"
+  common_install_handler "scheduler"
+  printf "========== Job install scheduler end ========== \n"
+}
+
+job_install_parser() {
+  printf "========== Job install parser start ========== \n"
+  common_install_handler "parser"
+  printf "========== Job install parser end ========== \n"
+}
+
+job_install_fs() {
+  printf "========== Job install fs start ========== \n"
+  common_install_handler "fs"
+  printf "========== Job install fs end ========== \n"
+}
+
+job_install_convert() {
+  printf "========== Job install convert start ========== \n"
+  common_install_handler "convert"
+  printf "========== Job install convert end ========== \n"
+}
+
+job_install_executor() {
+  printf "========== Job install executor start ========== \n"
+  common_install_handler "executor"
+  printf "========== Job install executor end ========== \n"
+}
+
+job_install_plugin
+job_install_notify
+job_install_scheduler
+job_install_parser
+job_install_fs
+job_install_convert
+job_install_executor
diff --git a/configure/etc/bin/startup.sh b/configure/etc/bin/startup.sh
@@ -52,8 +52,7 @@ job_runner_start_server() {
     printf "Server starting                        | %s\n" "$APPLICATION_NAME"
     cd "$HOME"
     get_jvm_conf
-    PLUGIN_DIR=`find plugins/* -type d | sed 's/\(.*\)/\1\/\*/' | xargs | tr ' ' ':'`
-    nohup "$JAVA_HOME"/bin/java -cp "$HOME/lib/*:$PLUGIN_DIR" ${JVM_CONF} "$APPLICATION_NAME" \
+    nohup "$JAVA_HOME"/bin/java -cp "$HOME/lib/*:$HOME/plugins/*:$HOME/notifys/*:$HOME/schedulers/*:$HOME/parsers/*:$HOME/fss/*:$HOME/converts/*:$HOME/executors/*" ${JVM_CONF} "$APPLICATION_NAME" \
                   --spring.config.location="$HOME/configure/" > /dev/null 2>&1 &
     sleep 5
     job_before_apply_server

diff --git a/configure/etc/conf/application.properties b/configure/etc/conf/application.properties
@@ -1,5 +1,5 @@
 ################################### Banner configure #################################
-app.version=2024.03.7
+app.version=2024.03.8-SNAPSHOT
 
 ################################### Basic configure #################################
 server.port=9096

diff --git a/configure/etc/conf/convert.conf b/configure/etc/conf/convert.conf
@@ -0,0 +1,6 @@
+-- Convert list --
+datacap-convert-txt
+datacap-convert-json
+datacap-convert-none
+datacap-convert-csv
+datacap-convert-xml
diff --git a/configure/etc/conf/executor.conf b/configure/etc/conf/executor.conf
@@ -0,0 +1,3 @@
+-- Executor list --
+datacap-executor-local
+datacap-executor-seatunnel
diff --git a/configure/etc/conf/fs.conf b/configure/etc/conf/fs.conf
@@ -0,0 +1,4 @@
+-- Fs list --
+datacap-fs-local
+datacap-fs-qiniu
+datacap-fs-alioss
diff --git a/configure/etc/conf/notify.conf b/configure/etc/conf/notify.conf
@@ -0,0 +1,2 @@
+-- Notify list --
+datacap-notify-dingtalk
diff --git a/configure/etc/conf/parser.conf b/configure/etc/conf/parser.conf
@@ -0,0 +1,3 @@
+-- Parser list --
+datacap-parser-trino
+datacap-parser-mysql
diff --git a/configure/etc/conf/plugin.conf b/configure/etc/conf/plugin.conf
@@ -0,0 +1,54 @@
+-- Plugin list --
+datacap-native-alioss
+datacap-native-zookeeper
+datacap-native-redis
+datacap-native-kafka
+datacap-native-h2
+datacap-native-hdfs
+datacap-http-cratedb
+datacap-http-clickhouse
+datacap-http-ceresdb
+datacap-http-greptime
+datacap-http-questdb
+datacap-jdbc-clickhouse
+datacap-jdbc-cratedb
+datacap-jdbc-db2
+datacap-jdbc-dm
+datacap-jdbc-dremio
+datacap-jdbc-druid
+datacap-jdbc-duckdb
+datacap-jdbc-elasticsearch
+datacap-jdbc-h2
+datacap-jdbc-hive
+datacap-jdbc-ignite
+datacap-jdbc-impala
+datacap-jdbc-iotdb
+datacap-jdbc-kylin
+datacap-jdbc-kyuubi
+datacap-jdbc-monetdb
+datacap-jdbc-mongo
+datacap-jdbc-ydb
+datacap-jdbc-mysql
+datacap-jdbc-neo4j
+datacap-jdbc-oceanbase
+datacap-jdbc-oracle
+datacap-jdbc-phoenix
+datacap-jdbc-postgresql
+datacap-jdbc-presto
+datacap-jdbc-redis
+datacap-jdbc-snowflake
+datacap-jdbc-sqlserver
+datacap-jdbc-tdengine
+datacap-jdbc-trino
+datacap-jdbc-doris
+datacap-jdbc-starrocks
+datacap-jdbc-hologres
+datacap-plugin-pinot
+datacap-plugin-mongo-community
+datacap-plugin-cassandra
+datacap-plugin-matrixone
+datacap-plugin-scylladb
+datacap-plugin-paradedb
+datacap-plugin-timescale
+datacap-plugin-solr
+datacap-plugin-influxdb
diff --git a/configure/etc/conf/scheduler.conf b/configure/etc/conf/scheduler.conf
@@ -0,0 +1,2 @@
+-- Scheduler list --
+datacap-scheduler-local
diff --git a/configure/publish/publish-newVersion.sh b/configure/publish/publish-newVersion.sh
@@ -36,6 +36,9 @@ job_runner_apply() {
     echo "Apply new version for application ..."
     perl -pi -e 's/app\.version=.*/app.version='"$VERSION"'/g' configure/etc/conf/application.properties
 
+    echo "Apply new version for plugin ..."
+    perl -pi -e 's/VERSION=.*/VERSION='"$VERSION"'/g' configure/etc/bin/install-plugin.sh
+
     printf "Apply new version for web ...\n"
     # shellcheck disable=SC2164
     cd "$HOME"/core/datacap-ui

diff --git a/convert/datacap-convert-csv/pom.xml b/convert/datacap-convert-csv/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.edurt.datacap</groupId>
         <artifactId>datacap</artifactId>
-        <version>2024.03.7</version>
+        <version>2024.03.8-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -38,6 +38,17 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <configuration>
+                    <appendOutput>true</appendOutput>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+            </plugin>
             <plugin>
                 <groupId>org.jetbrains.dokka</groupId>
                 <artifactId>dokka-maven-plugin</artifactId>

diff --git a/convert/datacap-convert-json/pom.xml b/convert/datacap-convert-json/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.edurt.datacap</groupId>
         <artifactId>datacap</artifactId>
-        <version>2024.03.7</version>
+        <version>2024.03.8-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -38,6 +38,17 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <configuration>
+                    <appendOutput>true</appendOutput>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+            </plugin>
             <plugin>
                 <groupId>org.jetbrains.dokka</groupId>
                 <artifactId>dokka-maven-plugin</artifactId>

diff --git a/convert/datacap-convert-none/pom.xml b/convert/datacap-convert-none/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.edurt.datacap</groupId>
         <artifactId>datacap</artifactId>
-        <version>2024.03.7</version>
+        <version>2024.03.8-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -38,6 +38,17 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <configuration>
+                    <appendOutput>true</appendOutput>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+            </plugin>
             <plugin>
                 <groupId>org.jetbrains.dokka</groupId>
                 <artifactId>dokka-maven-plugin</artifactId>

diff --git a/convert/datacap-convert-spi/pom.xml b/convert/datacap-convert-spi/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.edurt.datacap</groupId>
         <artifactId>datacap</artifactId>
-        <version>2024.03.7</version>
+        <version>2024.03.8-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>

diff --git a/convert/datacap-convert-txt/pom.xml b/convert/datacap-convert-txt/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.edurt.datacap</groupId>
         <artifactId>datacap</artifactId>
-        <version>2024.03.7</version>
+        <version>2024.03.8-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -38,6 +38,17 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <configuration>
+                    <appendOutput>true</appendOutput>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+            </plugin>
             <plugin>
                 <groupId>org.jetbrains.dokka</groupId>
                 <artifactId>dokka-maven-plugin</artifactId>