apache · imizao · Sep 30, 2022 · Oct 1, 2022 · Oct 1, 2022 · Oct 5, 2022
diff --git a/.asf.yaml b/.asf.yaml
@@ -16,24 +16,20 @@
 #
 
 github:
-  description: Apache DolphinScheduler is a distributed and extensible workflow scheduler platform with powerful DAG visual interfaces, dedicated to solving complex job dependencies in the data pipeline and providing various types of jobs available out of box.
+  description: Apache DolphinScheduler is the modern data workflow orchestration platform with powerful user interface, dedicated to solving complex task dependencies in the data pipeline and providing various types of jobs available `out of the box`
   homepage: https://dolphinscheduler.apache.org/
   labels:
+    - cloud-native
+    - orchestration
     - airflow
-    - schedule
+    - workflow
     - job-scheduler
-    - oozie
     - task-scheduler
     - azkaban
-    - distributed-schedule-system
-    - workflow-scheduling-system
-    - etl-dependency
-    - workflow-platform
-    - cronjob-schedule
-    - job-schedule
-    - task-schedule
+    - workflow-orchestration
     - workflow-schedule
-    - data-schedule
+    - powerful-data-pipelines
+    - data-pipelines
   enabled_merge_buttons:
     squash: true
     merge: false
@@ -45,6 +41,8 @@ github:
           - Build
           - Unit Test
           - E2E
+          - Docs
+          - "Mergeable: milestone-label-check"
       required_pull_request_reviews:
         dismiss_stale_reviews: true
         required_approving_review_count: 1
diff --git a/.dlc.json b/.dlc.json
@@ -20,6 +20,14 @@
       "replacement": "https://dolphinscheduler.apache.org/zh-cn/download/download.html"
     }
   ],
+  "httpHeaders": [
+    {
+      "urls": ["https://docs.github.com/"],
+      "headers": {
+        "Accept-Encoding": "zstd, br, gzip, deflate"
+      }
+    }
+  ],
   "timeout": "10s",
   "retryOn429": true,
   "retryCount": 10,

diff --git a/...heduler-python/pydolphinscheduler/.flake8 → .flake8 b/...heduler-python/pydolphinscheduler/.flake8 → .flake8
@@ -19,22 +19,10 @@
 max-line-length = 110
 exclude =
     .git,
-    __pycache__,
-    .pytest_cache,
-    *.egg-info,
-    docs/source/conf.py
-    old,
-    build,
-    dist,
-    htmlcov,
-    .tox,
     dist,
 ignore = 
     # It's clear and not need to add docstring
     D107,  # D107: Don't require docstrings on __init__
     D105,  # D105: Missing docstring in magic method
     # Conflict to Black
     W503   # W503: Line breaks before binary operators
-per-file-ignores =
-    */pydolphinscheduler/side/__init__.py:F401
-    */pydolphinscheduler/tasks/__init__.py:F401
@@ -38,7 +38,6 @@
 /dolphinscheduler-task-plugin/ @caishunfeng @SbloodyS @zhuangchong
 /dolphinscheduler-tools/ @caishunfeng @SbloodyS @zhongjiajie @EricGao888
 /script/ @caishunfeng @SbloodyS @zhongjiajie @EricGao888
-/dolphinscheduler-python/ @zhongjiajie
 /dolphinscheduler-ui/ @songjianet @Amy0104
 /docs/ @zhongjiajie @Tianqi-Dotes @EricGao888
 /licenses/ @kezhenxu94 @zhongjiajie

@@ -99,9 +99,9 @@ body:
         Which version of Apache DolphinScheduler are you running? We only accept bugs report from the LTS projects.
       options:
         - dev
-        - 3.0.0
-        - 2.0.6
-        - 2.0.5
+        - 3.1.x
+        - 3.0.x
+        - 2.0.x
     validations:
       required: true
 

@@ -17,7 +17,7 @@
 name: Feature request
 description: Suggest an idea for this project
 title: "[Feature][Module Name] Feature title"
-labels: [ "new feature", "Waiting for reply" ]
+labels: [ "feature", "Waiting for reply" ]
 body:
   - type: markdown
     attributes:

@@ -15,9 +15,6 @@
 # limitations under the License.
 #
 
-Python:
-  - any: ['dolphinscheduler-python/**/*']
-
 backend:
   - 'dolphinscheduler-alert/**/*'
   - 'dolphinscheduler-api/**/*'
@@ -40,7 +37,6 @@ backend:
 
 document:
   - 'docs/**/*'
-  - 'dolphinscheduler-python/pydolphinscheduler/docs/**/*'
 
 CI&CD:
   - any: ['.github/**/*']

@@ -32,7 +32,7 @@ runs:
   using: "composite"
   steps:
     - name: Check License Header
-      uses: apache/skywalking-eyes@30367d8286e324d5efc58de4c70c37ea3648306d
+      uses: apache/skywalking-eyes/header@main
 
     - shell: bash
       run: ./mvnw spotless:check
@@ -22,7 +22,7 @@ mergeable:
   # GitHub action workflow `pull_requests` https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request like,
   # which only trigger runs when a pull_request event's activity type is opened, synchronize, or reopened
   - when: pull_request.opened, pull_request.reopened, pull_request.synchronize
-    name: synchronize change for sql files
+    name: sync-sql-ddl
     validate:
       # Sql files must change synchronize
       - do: dependent
@@ -44,3 +44,19 @@ mergeable:
     pass:
       - do: labels
         delete: 'sql not sync'
+
+  - when: pull_request.*
+    name: milestone-label-check
+    validate:
+      - do: milestone
+        no_empty:
+          enabled: false # Cannot be empty when true.
+          message: 'Milestone is required and cannot be empty.'
+      - do: label
+        and:
+          - must_include:
+              regex: 'feature|bug|improvement|document|chore'
+              message: 'Label must include one of the following: `feature`, `bug`, `improvement`, `document`, `chore`'
+          - must_include:
+              regex: '\d+\.\d+\.\w+'
+              message: 'Label must include one or more version numbers.'
@@ -37,11 +37,17 @@ export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zo
 # Tasks related configurations, need to change the configuration if you use the related tasks.
 export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
 export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
-export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
-export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
+export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
 export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
 export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
 export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
 export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
 
-export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
+
+# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
+#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
+#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
+#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
+#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
+#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS
@@ -37,11 +37,17 @@ export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-zo
 # Tasks related configurations, need to change the configuration if you use the related tasks.
 export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
 export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
-export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
-export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
+export SPARK_HOME=${SPARK_HOME:-/opt/soft/spark}
 export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
 export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
 export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
 export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
 
-export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
+
+# applicationId auto collection related configuration, the following configurations are unnecessary if setting appId.collect=log
+#export HADOOP_CLASSPATH=`hadoop classpath`:${DOLPHINSCHEDULER_HOME}/tools/libs/*
+#export SPARK_DIST_CLASSPATH=$HADOOP_CLASSPATH:$SPARK_DIST_CLASS_PATH
+#export HADOOP_CLIENT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$HADOOP_CLIENT_OPTS
+#export SPARK_SUBMIT_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$SPARK_SUBMIT_OPTS
+#export FLINK_ENV_JAVA_OPTS="-javaagent:${DOLPHINSCHEDULER_HOME}/tools/libs/aspectjweaver-1.9.7.jar":$FLINK_ENV_JAVA_OPTS
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "CodeQL"
+
+on:
+  push:
+    branches:
+      - 'dev'
+  pull_request:
+    branches:
+      - 'dev'
+
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    if: (github.event_name == 'schedule' && github.repository == 'apache/dolphinscheduler') || (github.event_name != 'schedule')
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        submodules: true
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: java
+        queries: +security-and-quality
+
+    - run: |
+        ./mvnw -B clean install \
+          -Dmaven.test.skip \
+          -Dmaven.javadoc.skip \
+          -Dspotless.skip=true \
+          -Prelease
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
@@ -18,11 +18,6 @@ name: Docs
 
 on:
   pull_request:
-    paths:
-      - '.github/workflows/docs.yml'
-      - '**/*.md'
-      - 'docs/**'
-      - '.dlc.json'
   schedule:
     - cron: '0 18 * * *'  # TimeZone: UTC 0
 
@@ -31,8 +26,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  style:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Style Check
+        run: ./mvnw spotless:check
   img-check:
-    name: Image Check
     timeout-minutes: 15
     runs-on: ubuntu-latest
     defaults:
@@ -49,7 +50,6 @@ jobs:
       - name: Run Image Check
         run: python img_utils.py -v check
   dead-link:
-    name: Dead Link
     if: (github.event_name == 'schedule' && github.repository == 'apache/dolphinscheduler') || (github.event_name != 'schedule')
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -62,3 +62,17 @@ jobs:
           for file in $(find . -name "*.md"); do
             markdown-link-check -c .dlc.json -q "$file"
           done
+  result:
+    name: Docs
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    needs:
+      - style
+      - img-check
+      - dead-link
+    if: success()
+    steps:
+      - name: success
+        run: |
+          echo "Docs check success"
+          exit 0