Updated Documentation for the CI with mermaid sequence diagrams

.pre-commit-config.yaml

@@ -100,7 +100,7 @@ repos:
       - id: insert-license
         name: Add license for all XML files
         exclude: ^\.github/.*$
-        files: .*\.xml$
+        files: \.xml$
         args:
           - --comment-style
           - "<!--||-->"
@@ -128,6 +128,15 @@ repos:
           - --license-filepath
           - license-templates/LICENSE.txt
           - --fuzzy-match-generates-todo
+      - id: insert-license
+        name: Add license for all mermaid files
+        args:
+          - --comment-style
+          - "|%%|"
+          - --license-filepath
+          - license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
+        files: \.mermaid$
   - repo: https://github.com/thlorenz/doctoc.git
     rev: v1.4.0
     hooks:
@@ -335,7 +344,7 @@ repos:
         entry: ./scripts/ci/pre_commit/pre_commit_check_integrations.sh
         language: system
         pass_filenames: false
-        files: ^airflow/scripts/ci/libraries/_initialization.sh$|^breeze-complete$
+        files: ^common/_common_values.sh$|^breeze-complete$
       - id: build
         name: Check if image build is needed
         entry: ./scripts/ci/pre_commit/pre_commit_ci_build.sh 3.6 false
@@ -389,3 +398,9 @@ repos:
         files: ^airflow/www/.*\.(css|scss|sass)$
         # Keep dependency versions in sync w/ airflow/www/package.json
         additional_dependencies: ['stylelint@13.3.1', 'stylelint-config-standard@20.0.0']
+      - id: mermaid
+        name: Generate mermaid images
+        entry: ./scripts/ci/pre_commit/pre_commit_mermaid.sh
+        language: system
+        files: \.mermaid$
+        require_serial: true

.rat-excludes

@@ -31,6 +31,7 @@ metastore_db
 .*sql
 .*svg
 .*csv
+.*md5
 CHANGELOG.txt
 .*zip
 .*lock

images/ci/pull_request_ci_flow.md5

@@ -0,0 +1 @@
+e3aa93e8b46ca97b4aad06de23a10ad5  images/ci/pull_request_ci_flow.mermaid

images/ci/pull_request_ci_flow.mermaid

@@ -0,0 +1,108 @@
+%% Licensed to the Apache Software Foundation (ASF) under one
+%% or more contributor license agreements.  See the NOTICE file
+%% distributed with this work for additional information
+%% regarding copyright ownership.  The ASF licenses this file
+%% to you under the Apache License, Version 2.0 (the
+%% "License"); you may not use this file except in compliance
+%% with the License.  You may obtain a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+
+sequenceDiagram
+    Contributor -->> Fork: Submit PR
+    Fork -->> Airflow Repo: Start Pull Request
+    activate Airflow Repo
+    Note over Airflow Repo: Trigger PR build
+    Note over CI Build: pull_request<br>[Read Token]
+    Note over Build Images: workflow_run<br>[Write Token]
+    Airflow Repo -->> CI Build: Trigger 'pull_request' event
+    activate CI Build
+    CI Build -->> Build Images: Trigger 'workflow_run' event
+    activate Build Images
+    Note over Build Images: Find Duplicate<br>[CI Build]
+    opt Duplicated builds
+        Build Images -->> CI Build: Cancel
+    end
+    Note over Build Images: Find Duplicates<br>[Build Image]
+    opt Duplicated builds
+        Build Images -->> Build Images: Cancel
+    end
+    Note over Build Images: Find failed<br>[CI Build]
+    opt Failed builds
+        Build Images -->> CI Build: Cancel
+    end
+    Note over Build Images: Find failed<br>[Build Image]
+    opt Failed builds
+        Build Images -->> Build Images: Cancel
+    end
+    Note over Build Images: Build info
+    par 3.6, 3.7, 3.8
+        Note over Build Images: Build PROD Images<br>[GITHUB_RUN_ID]
+    and
+        Note over Build Images: Build CI Images<br>[GITHUB_RUN_ID]
+    end
+    par No CI image
+        Note over CI Build: Build info
+    and
+        Note over CI Build: Should trigger tests?
+    and
+        Note over CI Build: Helm tests
+    and
+        Note over CI Build: OpenAPI client gen
+    end
+    par 3.6, 3.7, 3.8
+        Build Images ->> GitHub Registry: Push CI Images
+        activate GitHub Registry
+    and
+        Build Images ->> GitHub Registry: Push PROD Images
+    end
+    opt If any step failed
+        Build Images -->> CI Build: Cancel triggering run
+    end
+    deactivate Build Images
+    Note over GitHub Registry: Tagged Images<br>[GITHUB_RUN_ID]
+    loop Wait for images
+        par 3.6, 3.7, 3.8
+            CI Build ->> CI Build: Pull CI Images
+            Note over CI Build: Wait for<br>[GITHUB_RUN_ID]
+        and
+            CI Build ->> CI Build: Pull PROD Images
+            Note over CI Build: Wait for<br>[GITHUB_RUN_ID]
+        end
+    end
+    par 3.6, 3.7, 3.8
+        GitHub Registry ->> CI Build: Pull CI Images
+    and
+        GitHub Registry ->> CI Build: Pull PROD Images
+    end
+    deactivate GitHub Registry
+    par 3.6, 3.7, 3.8
+        Note over CI Build: Run static checks
+    and
+        Note over CI Build: Build docs
+    and
+        Note over CI Build: Spell check docs
+    and
+        Note over CI Build: Backport packages
+    and
+        opt Triggered?
+            Note over CI Build: Tests
+        end
+    and
+        opt Triggered?
+            Note over CI Build: Kubernetes  Tests
+        end
+    end
+    Note over CI Build: Quarantined tests
+    Note over CI Build: Merge Coverage
+    CI Build -->> Coverage.io: Upload Coverage
+    CI Build -->> Airflow Repo: Status Check for CI Build
+    deactivate Airflow Repo
+    deactivate CI Build

images/ci/push_ci_flow.md5

@@ -0,0 +1 @@
+95db6b17e315a47318eaf6260b3f0fd3  images/ci/push_ci_flow.mermaid

images/ci/push_ci_flow.mermaid

@@ -0,0 +1,117 @@
+%% Licensed to the Apache Software Foundation (ASF) under one
+%% or more contributor license agreements.  See the NOTICE file
+%% distributed with this work for additional information
+%% regarding copyright ownership.  The ASF licenses this file
+%% to you under the Apache License, Version 2.0 (the
+%% "License"); you may not use this file except in compliance
+%% with the License.  You may obtain a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+
+sequenceDiagram
+    Committer -->> Airflow Repo: Merge PR
+    activate Airflow Repo
+    Note over Airflow Repo: Trigger PR build
+    Note over CI Build: push<br>[Write Token]
+    Note over Build Images: workflow_run<br>[Write Token]
+    Airflow Repo -->> CI Build: Trigger 'pull_request' event
+    activate CI Build
+    CI Build -->> Build Images: Trigger 'workflow_run' event
+    activate Build Images
+    Note over Build Images: Find Duplicate<br>[CI Build]
+    opt Duplicated builds
+        Build Images -->> CI Build: Cancel
+    end
+    Note over Build Images: Find Duplicates<br>[Build Image]
+    opt Duplicated builds
+        Build Images -->> Build Images: Cancel
+    end
+    Note over Build Images: Find failed<br>[CI Build]
+    opt Failed builds
+        Build Images -->> CI Build: Cancel
+    end
+    Note over Build Images: Find failed<br>[Build Image]
+    opt Failed builds
+        Build Images -->> Build Images: Cancel
+    end
+    Note over Build Images: Build info
+    par 3.6, 3.7, 3.8
+        Note over Build Images: Update constraints
+        Note over Build Images: Build PROD Images<br>[GITHUB_RUN_ID]
+    and
+        Note over Build Images: Update constraints
+        Note over Build Images: Build CI Images<br>[GITHUB_RUN_ID]
+    end
+    par No CI image
+        Note over CI Build: Build info
+    and
+        Note over CI Build: Test always!
+    and
+        Note over CI Build: Helm tests
+    and
+        Note over CI Build: OpenAPI client gen
+    end
+    par 3.6, 3.7, 3.8
+        Build Images ->> GitHub Registry: Push CI Images
+        activate GitHub Registry
+    and
+        Build Images ->> GitHub Registry: Push PROD Images
+    end
+    opt If any step failed
+        Build Images -->> CI Build: Cancel triggering run
+    end
+    deactivate Build Images
+    Note over GitHub Registry: Tagged Images<br>[GITHUB_RUN_ID]
+    loop Wait for images
+        par 3.6, 3.7, 3.8
+            CI Build ->> CI Build: Pull CI Images
+            Note over CI Build: Wait for<br>[GITHUB_RUN_ID]
+        and
+            CI Build ->> CI Build: Pull PROD Images
+            Note over CI Build: Wait for<br>[GITHUB_RUN_ID]
+        end
+    end
+    par 3.6, 3.7, 3.8
+        GitHub Registry ->> CI Build: Pull CI Images
+    and
+        GitHub Registry ->> CI Build: Pull PROD Images
+    end
+    deactivate GitHub Registry
+    par 3.6, 3.7, 3.8
+        Note over CI Build: Run static checks
+    and
+        Note over CI Build: Build docs
+    and
+        Note over CI Build: Spell check docs
+    and
+        Note over CI Build: Backport packages
+    and
+        Note over CI Build: Tests
+    and
+        Note over CI Build: Kubernetes  Tests
+    end
+    Note over CI Build: Quarantined tests
+    Note over CI Build: Merge Coverage
+    CI Build -->> Coverage.io: Upload Coverage
+    Note over CI Build: Tag image<br>[COMMIT_SHA]
+    par 3.6, 3.7, 3.8
+        CI Build ->> GitHub Registry: Push CI Images
+        activate GitHub Registry
+    and
+        CI Build ->> GitHub Registry: Push PROD Images
+    end
+    Note over GitHub Registry: Tagged Images<br>[COMMIT_SHA]
+    par 3.6, 3.7, 3.8
+        CI Build ->> Airflow Repo: Push Constraints
+    end
+    deactivate GitHub Registry
+    CI Build -->> Airflow Repo: Status Check for CI Build
+    deactivate Airflow Repo
+    deactivate CI Build

images/ci/scheduled_ci_flow.md5

@@ -0,0 +1 @@
+f6ea87f0b4d990fd48a9dbec2a2a4f2d  images/ci/scheduled_ci_flow.mermaid

images/ci/scheduled_ci_flow.mermaid

@@ -0,0 +1,111 @@
+%% Licensed to the Apache Software Foundation (ASF) under one
+%% or more contributor license agreements.  See the NOTICE file
+%% distributed with this work for additional information
+%% regarding copyright ownership.  The ASF licenses this file
+%% to you under the Apache License, Version 2.0 (the
+%% "License"); you may not use this file except in compliance
+%% with the License.  You may obtain a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+
+sequenceDiagram
+    CRON job -->> Airflow Repo: Nightly run
+    activate Airflow Repo
+    Note over Airflow Repo: Trigger PR build
+    Note over CI Build: push<br>[Write Token]
+    Note over Build Images: workflow_run<br>[Write Token]
+    Airflow Repo -->> CI Build: Trigger 'pull_request' event
+    activate CI Build
+    CI Build -->> Build Images: Trigger 'workflow_run' event
+    activate Build Images
+    Note over Build Images: Find Duplicate<br>[CI Build]
+    opt Duplicated builds
+        Build Images -->> CI Build: Cancel
+    end
+    Note over Build Images: Find Duplicates<br>[Build Image]
+    opt Duplicated builds
+        Build Images -->> Build Images: Cancel
+    end
+    Note over Build Images: Find failed<br>[CI Build]
+    opt Failed builds
+        Build Images -->> CI Build: Cancel
+    end
+    Note over Build Images: Find failed<br>[Build Image]
+    opt Failed builds
+        Build Images -->> Build Images: Cancel
+    end
+    Note over Build Images: Build info
+    par 3.6, 3.7, 3.8
+        Note over Build Images: Build from scratch
+        Note over Build Images: Update constraints
+        Note over Build Images: Build PROD Images<br>[GITHUB_RUN_ID]
+    and
+        Note over Build Images: Build from scratch
+        Note over Build Images: Update constraints
+        Note over Build Images: Build CI Images<br>[GITHUB_RUN_ID]
+    end
+    par No CI image
+        Note over CI Build: Build info
+    and
+        Note over CI Build: Test always!
+    and
+        Note over CI Build: Helm tests
+    and
+        Note over CI Build: OpenAPI client gen
+    end
+    par 3.6, 3.7, 3.8
+        Build Images ->> GitHub Registry: Push CI Images
+        activate GitHub Registry
+    and
+        Build Images ->> GitHub Registry: Push PROD Images
+    end
+    opt If any step failed
+        Build Images -->> CI Build: Cancel triggering run
+    end
+    deactivate Build Images
+    Note over GitHub Registry: Tagged Images<br>[GITHUB_RUN_ID]
+    loop Wait for images
+        par 3.6, 3.7, 3.8
+            CI Build ->> CI Build: Pull CI Images
+            Note over CI Build: Wait for<br>[GITHUB_RUN_ID]
+        and
+            CI Build ->> CI Build: Pull PROD Images
+            Note over CI Build: Wait for<br>[GITHUB_RUN_ID]
+        end
+    end
+    par 3.6, 3.7, 3.8
+        GitHub Registry ->> CI Build: Pull CI Images
+    and
+        GitHub Registry ->> CI Build: Pull PROD Images
+    end
+    deactivate GitHub Registry
+    par 3.6, 3.7, 3.8
+        Note over CI Build: Run static checks
+    and
+        Note over CI Build: Build docs
+    and
+        Note over CI Build: Spell check docs
+    and
+        Note over CI Build: Backport packages
+    and
+        Note over CI Build: Tests
+    and
+        Note over CI Build: Kubernetes  Tests
+    end
+    Note over CI Build: Quarantined tests
+    Note over CI Build: Merge Coverage
+    CI Build -->> Coverage.io: Upload Coverage
+    par 3.6, 3.7, 3.8
+        CI Build ->> Airflow Repo: Push Constraints
+    end
+    CI Build ->> Airflow Repo: Push Nightly Tag
+    CI Build -->> Airflow Repo: Status Check for CI Build
+    deactivate Airflow Repo
+    deactivate CI Build