Merge pull request #14226 from MinaProtocol/dkijania/fluent_ci_defini…

…tion [Develop][CI] Divide CI into gates
MinaProtocol · Oct 10, 2023 · cce3528 · cce3528
2 parents 695865f + 2aa43f1
commit cce3528
Show file tree

Hide file tree

Showing 43 changed files with 322 additions and 129 deletions.
diff --git a/buildkite/src/Command/MinaArtifact.dhall b/buildkite/src/Command/MinaArtifact.dhall
@@ -5,6 +5,9 @@ let S = ../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
 let Pipeline = ../Pipeline/Dsl.dhall
+let PipelineTag = ../Pipeline/Tag.dhall
+let PipelineMode = ../Pipeline/Mode.dhall
+
 let JobSpec = ../Pipeline/JobSpec.dhall
 
 let Command = ./Base.dhall
@@ -16,13 +19,16 @@ let DebianVersions = ../Constants/DebianVersions.dhall
 
 in
 
-let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
+let pipeline : DebianVersions.DebVersion -> PipelineMode.Type -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
+\(mode: PipelineMode.Type) -> 
     Pipeline.Config::{
       spec =
         JobSpec::{
           dirtyWhen = DebianVersions.dirtyWhen debVersion,
           path = "Release",
-          name = "MinaArtifact${DebianVersions.capitalName debVersion}"
+          name = "MinaArtifact${DebianVersions.capitalName debVersion}",
+          tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ],
+          mode = mode
         },
       steps = [
         Libp2p.step debVersion,
@@ -119,7 +125,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
 
 in
 {
-  bullseye  = pipeline DebianVersions.DebVersion.Bullseye
-  , buster  = pipeline DebianVersions.DebVersion.Buster
-  , focal   = pipeline DebianVersions.DebVersion.Focal
+  bullseye  = pipeline DebianVersions.DebVersion.Bullseye PipelineMode.Type.PullRequest
+  , buster  = pipeline DebianVersions.DebVersion.Buster PipelineMode.Type.PullRequest
+  , focal   = pipeline DebianVersions.DebVersion.Focal PipelineMode.Type.PullRequest
 }
diff --git a/buildkite/src/Jobs/Lint/Fast.dhall b/buildkite/src/Jobs/Lint/Fast.dhall
@@ -5,6 +5,7 @@ let B = ../../External/Buildkite.dhall
 let S = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
 
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
@@ -36,6 +37,7 @@ in  Pipeline.build
           ]
         , path = "Lint"
         , name = "Fast"
+        , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
         }
       , steps =
         [ Command.build

diff --git a/buildkite/src/Jobs/Lint/HelmChart.dhall b/buildkite/src/Jobs/Lint/HelmChart.dhall
@@ -4,6 +4,7 @@ let S = ../../Lib/SelectFiles.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -23,7 +24,8 @@ Pipeline.build
         S.exactly "buildkite/scripts/helm-ci" "sh"
       ],
       path = "Lint",
-      name = "HelmChart"
+      name = "HelmChart",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Lint/Merge.dhall b/buildkite/src/Jobs/Lint/Merge.dhall
@@ -4,6 +4,7 @@ let B = ../../External/Buildkite.dhall
 let SelectFiles = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Cmd = ../../Lib/Cmds.dhall
@@ -20,7 +21,8 @@ Pipeline.build
     spec = JobSpec::{
       dirtyWhen = [ SelectFiles.everything ],
       path = "Lint",
-      name = "Merge"
+      name = "Merge",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Lint/OCaml.dhall b/buildkite/src/Jobs/Lint/OCaml.dhall
@@ -7,6 +7,7 @@ let JobSpec = ../../Pipeline/JobSpec.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
 
 let RunInToolchain = ../../Command/RunInToolchain.dhall
 
@@ -31,6 +32,7 @@ in  Pipeline.build
                 [ dirtyDhallDir, S.strictlyStart (S.contains "src/") ]
               , path = "Lint"
               , name = "OCaml"
+              , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
               }
       , steps =
         [ Command.build

diff --git a/buildkite/src/Jobs/Lint/Rust.dhall b/buildkite/src/Jobs/Lint/Rust.dhall
@@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -19,7 +21,8 @@ Pipeline.build
     spec = JobSpec::{
       dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/Lint/Rust") ],
       path = "Lint",
-      name = "Rust"
+      name = "Rust",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Lint/TestnetAlerts.dhall b/buildkite/src/Jobs/Lint/TestnetAlerts.dhall
@@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -25,7 +27,8 @@ Pipeline.build
         S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts")
       ],
       path = "Lint",
-      name = "TestnetAlerts"
+      name = "TestnetAlerts",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Lint/ValidationService.dhall b/buildkite/src/Jobs/Lint/ValidationService.dhall
@@ -4,6 +4,8 @@ let List/map = Prelude.List.map
 
 let S = ../../Lib/SelectFiles.dhall
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let Cmd = ../../Lib/Cmds.dhall
 let Command = ../../Command/Base.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
@@ -52,7 +54,8 @@ in Pipeline.build Pipeline.Config::{
         S.strictlyStart (S.contains ValidationService.rootPath)
       ],
       path = "Lint",
-      name = "ValidationService"
+      name = "ValidationService",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
   steps = [
     Command.build Command.Config::{

diff --git a/buildkite/src/Jobs/Lint/Xrefcheck.dhall b/buildkite/src/Jobs/Lint/Xrefcheck.dhall
@@ -4,6 +4,8 @@ let B = ../../External/Buildkite.dhall
 let SelectFiles = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Cmd = ../../Lib/Cmds.dhall
@@ -23,7 +25,8 @@ Pipeline.build
         SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yml")
       ],
       path = "Lint",
-      name = "Xrefcheck"
+      name = "Xrefcheck",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Release/HelmRelease.dhall b/buildkite/src/Jobs/Release/HelmRelease.dhall
@@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -21,7 +23,8 @@ Pipeline.build
         S.exactly "buildkite/scripts/helm-ci" "sh"
       ],
       path = "Release",
-      name = "HelmRelease"
+      name = "HelmRelease",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall b/buildkite/src/Jobs/Release/ItnOrchestratorArtifact.dhall
@@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -29,7 +31,8 @@ Pipeline.build
           S.strictlyStart (S.contains "src/app/itn_orchestrator")
         ],
         path = "Release",
-        name = "ItnOrchestratorArtifact"
+        name = "ItnOrchestratorArtifact",
+        tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
       },
     steps = [
       DockerImage.generateStep spec

diff --git a/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall b/buildkite/src/Jobs/Release/LeaderboardArtifact.dhall
@@ -5,6 +5,8 @@ let S = ../../Lib/SelectFiles.dhall
 let D = S.PathPattern
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -27,7 +29,8 @@ Pipeline.build
           S.strictlyStart (S.contains "frontend/leaderboard")
         ],
         path = "Release",
-        name = "LeaderboardArtifact"
+        name = "LeaderboardArtifact",
+        tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
       },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall b/buildkite/src/Jobs/Release/MinaToolchainArtifactBullseye.dhall
@@ -0,0 +1,50 @@
+let Prelude = ../../External/Prelude.dhall
+
+let Cmd = ../../Lib/Cmds.dhall
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let Command = ../../Command/Base.dhall
+let Size = ../../Command/Size.dhall
+let DockerImage = ../../Command/DockerImage.dhall
+let DockerLogin = ../../Command/DockerLogin/Type.dhall
+
+
+in
+
+Pipeline.build
+  Pipeline.Config::{
+    spec =
+      JobSpec::{
+        dirtyWhen = [
+          S.strictlyStart (S.contains "dockerfiles/stages/1-"),
+          S.strictlyStart (S.contains "dockerfiles/stages/2-"),
+          S.strictlyStart (S.contains "dockerfiles/stages/3-"),
+          S.strictlyStart (S.contains "buildkite/src/Jobs/Release/MinaToolchainArtifact"),
+          S.strictly (S.contains "opam.export"),
+          -- Rust version has changed
+          S.strictlyEnd (S.contains "rust-toolchain.toml")
+        ],
+        path = "Release",
+        name = "MinaToolchainArtifactBullseye",
+        tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ]
+      },
+    steps = [
+
+      -- mina-toolchain Debian 11 "Bullseye" Toolchain
+      let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{
+        service="mina-toolchain",
+        deb_codename="bullseye",
+        extra_args="--no-cache",
+        step_key="toolchain-bullseye-docker-image"
+      }
+
+      in
+
+      DockerImage.generateStep toolchainBullseyeSpec
+
+    ]
+  }
diff --git a/.../Jobs/Release/MinaToolchainArtifact.dhall → ...Release/MinaToolchainArtifactBuster.dhall b/.../Jobs/Release/MinaToolchainArtifact.dhall → ...Release/MinaToolchainArtifactBuster.dhall
@@ -4,6 +4,8 @@ let Cmd = ../../Lib/Cmds.dhall
 let S = ../../Lib/SelectFiles.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+let PipelineMode = ../../Pipeline/Mode.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -28,22 +30,12 @@ Pipeline.build
           S.strictlyEnd (S.contains "rust-toolchain.toml")
         ],
         path = "Release",
-        name = "MinaToolchainArtifact"
+        name = "MinaToolchainArtifactBuster",
+        tags = [ PipelineTag.Type.Long, PipelineTag.Type.Release ],
+        mode = PipelineMode.Type.Stable        
       },
     steps = [
 
-      -- mina-toolchain Debian 11 "Bullseye" Toolchain
-      let toolchainBullseyeSpec = DockerImage.ReleaseSpec::{
-        service="mina-toolchain",
-        deb_codename="bullseye",
-        extra_args="--no-cache",
-        step_key="toolchain-bullseye-docker-image"
-      }
-
-      in
-
-      DockerImage.generateStep toolchainBullseyeSpec,
-
       -- mina-toolchain Debian 10 "Buster" Toolchain
       let toolchainBusterSpec = DockerImage.ReleaseSpec::{
         service="mina-toolchain",

diff --git a/buildkite/src/Jobs/Release/TestnetAlerts.dhall b/buildkite/src/Jobs/Release/TestnetAlerts.dhall
@@ -7,6 +7,8 @@ let S = ../../Lib/SelectFiles.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -24,7 +26,8 @@ Pipeline.build
         S.strictlyStart (S.contains "buildkite/src/Jobs/Release/TestnetAlerts")
       ],
       path = "Release",
-      name = "TestnetAlerts"
+      name = "TestnetAlerts",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Release/TraceTool.dhall b/buildkite/src/Jobs/Release/TraceTool.dhall
@@ -4,6 +4,8 @@ let S = ../../Lib/SelectFiles.dhall
 let Cmd = ../../Lib/Cmds.dhall
 
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
+
 let JobSpec = ../../Pipeline/JobSpec.dhall
 
 let Command = ../../Command/Base.dhall
@@ -20,7 +22,8 @@ Pipeline.build
     spec = JobSpec::{
       dirtyWhen = [ S.contains "src/app/trace-tool", S.strictlyStart (S.contains "buildkite/src/Jobs/TraceTool") ],
       path = "Release",
-      name = "TraceTool"
+      name = "TraceTool",
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Release ]
     },
     steps = [
       Command.build

diff --git a/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall b/buildkite/src/Jobs/Test/ArchiveNodeUnitTest.dhall
@@ -2,6 +2,7 @@ let Prelude = ../../External/Prelude.dhall
 let Cmd = ../../Lib/Cmds.dhall
 let S = ../../Lib/SelectFiles.dhall
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
 let JobSpec = ../../Pipeline/JobSpec.dhall
 let Command = ../../Command/Base.dhall
 let RunInToolchain = ../../Command/RunInToolchain.dhall
@@ -27,6 +28,7 @@ Pipeline.build
           ]
         , path = "Test"
         , name = "ArchiveNodeUnitTest"
+        , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ]
         }
     , steps =
     let outerDir : Text =

diff --git a/buildkite/src/Jobs/Test/CheckDhall.dhall b/buildkite/src/Jobs/Test/CheckDhall.dhall
@@ -3,6 +3,7 @@ let D = S.PathPattern
 
 let JobSpec = ../../Pipeline/JobSpec.dhall
 let Pipeline = ../../Pipeline/Dsl.dhall
+let PipelineTag = ../../Pipeline/Tag.dhall
 let Command = ../../Command/Base.dhall
 let Docker = ../../Command/Docker/Type.dhall
 let Size = ../../Command/Size.dhall
@@ -23,7 +24,8 @@ Pipeline.build
         S.exactly "buildkite/scripts/generate-jobs" "sh"
       ],
       path = "Test",
-      name = "CheckDhall"
+      name = "CheckDhall", 
+      tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Test ]
     },
     steps = [
     Command.build