Merge pull request #10213 from MinaProtocol/lk86/unreviewed-changes-f…

…or-beta2

Lk86/unreviewed changes for beta2

.gitignore

@@ -36,6 +36,8 @@ src/libp2p_ipc/libp2p_ipc_capnp.mli
 maintenance/*.dot
 maintenance/*.png
 
+scripts/genesis_ledger.json
+
 automation/*config.json
 automation/!bsconfig.json
 automation/*/venv/*

buildkite/scripts/merges-cleanly.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+BRANCH=$1
+CURRENT=$(git branch --show-current)
+echo 'Testing for conflicts between the current branch `'"${CURRENT}"'` and `'"${BRANCH}"'`...'
+
+# Adapted from this stackoverflow answer: https://stackoverflow.com/a/10856937
+# The git merge-tree command shows the content of a 3-way merge without
+# touching the index, which we can then search for conflict markers.
+
+git merge-tree `git merge-base origin/$BRANCH HEAD` HEAD origin/$BRANCH | grep -A 25 "^+<<<<<<<"
+
+RET=$?
+
+if [ $RET -eq 0 ]; then
+  # Found a conflict
+  echo "[ERROR] This pull request conflicts with $BRANCH, please open a new pull request against $BRANCH at this link:"
+  echo "https://github.com/MinaProtocol/mina/compare/${BRANCH}...${BUILDKITE_BRANCH}"
+  exit 1
+else
+  echo "No conflicts found against upstream branch ${BRANCH}"
+  exit 0
+fi

buildkite/src/Command/DockerImage.dhall

@@ -58,7 +58,7 @@ let generateStep = \(spec : ReleaseSpec.Type) ->
     Command.build
       Command.Config::{
         commands  = commands,
-        label = "Release Docker Image: ${spec.step_key}",
+        label = "Docker: ${spec.step_key}",
         key = spec.step_key,
         target = Size.XLarge,
         docker_login = Some DockerLogin::{=},

buildkite/src/Command/MinaArtifact.dhall

@@ -20,6 +20,8 @@ let dirtyWhen = [
   S.strictlyStart (S.contains "buildkite/src/Command/MinaArtifact"),
   S.exactly "buildkite/scripts/build-artifact" "sh",
   S.exactly "buildkite/scripts/connect-to-mainnet-on-compatible" "sh",
+  S.strictlyStart (S.contains "buildkite/src/Jobs/Test"),
+  S.strictlyStart (S.contains "buildkite/src/Command"),
   S.strictlyStart (S.contains "dockerfiles"),
   S.strictlyStart (S.contains "scripts")
 ]
@@ -49,7 +51,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
               -- add zexe standardization preprocessing step (see: https://github.com/MinaProtocol/mina/pull/5777)
               "PREPROCESSOR=./scripts/zexe-standardize.sh"
             ] "./buildkite/scripts/build-artifact.sh",
-            label = "Build Mina packages for Debian ${DebianVersions.capitalName debVersion}",
+            label = "Build Mina for ${DebianVersions.capitalName debVersion}",
             key = "build-deb-pkg",
             target = Size.XLarge,
             retries = [ Command.Retry::{ exit_status = +2, limit = Some 2 } ] -- libp2p error
@@ -97,9 +99,9 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
         let rosettaSpec = DockerImage.ReleaseSpec::{
           deps=DebianVersions.dependsOnGitEnv,
           service="mina-rosetta",
-          extra_args="--build-arg MINA_BRANCH=\\\${BUILDKITE_BRANCH}",
+          extra_args="--build-arg MINA_BRANCH=\\\${BUILDKITE_BRANCH} --no-cache",
           deb_codename="${DebianVersions.lowerName debVersion}",
-          step_key="rosetta-mainnet-${DebianVersions.lowerName debVersion}-docker-image"
+          step_key="rosetta-${DebianVersions.lowerName debVersion}-docker-image"
         }
 
         in
@@ -111,7 +113,10 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
 
 in
 {
-  buster  = pipeline DebianVersions.DebVersion.Buster,
-  stretch = pipeline DebianVersions.DebVersion.Stretch,
-  dirtyWhen = dirtyWhen
+  bullseye  = pipeline DebianVersions.DebVersion.Bullseye
+  , buster  = pipeline DebianVersions.DebVersion.Buster
+  , stretch = pipeline DebianVersions.DebVersion.Stretch
+  , bionic = pipeline DebianVersions.DebVersion.Bionic
+  , focal   = pipeline DebianVersions.DebVersion.Focal
+  , dirtyWhen = dirtyWhen
 }

buildkite/src/Command/RunInToolchain.dhall

@@ -5,6 +5,17 @@ let S = ../Lib/SelectFiles.dhall
 
 let r = Cmd.run
 
+let runInToolchainBullseye : List Text -> Text -> List Cmd.Type =
+  \(environment : List Text) ->
+  \(innerScript : Text) ->
+    [ Mina.fixPermissionsCommand ] # [
+      Cmd.runInDocker
+        (Cmd.Docker::{ image = (../Constants/ContainerImages.dhall).minaToolchainBullseye, extraEnv = environment })
+        (innerScript)
+    ]
+
+in
+
 let runInToolchainBuster : List Text -> Text -> List Cmd.Type =
   \(environment : List Text) ->
   \(innerScript : Text) ->
@@ -25,6 +36,20 @@ let runInToolchainStretch : List Text -> Text -> List Cmd.Type =
 
 in
 
-{ runInToolchainBuster = runInToolchainBuster
-, runInToolchainStretch = runInToolchainStretch
+let runInToolchainFocal : List Text -> Text -> List Cmd.Type =
+  \(environment : List Text) ->
+  \(innerScript : Text) ->
+    [ Mina.fixPermissionsCommand ] # [
+      Cmd.runInDocker
+        (Cmd.Docker::{ image = (../Constants/ContainerImages.dhall).minaToolchainFocal, extraEnv = environment })
+        (innerScript)
+    ]
+
+in
+
+{
+  runInToolchainBullseye = runInToolchainBullseye
+  , runInToolchainBuster = runInToolchainBuster
+  , runInToolchainStretch = runInToolchainStretch
+  , runInToolchainFocal = runInToolchainFocal
 }

buildkite/src/Constants/ContainerImages.dhall

@@ -1,9 +1,11 @@
 -- TODO: Automatically push, tag, and update images #4862
-
+-- NOTE: minaToolchainStretch is also used for building Ubuntu Bionic packages in CI
 {
   toolchainBase = "codaprotocol/ci-toolchain-base:v3",
-  minaToolchainStretch = "gcr.io/o1labs-192920/mina-toolchain@sha256:7ef5827fa0854a0bcfdee69dcc0c2c7aef86e1662c630e71f07c1f9162e757fa",
-  minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:71477097441872b96cce1ec6468c6aa3250bbc7159ecbcc9a716f5b54e16ec05",
+  minaToolchainStretch = "gcr.io/o1labs-192920/mina-toolchain@sha256:c4873c52041996a4eacdd96d0310b909a099689552c8b4fec7d464872abd1e1f",
+  minaToolchainBuster = "gcr.io/o1labs-192920/mina-toolchain@sha256:b53c4948599ec40ec1ce9178d84ce48e88db472da0005b9a6b4260e0d4aaa95c",
+  minaToolchainBullseye = "gcr.io/o1labs-192920/mina-toolchain@sha256:00dbcb936954f10b4995ad28535d08e32d6d2dbc31531e846307d156e610af04",
+  minaToolchainFocal = "gcr.io/o1labs-192920/mina-toolchain@sha256:ad94c2373f45a8a5af354ed12d5eb34ad64b1140c0811a9935212ea0a5d3c4b0",
   delegationBackendToolchain = "gcr.io/o1labs-192920/delegation-backend-production@sha256:8ca5880845514ef56a36bf766a0f9de96e6200d61b51f80d9f684a0ec9c031f4",
   elixirToolchain = "elixir:1.10-alpine",
   rustToolchain = "codaprotocol/coda:toolchain-rust-e855336d087a679f76f2dd2bbdc3fdfea9303be3",

buildkite/src/Constants/DebianVersions.dhall

@@ -4,34 +4,63 @@ let ContainerImages = ./ContainerImages.dhall
 
 let dependsOnGitEnv = [ { name = "GitEnvUpload", key = "upload-git-env" } ]
 
-let DebVersion = < Buster | Stretch >
+let DebVersion = < Bullseye | Buster | Stretch | Focal | Bionic >
 
 let capitalName = \(debVersion : DebVersion) ->
-  merge { Buster = "Buster", Stretch = "Stretch" } debVersion
+  merge {
+    Bullseye = "Bullseye"
+    , Buster = "Buster"
+    , Stretch = "Stretch"
+    , Focal = "Focal"
+    , Bionic = "Bionic"
+  } debVersion
 
 let lowerName = \(debVersion : DebVersion) ->
-  merge { Buster = "buster", Stretch = "stretch" } debVersion
+  merge {
+    Bullseye = "bullseye"
+    , Buster = "buster"
+    , Stretch = "stretch"
+    , Focal = "focal"
+    , Bionic = "bionic"
+  } debVersion
 
+--- Bionic and Stretch are so similar that they share a toolchain image
 let toolchainRunner = \(debVersion : DebVersion) ->
-  merge { Buster = RunInToolchain.runInToolchainBuster, Stretch = RunInToolchain.runInToolchainStretch } debVersion
+  merge {
+    Bullseye = RunInToolchain.runInToolchainBullseye
+    , Buster = RunInToolchain.runInToolchainBuster
+    , Stretch = RunInToolchain.runInToolchainStretch
+    , Focal = RunInToolchain.runInToolchainFocal
+    , Bionic = RunInToolchain.runInToolchainStretch
+  } debVersion
 
+--- Bionic and Stretch are so similar that they share a toolchain image
 let toolchainImage = \(debVersion : DebVersion) ->
-  merge { Buster = ContainerImages.minaToolchainBuster, Stretch = ContainerImages.minaToolchainStretch } debVersion
+  merge { 
+    Bullseye = ContainerImages.minaToolchainBullseye
+    , Buster = ContainerImages.minaToolchainBuster
+    , Stretch = ContainerImages.minaToolchainStretch
+    , Focal = ContainerImages.minaToolchainFocal
+    , Bionic = ContainerImages.minaToolchainStretch
+  } debVersion
 
 let dependsOn = \(debVersion : DebVersion) ->
   merge {
-    Buster = dependsOnGitEnv # [{ name = "MinaArtifactBuster", key = "build-deb-pkg" }],
-    Stretch = dependsOnGitEnv # [{ name = "MinaArtifactStretch", key = "build-deb-pkg" }]
+    Bullseye = dependsOnGitEnv # [{ name = "MinaArtifactBullseye", key = "build-deb-pkg" }]
+    , Buster = dependsOnGitEnv # [{ name = "MinaArtifactBuster", key = "build-deb-pkg" }]
+    , Stretch = dependsOnGitEnv # [{ name = "MinaArtifactStretch", key = "build-deb-pkg" }]
+    , Bionic = dependsOnGitEnv # [{ name = "MinaArtifactBionic", key = "build-deb-pkg" }]
+    , Focal = dependsOnGitEnv # [{ name = "MinaArtifactFocal", key = "build-deb-pkg" }]
   } debVersion
 
 in
 
 {
-  DebVersion = DebVersion,
-  capitalName = capitalName,
-  lowerName = lowerName,
-  toolchainRunner = toolchainRunner,
-  toolchainImage = toolchainImage,
-  dependsOn = dependsOn,
-  dependsOnGitEnv = dependsOnGitEnv
+  DebVersion = DebVersion
+  , capitalName = capitalName
+  , lowerName = lowerName
+  , toolchainRunner = toolchainRunner
+  , toolchainImage = toolchainImage
+  , dependsOn = dependsOn
+  , dependsOnGitEnv = dependsOnGitEnv
 }

buildkite/src/Jobs/Lint/Merge.dhall

@@ -22,7 +22,17 @@ Pipeline.build
     steps = [
       Command.build
         Command.Config::{
-          commands = [ Cmd.run "buildkite/scripts/merges-cleanly-to-develop.sh" ]
+          commands = [ Cmd.run "buildkite/scripts/merges-cleanly.sh compatible"]
+          , label = "Check merges cleanly into compatible"
+          , key = "clean-merge-compatible"
+          , target = Size.Small
+          , docker = Some Docker::{
+              image = (../../Constants/ContainerImages.dhall).toolchainBase
+            }
+        },
+      Command.build
+        Command.Config::{
+          commands = [ Cmd.run "buildkite/scripts/merges-cleanly.sh develop"]
           , label = "Check merges cleanly into develop"
           , key = "clean-merge-develop"
           , target = Size.Small

buildkite/src/Jobs/Release/MinaArtifactBionic.dhall

@@ -0,0 +1,6 @@
+let ArtifactPipelines = ../../Command/MinaArtifact.dhall
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+in
+
+Pipeline.build ArtifactPipelines.bionic

buildkite/src/Jobs/Release/MinaArtifactBullseye.dhall

@@ -0,0 +1,6 @@
+let ArtifactPipelines = ../../Command/MinaArtifact.dhall
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+in
+
+Pipeline.build ArtifactPipelines.bullseye

buildkite/src/Jobs/Release/MinaArtifactFocal.dhall

@@ -0,0 +1,6 @@
+let ArtifactPipelines = ../../Command/MinaArtifact.dhall
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+in
+
+Pipeline.build ArtifactPipelines.focal