Merge main into canton-3.4

LATEST_RELEASE

@@ -1 +1 @@
-0.4.20
+0.4.21

apps/app/src/main/resources/application.conf

@@ -1,3 +1,4 @@
 # This is deliberately lower than ConsoleCommandTimeout.defaultRequestTimeout
 pekko.http.server.request-timeout = 38 seconds
-pekko.http.server.parsing.error-handler="org.lfdecentralizedtrust.splice.http.PekkoHttpParsingErrorHandler$"
+pekko.http.server.parsing.error-handler="org.lfdecentralizedtrust.splice.http.PekkoHttpParsingErrorHandler$"
+pekko.http.server.parsing.ignore-illegal-header-for = ["user-agent"]

apps/app/src/main/scala/org/lfdecentralizedtrust/splice/config/ConfigTransforms.scala

@@ -197,6 +197,9 @@ object ConfigTransforms {
       setDefaultGrpcDeadlineForBuyExtraTraffic(),
       setDefaultGrpcDeadlineForTreasuryService(),
       disableZeroFees(),
+      updateAllAutomationConfigs(
+        _.copy(rewardOperationRoundsCloseBufferDuration = NonNegativeFiniteDuration.ofMillis(100))
+      ),
     )
   }
 
@@ -336,6 +339,12 @@ object ConfigTransforms {
   def updateInitialTickDuration(tick: NonNegativeFiniteDuration): ConfigTransform = {
     ConfigTransforms.updateAllSvAppFoundDsoConfigs_(
       _.copy(initialTickDuration = tick)
+    ) compose ConfigTransforms.updateAllAutomationConfigs(config =>
+      if (config.pollingInterval.toInternal > tick.toInternal)
+        config.copy(
+          pollingInterval = tick
+        )
+      else config
     )
   }
 

apps/app/src/test/resources/include/sequencers.conf

@@ -25,6 +25,9 @@ _sequencer_reference_template {
   sequencer-client {
     use-new-connection-pool = false
   }
+  parameters {
+    batching.max-pruning-time-interval = "10 minutes"
+  }
   sequencer {
     config {
       storage = ${_shared.storage}

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/EnvironmentDefinition.scala

@@ -429,12 +429,6 @@ case class EnvironmentDefinition(
               .replace(NonNegativeFiniteDuration.Zero)
           )(conf)
       )
-      .addConfigTransform((_, conf) =>
-        ConfigTransforms.updateAllAutomationConfigs(
-          // disable round based triggers because tests don't advance time for the triggers to run
-          _.focus(_.enableNewRewardTriggerScheduling).replace(false)
-        )(conf)
-      )
       .withSequencerConnectionsFromScanDisabled(10_000)
 
   override lazy val environmentFactory: EnvironmentFactory[SpliceConfig, SpliceEnvironment] =

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/plugins/EventHistorySanityCheckPlugin.scala

@@ -1,20 +1,25 @@
 package org.lfdecentralizedtrust.splice.integration.plugins
 
 import cats.data.Chain
+import com.digitalasset.canton.ScalaFuturesWithPatience
+import com.digitalasset.canton.integration.EnvironmentSetupPlugin
+import com.digitalasset.canton.logging.NamedLoggerFactory
+import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.DsoRules_AddSv
+import org.lfdecentralizedtrust.splice.config.SpliceConfig
 import org.lfdecentralizedtrust.splice.console.ScanAppBackendReference
 import org.lfdecentralizedtrust.splice.environment.SpliceEnvironment
 import org.lfdecentralizedtrust.splice.http.v0.definitions.DamlValueEncoding.members.CompactJson
-import org.lfdecentralizedtrust.splice.http.v0.definitions.EventHistoryItem
+import org.lfdecentralizedtrust.splice.http.v0.definitions.{
+  EventHistoryItem,
+  TreeEvent,
+  UpdateHistoryItemV2,
+}
 import org.lfdecentralizedtrust.splice.http.v0.definitions.UpdateHistoryItemV2.members
 import org.lfdecentralizedtrust.splice.http.v0.definitions.UpdateHistoryReassignment.Event.members as reassignmentMembers
 import org.lfdecentralizedtrust.splice.integration.tests.SpliceTests.SpliceTestConsoleEnvironment
-import com.digitalasset.canton.ScalaFuturesWithPatience
-import com.digitalasset.canton.integration.EnvironmentSetupPlugin
-import com.digitalasset.canton.logging.NamedLoggerFactory
-import org.lfdecentralizedtrust.splice.config.SpliceConfig
-import org.scalatest.{Inspectors, LoneElement}
 import org.scalatest.concurrent.Eventually
 import org.scalatest.matchers.should.Matchers
+import org.scalatest.{Inspectors, LoneElement}
 
 import scala.annotation.tailrec
 
@@ -60,28 +65,74 @@ class EventHistorySanityCheckPlugin(
   ): Unit = {
     val (founders, others) = scans.partition(_.config.isFirstSv)
     val founder = founders.loneElement
-    val founderHistory = paginateEventHistory(founder, None, Chain.empty).toVector
+    val completeFounderHistory = paginateEventHistory(founder, None, Chain.empty).toVector
     forAll(others) { otherScan =>
-      val otherScanHistory = paginateEventHistory(otherScan, None, Chain.empty).toVector
-      val minSize = Math.min(founderHistory.size, otherScanHistory.size)
-      val otherComparable = otherScanHistory.take(minSize)
-      val founderComparable = founderHistory.take(minSize)
+      // we have to exclude all history before the otherScan was onboarded because:
+      // - they will be missing verdicts for everything that happened before they joined (we don't do backfilling yet)
+      // - they will have extra verdicts that do not involve the DSO party as part of onboarding, so the founder doesn't have them
+      val founderHistorySinceOtherOnboarding = completeFounderHistory
+        .dropWhile(item =>
+          !item.update.exists {
+            case UpdateHistoryItemV2.members.UpdateHistoryReassignment(_) => false
+            case UpdateHistoryItemV2.members.UpdateHistoryTransactionV2(tx) =>
+              tx.eventsById.exists(_._2 match {
+                case TreeEvent.members.ExercisedEvent(exercised)
+                    if exercised.choice == "DsoRules_AddSv" =>
+                  DsoRules_AddSv
+                    .fromJson(exercised.choiceArgument.noSpaces)
+                    .newSvParty == otherScan
+                    .getDsoInfo()
+                    .svPartyId
+                case _ => false
+              })
+          }
+        )
+
+      val otherStart = founderHistorySinceOtherOnboarding.headOption
+        .flatMap(_.update)
+        .getOrElse(
+          throw new IllegalStateException(
+            s"SV ${otherScan.config.svUser} doesn't appear in founder history"
+          )
+        ) match {
+        case members.UpdateHistoryReassignment(_) =>
+          throw new IllegalStateException("This is most certainly not a Reassignment")
+        case members.UpdateHistoryTransactionV2(value) =>
+          (value.migrationId, value.recordTime)
+      }
+
+      val otherScanHistory = paginateEventHistory(otherScan, Some(otherStart), Chain.empty).toVector
+        // the mediator takes some time after onboarding until it starts producing verdicts
+        .dropWhile(_.verdict.isEmpty)
+
+      val founderHistoryToUse = founderHistorySinceOtherOnboarding.dropWhile(item =>
+        item.verdict != otherScanHistory.headOption.flatMap(_.verdict)
+      )
+      val minSize = Math.min(founderHistoryToUse.size, otherScanHistory.size)
+      val (otherComparable, otherRestDebug) = otherScanHistory.splitAt(minSize)
+      val (founderComparable, founderRestDebug) = founderHistoryToUse.splitAt(minSize)
       val different = otherComparable
         .zip(founderComparable)
         .collect {
-          case (otherItem, founderItem) if founderItem != otherItem =>
+          case (otherItem, founderItem)
+              if makeComparable(founderItem) != makeComparable(otherItem) =>
             otherItem -> founderItem
         }
 
-      different should be(empty)
+      // custom error message to help debugging
+      if (different.nonEmpty) {
+        val debug: Seq[(Option[EventHistoryItem], Option[EventHistoryItem])] =
+          otherRestDebug.map(Some(_)).zipAll(founderRestDebug.map(Some(_)), None, None)
+        fail(s"Mismatched Events: $different. The ones that come after are: $debug")
+      }
     }
   }
 
   private def eventCursor(item: EventHistoryItem): Option[(Long, String)] = {
     val updateCursor = item.update.flatMap {
-      case members.UpdateHistoryTransactionV2(tx) =>
+      case UpdateHistoryItemV2.members.UpdateHistoryTransactionV2(tx) =>
         Some((tx.migrationId, tx.recordTime))
-      case members.UpdateHistoryReassignment(reassignment) =>
+      case UpdateHistoryItemV2.members.UpdateHistoryReassignment(reassignment) =>
         reassignment.event match {
           case reassignmentMembers.UpdateHistoryAssignment(event) =>
             Some((event.migrationId, reassignment.recordTime))
@@ -91,4 +142,10 @@ class EventHistorySanityCheckPlugin(
     }
     updateCursor.orElse(item.verdict.map(v => (v.migrationId, v.recordTime)))
   }
+
+  private def makeComparable(item: EventHistoryItem): EventHistoryItem = {
+    item.copy(verdict =
+      item.verdict.map(_.copy(finalizationTime = "this is not equal across scans/mediators"))
+    )
+  }
 }

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/DisabledWalletTimeBasedIntegrationTest.scala

@@ -74,6 +74,9 @@ class DisabledWalletTimeBasedIntegrationTest
           )
         ) - smallAmount
 
+      // advance rounds for the reward triggers to run
+      advanceRoundsToNextRoundOpening
+
       eventually(30.seconds) {
         sv1Backend.participantClient.ledger_api_extensions.acs
           .filterJava(SvRewardCoupon.COMPANION)(
@@ -84,7 +87,7 @@ class DisabledWalletTimeBasedIntegrationTest
             },
           ) should not be empty
 
-        advanceRoundsByOneTick
+        advanceRoundsToNextRoundOpening
         currentRound += 1
 
         silentClue(

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/DockerComposeFullNetworkFrontendIntegrationTest.scala

@@ -5,6 +5,10 @@ import org.lfdecentralizedtrust.splice.util.{FrontendLoginUtil, WalletFrontendTe
 
 import scala.concurrent.duration.*
 import scala.sys.process.*
+import java.io.{BufferedReader, InputStreamReader}
+import scala.util.Try
+import java.lang.ProcessBuilder
+import scala.jdk.CollectionConverters.*
 
 class DockerComposeFullNetworkFrontendIntegrationTest
     extends FrontendIntegrationTest("frontend")
@@ -91,10 +95,61 @@ class DockerComposeFullNetworkFrontendIntegrationTest
 
           }
         }
+        clue("full network (SV) stack should bind to localhost by default") {
+          withComposeFullNetwork() { () =>
+            var binding = getPortBinding("splice-sv-nginx-1", "80")
+            binding should startWith("127.0.0.1")
+            binding = getPortBinding("splice-validator-nginx-1", "80")
+            binding should startWith("127.0.0.1")
+          }
+        }
+      }
+    } finally {
+      Seq("build-tools/splice-compose.sh", "stop_network", "-D", "-f").!
+    }
+  }
+
+  def getPortBinding(containerName: String, internalPort: String): String = {
+    val command = Seq("docker", "port", containerName, internalPort)
+    val process = new ProcessBuilder(command.asJava).start()
+    val reader = new BufferedReader(new InputStreamReader(process.getInputStream))
+    val output = Try(reader.readLine()).toOption.getOrElse("")
+    if (process.waitFor() != 0 || output.isEmpty) {
+      fail(
+        s"Could not get port binding for $containerName:$internalPort. Is the container running?"
+      )
+    }
+    output
+  }
+
+  def withComposeFullNetwork[A](startFlags: Seq[String] = Seq.empty)(
+      test: () => A
+  ): A = {
+    try {
+      val command =
+        (Seq("build-tools/splice-compose.sh", "start_network", "-w") ++ startFlags).asJava
+
+      val builder = new ProcessBuilder(command)
+      if (builder.! != 0) {
+        fail("Failed to start docker-compose full network")
       }
+      test()
     } finally {
       Seq("build-tools/splice-compose.sh", "stop_network", "-D", "-f").!
     }
   }
 
+  "docker-compose networking bindings work" in { implicit env =>
+    registerHttpConnectionPoolsCleanup(env)
+
+    clue("full network (SV) stack should bind to 0.0.0.0 with -E flag") {
+      withComposeFullNetwork(startFlags = Seq("-E")) { () =>
+        var binding = getPortBinding("splice-sv-nginx-1", "80")
+        binding should startWith("0.0.0.0")
+        binding = getPortBinding("splice-validator-nginx-1", "80")
+        binding should startWith("0.0.0.0")
+      }
+    }
+  }
+
 }

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanFrontendTimeBasedIntegrationTest.scala

@@ -78,14 +78,14 @@ class ScanFrontendTimeBasedIntegrationTest
         s"Feature alice's validator and transfer some Amulet, to generate reward coupons"
       )({
         p2pTransfer(aliceWalletClient, bobWalletClient, bobUserParty, 40.0)
-        advanceRoundsByOneTick
-        advanceRoundsByOneTick
-        advanceRoundsByOneTick
+        advanceRoundsToNextRoundOpening
+        advanceRoundsToNextRoundOpening
+        advanceRoundsToNextRoundOpening
         p2pTransfer(aliceValidatorWalletClient, bobWalletClient, bobUserParty, 10.0)
       })
 
       clue("Advance rounds to collect rewards") {
-        Range(0, 6).foreach(_ => advanceRoundsByOneTick)
+        Range(0, 6).foreach(_ => advanceRoundsToNextRoundOpening)
       }
 
       val aliceValidatorWalletParty = aliceValidatorWalletClient.userStatus().party
@@ -145,7 +145,7 @@ class ScanFrontendTimeBasedIntegrationTest
         (1 to 5).foreach { i =>
           p2pTransfer(bobWalletClient, aliceWalletClient, aliceUserParty, i)
         }
-        advanceRoundsByOneTick
+        advanceRoundsToNextRoundOpening
       }
 
       withFrontEnd("scan-ui") { implicit webDriver =>
@@ -377,7 +377,7 @@ class ScanFrontendTimeBasedIntegrationTest
             trafficAmount,
             env.environment.clock.now,
           )
-          advanceRoundsByOneTick
+          advanceRoundsToNextRoundOpening
           buyMemberTraffic(
             aliceValidatorBackend,
             trafficAmount,
@@ -388,7 +388,7 @@ class ScanFrontendTimeBasedIntegrationTest
             trafficAmount,
             env.environment.clock.now,
           )
-          (1 to 5).foreach(_ => advanceRoundsByOneTick)
+          (1 to 5).foreach(_ => advanceRoundsToNextRoundOpening)
         },
       )(
         "Wait for round to close in scan",
@@ -430,12 +430,12 @@ class ScanFrontendTimeBasedIntegrationTest
         .round
         .number
 
-      advanceRoundsByOneTick
+      advanceRoundsToNextRoundOpening
       aliceWalletClient.tap(100.0)
 
       actAndCheck(
         "Advance rounds",
-        (1 to 5).foreach(_ => advanceRoundsByOneTick),
+        (1 to 5).foreach(_ => advanceRoundsToNextRoundOpening),
       )(
         "Wait for round to close in scan",
         _ => sv1ScanBackend.getRoundOfLatestData()._1 shouldBe (firstRound + 1),
@@ -497,8 +497,8 @@ class ScanFrontendTimeBasedIntegrationTest
         }
       }
 
-      openRounds.foreach(_ => advanceRoundsByOneTick)
-      advanceRoundsByOneTick
+      openRounds.foreach(_ => advanceRoundsToNextRoundOpening)
+      advanceRoundsToNextRoundOpening
       eventually() {
         aliceValidatorWalletClient.listValidatorLivenessActivityRecords() should have length 0
       }

apps/app/src/test/scala/org/lfdecentralizedtrust/splice/integration/tests/ScanIntegrationTest.scala

@@ -8,6 +8,7 @@ import com.digitalasset.canton.topology.PartyId
 import org.apache.pekko.http.scaladsl.Http
 import org.apache.pekko.http.scaladsl.client.RequestBuilding.{Get, Post}
 import org.apache.pekko.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes}
+import org.apache.pekko.http.scaladsl.model.headers.RawHeader
 import org.apache.pekko.http.scaladsl.unmarshalling.Unmarshal
 import org.lfdecentralizedtrust.splice.codegen.java.splice.amuletrules.AmuletRules
 import org.lfdecentralizedtrust.splice.codegen.java.splice.dso.svstate.SvNodeState
@@ -790,6 +791,27 @@ class ScanIntegrationTest extends IntegrationTest with WalletTestUtil with TimeT
     )
   }
 
+  "accept invalid user-agent headers" in { implicit env =>
+    import env.actorSystem
+    registerHttpConnectionPoolsCleanup(env)
+
+    val invalidUserAgentHeader = RawHeader("User-Agent", "OpenAPI-Generator/0.0.1/java")
+    // using `User-Agent` fails the following check, it cleans away the /java
+    // so we have to use RawHeader to simulate the actual client case
+    invalidUserAgentHeader.value shouldBe "OpenAPI-Generator/0.0.1/java"
+
+    // SuppressingLogger does not catch the warning (from pekko-http)
+    // if present, it's seen in checkErrors instead
+    val response = Http()
+      .singleRequest(
+        Get(
+          s"${sv1ScanBackend.httpClientConfig.url}/api/scan/v0/splice-instance-names"
+        ).withHeaders(invalidUserAgentHeader)
+      )
+      .futureValue
+    response.status shouldBe StatusCodes.OK
+  }
+
   def triggerTopupAliceAndBob()(implicit env: SpliceTestConsoleEnvironment): (Boolean, Boolean) = {
     val aliceTopupTrigger =
       aliceValidatorBackend.appState.automation.trigger[TopupMemberTrafficTrigger]