Merge canton 3.4 into main

Merge canton 3.4 into main

DACH-NY/canton-network-internal#2301

Author: martinflorian-da

.github/workflows/build.yml

@@ -168,6 +168,23 @@ jobs:
       parallelism: 10
       test_name: wall-clock-time
       with_gcp_creds: true
+      skip_if_regex: "\\[bft\\]"
+      commit_sha: ${{ inputs.commit_sha }}
+      daml_base_version: ${{ inputs.daml_base_version }}
+      oss_only: ${{ inputs.oss_only }}
+    secrets: inherit
+
+  scala_test_wall_clock_time_bft:
+    uses: ./.github/workflows/build.scala_test.yml
+    with:
+      runs_on: self-hosted-k8s-x-large
+      test_names_file: 'test-full-class-names.log'
+      start_canton_options: -we
+      parallelism: 10
+      test_name: wall-clock-time-bft
+      with_gcp_creds: true
+      pre_sbt_cmd: "export SPLICE_USE_BFT_SEQUENCER=1"
+      run_if_commit_tag: 'bft'
       commit_sha: ${{ inputs.commit_sha }}
       daml_base_version: ${{ inputs.daml_base_version }}
       oss_only: ${{ inputs.oss_only }}

.gitignore

@@ -76,3 +76,4 @@ docs/src/deployment/observability/metrics_reference.rst
 # For Vagrant
 /.vagrant/
 /vagrant-nix-cache/
+lnav*work

MAINTENANCE.md

@@ -47,7 +47,7 @@ Initial setup:
 1. Check out the [Canton **Open Source** repo](https://github.com/digital-asset/canton)
 2. Define the environment variable used in the commands below using `export PATH_TO_CANTON_OSS=<your-canton-oss-repo-path>`. This can be added to your private env vars.
 
-Current Canton commit: `3ee44a6d0f252aa8c52a86ab368044265d025c2f`
+Current Canton commit: `79e645eb60ba378536a6d62cabbeab78d1be6c61`
 
 1. Checkout the **current Canton commit listed above** in the Canton open source repo from above, so we can diff our current fork against this checkout.
 2. Change to your checkout of the Splice repo and execute the following steps:

VERSION

@@ -1 +1 @@
-0.4.22
+0.5.0

apps/app/src/main/scala/org/lfdecentralizedtrust/splice/config/ConfigTransforms.scala

@@ -375,42 +375,29 @@ object ConfigTransforms {
         .seq
     )
 
-  def updateAllSplitwellAppConfigs_(
-      update: SplitwellAppTransform
+  def updateAllSplitwellAppConfigs(
+      update: (String, SplitwellAppBackendConfig) => SplitwellAppBackendConfig
   ): ConfigTransform =
     _.focus(_.splitwellApps).modify(_.map { case (name, config) =>
-      (name, update(config))
+      (name, update(name.unwrap, config))
     })
 
-  def updateAllRemoteSplitwellAppConfigs_(
-      update: RemoteSplitwellAppTransform
+  def updateAllSplitwellAppConfigs_(
+      update: SplitwellAppTransform
+  ): ConfigTransform =
+    updateAllSplitwellAppConfigs((_, config) => update(config))
+
+  def updateAllRemoteSplitwellAppConfigs(
+      update: (String, SplitwellAppClientConfig) => SplitwellAppClientConfig
   ): ConfigTransform =
     _.focus(_.splitwellAppClients).modify(_.map { case (name, config) =>
-      (name, update(config))
+      (name, update(name.unwrap, config))
     })
 
-  def bumpCantonDomainPortsBy(bump: Int): ConfigTransform =
-    bumpSvAppCantonDomainPortsBy(bump) compose bumpValidatorAppCantonDomainPortsBy(
-      bump
-    ) compose bumpScanCantonDomainPortsBy(bump)
-
-  def bumpSvAppCantonDomainPortsBy(bump: Int): ConfigTransform = {
-    updateAllSvAppConfigs_(
-      _.focus(_.domains.global.url)
-        .modify(_.map(bumpUrl(bump, _)))
-        .focus(_.localSynchronizerNode)
-        .modify(
-          _.map(d =>
-            d.copy(
-              sequencer = d.sequencer
-                .copy(
-                  externalPublicApiUrl = bumpUrl(bump, d.sequencer.externalPublicApiUrl)
-                )
-            )
-          )
-        )
-    )
-  }
+  def updateAllRemoteSplitwellAppConfigs_(
+      update: RemoteSplitwellAppTransform
+  ): ConfigTransform =
+    updateAllRemoteSplitwellAppConfigs((_, config) => update(config))
 
   def bumpScanCantonDomainPortsBy(bump: Int) = {
     updateAllScanAppConfigs_(
@@ -424,47 +411,57 @@ object ConfigTransforms {
     )
   }
 
-  def bumpValidatorAppCantonDomainPortsBy(bump: Int): ConfigTransform = {
-    def bumpUrl(s: String): String = {
-      val uri = Uri(s)
-      uri.withPort(uri.effectivePort + bump).toString
-    }
-    def bumpOptionalUrl(o: Option[String]): Option[String] = {
-      o.map(bumpUrl(_))
-    }
-    updateAllValidatorConfigs_(
-      _.focus(_.domains.global.url)
-        .modify(bumpOptionalUrl(_))
-        .focus(_.domains.extra)
-        .modify(_.map(d => d.copy(url = bumpUrl(d.url))))
-    )
+  def bumpOptionalUrl(o: Option[String], bump: Int): Option[String] = {
+    o.map(bumpUrl(bump, _))
   }
 
-  def bumpCantonPortsBy(bump: Int): ConfigTransform = {
+  def bumpCantonPortsBy(bump: Int, predicate: String => Boolean = _ => true): ConfigTransform = {
 
     val transforms = Seq(
-      updateAllSvAppConfigs_(
-        _.focus(_.participantClient)
-          .modify(portTransform(bump, _))
-          .focus(_.localSynchronizerNode)
-          .modify(_.map(portTransform(bump, _)))
+      updateAllSvAppConfigs((name, conf) =>
+        if (predicate(name))
+          conf
+            .focus(_.participantClient)
+            .modify(portTransform(bump, _))
+            .focus(_.domains.global.url)
+            .modify(_.map(bumpUrl(bump, _)))
+            .focus(_.localSynchronizerNode)
+            .modify(_.map(portTransform(bump, _)))
+        else conf
       ),
-      updateAllScanAppConfigs_(
-        _.focus(_.participantClient)
-          .modify(portTransform(bump, _))
-          .focus(_.sequencerAdminClient)
-          .modify(portTransform(bump, _))
-          .focus(_.mediatorAdminClient)
-          .modify(portTransform(bump, _))
-          .focus(_.bftSequencers)
-          .modify(_.map(_.focus(_.sequencerAdminClient).modify(portTransform(bump, _))))
+      updateAllScanAppConfigs((name, conf) =>
+        if (predicate(name))
+          conf
+            .focus(_.participantClient)
+            .modify(portTransform(bump, _))
+            .focus(_.sequencerAdminClient)
+            .modify(portTransform(bump, _))
+            .focus(_.mediatorAdminClient)
+            .modify(portTransform(bump, _))
+            .focus(_.bftSequencers)
+            .modify(_.map(_.focus(_.sequencerAdminClient).modify(portTransform(bump, _))))
+        else conf
       ),
-      updateAllValidatorConfigs_(
-        _.focus(_.participantClient)
-          .modify(portTransform(bump, _))
+      updateAllValidatorConfigs((name, conf) =>
+        if (predicate(name))
+          conf
+            .focus(_.participantClient)
+            .modify(portTransform(bump, _))
+            .focus(_.domains.global.url)
+            .modify(bumpOptionalUrl(_, bump))
+            .focus(_.domains.extra)
+            .modify(_.map(d => d.copy(url = bumpUrl(bump, d.url))))
+        else conf
+      ),
+      updateAllSplitwellAppConfigs((name, conf) =>
+        if (predicate(name))
+          conf.focus(_.participantClient).modify(portTransform(bump, _))
+        else conf
       ),
-      updateAllSplitwellAppConfigs_(
-        _.focus(_.participantClient).modify(portTransform(bump, _))
+      updateAllRemoteSplitwellAppConfigs((name, conf) =>
+        if (predicate(name))
+          conf.focus(_.participantClient).modify(portTransform(bump, _))
+        else conf
       ),
     )
 
@@ -476,34 +473,27 @@ object ConfigTransforms {
     updateAllSvAppConfigs((name, config) => {
       if (svApps.contains(name)) {
         config
-          .focus(_.participantClient)
-          .modify(portTransform(bump, _))
-          .focus(_.localSynchronizerNode)
-          .modify(_.map(portTransform(bump, _)))
           .focus(_.adminApi)
           .modify(portTransform(bump, _))
       } else {
         config
       }
-    })
+    }) compose bumpSomeSvAppCantonPortsBy(bump, svApps)
   }
 
-  def bumpSomeSvAppCantonDomainPortsBy(bump: Int, svApps: Seq[String]): ConfigTransform = {
+  def bumpSomeSvAppCantonPortsBy(bump: Int, svApps: Seq[String]): ConfigTransform = {
     updateAllSvAppConfigs((name, config) => {
       if (svApps.contains(name)) {
         config
           .focus(_.domains.global.url)
           .modify(_.map(bumpUrl(bump, _)))
+          .focus(_.participantClient)
+          .modify(portTransform(bump, _))
           .focus(_.localSynchronizerNode)
-          .modify(
-            _.map(d =>
-              d.copy(
-                sequencer = d.sequencer
-                  .copy(externalPublicApiUrl = bumpUrl(bump, d.sequencer.externalPublicApiUrl))
-              )
-            )
-          )
-      } else config
+          .modify(_.map(portTransform(bump, _)))
+      } else {
+        config
+      }
     })
   }
 
@@ -562,10 +552,20 @@ object ConfigTransforms {
     updateAllScanAppConfigs((name, config) => {
       if (scanApps.contains(name)) {
         config
-          .focus(_.participantClient)
-          .modify(portTransform(bump, _))
           .focus(_.adminApi)
           .modify(portTransform(bump, _))
+      } else {
+        config
+      }
+    }) compose bumpSomeScanAppCantonPortsBy(bump, scanApps)
+  }
+
+  def bumpSomeScanAppCantonPortsBy(bump: Int, scanApps: Seq[String]): ConfigTransform = {
+    updateAllScanAppConfigs((name, config) => {
+      if (scanApps.contains(name)) {
+        config
+          .focus(_.participantClient)
+          .modify(portTransform(bump, _))
           .focus(_.sequencerAdminClient)
           .modify(portTransform(bump, _))
           .focus(_.mediatorAdminClient)
@@ -602,13 +602,23 @@ object ConfigTransforms {
     updateAllValidatorAppConfigs((name, config) => {
       if (validatorApps.contains(name)) {
         config
-          .focus(_.participantClient)
-          .modify(portTransform(bump, _))
           .focus(_.adminApi)
           .modify(portTransform(bump, _))
       } else {
         config
       }
+    }) compose bumpSomeValidatorAppCantonPortsBy(bump, validatorApps)
+  }
+
+  def bumpSomeValidatorAppCantonPortsBy(bump: Int, validatorApps: Seq[String]): ConfigTransform = {
+    updateAllValidatorAppConfigs((name, config) => {
+      if (validatorApps.contains(name)) {
+        config
+          .focus(_.participantClient)
+          .modify(portTransform(bump, _))
+      } else {
+        config
+      }
     })
   }
 
@@ -672,31 +682,38 @@ object ConfigTransforms {
     transforms.foldLeft((c: SpliceConfig) => c)((f, tf) => f compose tf)
   }
 
-  def withBftSequencers(): ConfigTransform = {
-    updateAllSvAppConfigs_(appConfig =>
-      appConfig
-        .focus(_.localSynchronizerNode)
-        .modify(
-          _.map(
-            _.focus(_.sequencer).modify(
-              _.copy(
-                isBftSequencer = true
-              )
+  def withBftSequencer(config: SvAppBackendConfig): SvAppBackendConfig =
+    config
+      .focus(_.localSynchronizerNode)
+      .modify(
+        _.map(
+          _.focus(_.sequencer).modify(
+            _.copy(
+              isBftSequencer = true
             )
           )
         )
-    ) compose {
-      updateAllScanAppConfigs((scan, config) =>
-        config.copy(
-          bftSequencers = Seq(
-            BftSequencerConfig(
-              0,
-              config.sequencerAdminClient,
-              s"http://localhost:${5010 + Integer.parseInt(scan.stripPrefix("sv").take(1)) * 100}",
-            )
-          )
+      )
+
+  def withBftSequencer(
+      name: String,
+      config: ScanAppBackendConfig,
+      migrationId: Long = 0,
+      basePort: Int = 5010,
+  ): ScanAppBackendConfig =
+    config.copy(
+      bftSequencers = Seq(
+        BftSequencerConfig(
+          migrationId,
+          config.sequencerAdminClient,
+          s"http://localhost:${basePort + Integer.parseInt(name.stripPrefix("sv").take(1)) * 100}",
         )
       )
+    )
+
+  def withBftSequencers(): ConfigTransform = {
+    updateAllSvAppConfigs_(withBftSequencer(_)) compose {
+      updateAllScanAppConfigs((scan, config) => withBftSequencer(scan, config))
     }
   }
 
@@ -744,6 +761,8 @@ object ConfigTransforms {
       .modify(portTransform(bump, _))
       .focus(_.internalApi)
       .modify(portTransform(bump, _))
+      .focus(_.externalPublicApiUrl)
+      .modify(bumpUrl(bump, _))
 
   private def portTransform(bump: Int, c: SvMediatorConfig): SvMediatorConfig =
     c.focus(_.adminApi).modify(portTransform(bump, _))
@@ -816,16 +835,15 @@ object ConfigTransforms {
     })
 
   def useDecentralizedSynchronizerSplitwell(): ConfigTransform =
-    updateAllSplitwellAppConfigs_(c => {
-      c.copy(
-        domains = c.domains.copy(
-          splitwell = SplitwellDomains(
+    updateAllSplitwellAppConfigs_(
+      _.focus(_.domains.splitwell)
+        .replace(
+          SplitwellDomains(
             SynchronizerConfig(SynchronizerAlias.tryCreate("global")),
             Seq.empty,
           )
         )
-      )
-    })
+    ) compose updateAllValidatorAppConfigs_(_.focus(_.domains.extra).replace(Seq.empty))
 
   def modifyAllANStorageConfigs(
       storageConfigModifier: (