Fix endorsed certificate data race

src/node/history.h

@@ -26,7 +26,9 @@
 #include "tasks/task_system.h"
 
 #include <array>
+#include <atomic>
 #include <deque>
+#include <memory>
 #include <string.h>
 
 #define HAVE_OPENSSL
@@ -313,7 +315,7 @@ namespace ccf
     NodeId id;
     ccf::crypto::ECKeyPair& node_kp;
     ccf::crypto::ECKeyPair_OpenSSL& service_kp;
-    ccf::crypto::Pem& endorsed_cert;
+    std::shared_ptr<const ccf::crypto::Pem> endorsed_cert;
     const ccf::COSESignaturesConfig& cose_signatures_config;
     const ccf::LedgerSignMode ledger_sign_mode;
     std::unordered_map<std::string, CoseKey>& cose_key_cache;
@@ -326,7 +328,7 @@ namespace ccf
       NodeId id_,
       ccf::crypto::ECKeyPair& node_kp_,
       ccf::crypto::ECKeyPair_OpenSSL& service_kp_,
-      ccf::crypto::Pem& endorsed_cert_,
+      std::shared_ptr<const ccf::crypto::Pem> endorsed_cert_,
       const ccf::COSESignaturesConfig& cose_signatures_config_,
       ccf::LedgerSignMode ledger_sign_mode_,
       std::unordered_map<std::string, CoseKey>& cose_key_cache_) :
@@ -336,7 +338,7 @@ namespace ccf
       id(std::move(id_)),
       node_kp(node_kp_),
       service_kp(service_kp_),
-      endorsed_cert(endorsed_cert_),
+      endorsed_cert(std::move(endorsed_cert_)),
       cose_signatures_config(cose_signatures_config_),
       ledger_sign_mode(ledger_sign_mode_),
       cose_key_cache(cose_key_cache_)
@@ -364,7 +366,7 @@ namespace ccf
           root,
           {}, // Nonce is currently empty
           primary_sig,
-          endorsed_cert);
+          *endorsed_cert);
 
         signatures->put(sig_value);
       }
@@ -577,7 +579,8 @@ namespace ccf
     ccf::kv::Term term_of_last_version = 0;
     ccf::kv::Term term_of_next_version{};
 
-    std::optional<ccf::crypto::Pem> endorsed_cert = std::nullopt;
+    std::atomic<std::shared_ptr<const ccf::crypto::Pem>> endorsed_cert =
+      nullptr;
 
     struct ServiceSigningIdentity
     {
@@ -943,7 +946,8 @@ namespace ccf
         return;
       }
 
-      if (!endorsed_cert.has_value())
+      auto endorsed_cert_ = endorsed_cert.load(std::memory_order_acquire);
+      if (endorsed_cert_ == nullptr)
       {
         throw std::logic_error(
           fmt::format("No endorsed certificate set to emit signature"));
@@ -968,7 +972,7 @@ namespace ccf
           id,
           node_kp,
           *signing_identity->service_kp,
-          endorsed_cert.value(),
+          std::move(endorsed_cert_),
           signing_identity->cose_signatures_config,
           signing_identity->ledger_sign_mode,
           cose_key_cache),
@@ -1022,7 +1026,9 @@ namespace ccf
 
     void set_endorsed_certificate(const ccf::crypto::Pem& cert) override
     {
-      endorsed_cert = cert;
+      endorsed_cert.store(
+        std::make_shared<const ccf::crypto::Pem>(cert),
+        std::memory_order_release);
     }
 
   private:

src/node/test/history.cpp

@@ -19,7 +19,9 @@
 #include <doctest/doctest.h>
 #undef FAIL
 
+#include <atomic>
 #include <exception>
+#include <stop_token>
 #include <thread>
 
 using MapT = ccf::kv::Map<size_t, size_t>;
@@ -314,6 +316,77 @@ class TestPendingTx : public ccf::kv::PendingTx
   }
 };
 
+TEST_CASE("Pending signatures retain their endorsed certificate")
+{
+  auto encryptor = std::make_shared<ccf::kv::NullTxEncryptor>();
+  auto consensus = std::make_shared<ccf::kv::test::PrimaryStubConsensus>();
+  auto node_kp = ccf::crypto::make_ec_key_pair();
+  auto service_kp = std::dynamic_pointer_cast<ccf::crypto::ECKeyPair_OpenSSL>(
+    ccf::crypto::make_ec_key_pair());
+
+  const auto first_cert =
+    node_kp->self_sign("CN=First Node", valid_from, valid_to);
+  const auto second_cert =
+    node_kp->self_sign("CN=Second Node", valid_from, valid_to);
+
+  ccf::kv::Store store;
+  store.set_encryptor(encryptor);
+  store.set_consensus(consensus);
+
+  auto history = std::make_shared<ccf::MerkleTxHistory>(
+    store, ccf::kv::test::PrimaryNodeId, *node_kp);
+  history->set_endorsed_certificate(first_cert);
+  history->set_service_signing_identity(
+    service_kp, ccf::COSESignaturesConfig{});
+  store.set_history(history);
+
+  constexpr auto store_term = 2;
+  store.initialise_term(store_term);
+
+  MapT table("public:table");
+  const auto gap_txid = store.next_txid();
+
+  history->emit_signature();
+  REQUIRE(consensus->number_of_replicas() == 0);
+
+  history->set_endorsed_certificate(second_cert);
+  REQUIRE(
+    store.commit(
+      gap_txid,
+      std::make_unique<TestPendingTx>(gap_txid, store, table),
+      false) == ccf::kv::CommitResult::SUCCESS);
+
+  auto tx = store.create_read_only_tx();
+  auto signatures = tx.ro<ccf::Signatures>(ccf::Tables::SIGNATURES);
+  const auto signature = signatures->get();
+  REQUIRE(signature.has_value());
+  REQUIRE(signature->cert == first_cert);
+
+  std::atomic<bool> updater_started = false;
+  std::jthread updater([&](std::stop_token stop_token) {
+    updater_started.store(true, std::memory_order_release);
+    while (!stop_token.stop_requested())
+    {
+      history->set_endorsed_certificate(first_cert);
+      history->set_endorsed_certificate(second_cert);
+      std::this_thread::yield();
+    }
+  });
+
+  while (!updater_started.load(std::memory_order_acquire))
+  {
+    std::this_thread::yield();
+  }
+
+  for (size_t i = 0; i < 32; ++i)
+  {
+    history->emit_signature();
+  }
+
+  updater.request_stop();
+  updater.join();
+}
+
 struct PausedSignatureCommit
 {
   ccf::ds::Mutex lock;