Automatically expire stale Pending node entries

CHANGELOG.md

@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 
 - C++ endpoints can now use `ccf::endpoints::Endpoint::add_openapi_response<Out>()` to document additional HTTP responses in their generated OpenAPI schema without changing the endpoint's primary success response (#8115).
+- Pending node entries are now automatically removed when they stop retrying joins for the configurable `pending_node_timeout` delay (1 hour by default). Set it to `0s` to disable automatic removal (#8173).
 - New `ledger.max_transaction_size` node configuration option (default `32MB`), which caps the total serialised size of transactions written to the ledger. The limit covers the whole ledger entry: the fixed 8-byte ledger entry header, the ledger encryption header, public domain size field, public domain and encrypted private domain. It is checked before a transaction is applied, so an oversized transaction is now rejected with `413 Payload Too Large` and error code `TransactionTooLarge`, and subsequent transactions are unaffected, where previously an excessively large transaction could terminate the node. Reserved internal signature transactions are exempt because they must fill their reserved ledger version. The limit applies only to newly serialised non-reserved transactions; deserialising existing entries (including during recovery), historical queries and snapshots are unaffected, so entries written under a larger or unset limit remain readable. It must be smaller than `memory.max_msg_size` by at least the ring-buffer range response overhead, which is validated at node startup and by `--check` (#7992).
 
 ### Changed

doc/host_config_schema/host_config.json

@@ -575,6 +575,11 @@
       "description": "This section includes configuration for periodic cleanup of old files (snapshots, ledger chunks)",
       "additionalProperties": false
     },
+    "pending_node_timeout": {
+      "type": "string",
+      "default": "1h",
+      "description": "Delay after which the primary automatically removes a Pending node that has stopped sending join requests. Cleanup runs at least once per minute, so removal may occur up to one minute after this delay. Set to 0s to disable automatic removal."
+    },
     "identity_history_fetch": {
       "type": "object",
       "properties": {

doc/operations/start_network.rst

@@ -62,6 +62,8 @@ If the network has not yet been opened by members (see :ref:`governance/open_net
 
 The ``Pending`` joining node automatically polls the service (interval configurable via ``join.retry_timeout`` configuration entry) until the members have successfully transitioned the node to the ``Trusted`` state. It is only then that the joining node transitions to the ``PartOfNetwork`` state and starts updating its ledger.
 
+The primary automatically removes a ``Pending`` node that has stopped sending join requests for longer than the ``pending_node_timeout`` configuration entry (1 hour by default). Cleanup runs at least once per minute. Set ``pending_node_timeout`` to ``0s`` to disable automatic removal.
+
 .. tip:: After the node has been trusted by members, operators should poll the :http:GET:`/node/state` endpoint on the newly added node, using the node's self-signed certificate as TLS CA, until the ``{"state": "PartOfNetwork"}`` is reported. This status confirms that the replication of the ledger has started on this node.
 
 .. note:: To accelerate the joining procedure, it is possible for new nodes to join from a snapshot. More information on snapshots :ref:`here <operations/ledger_snapshot:Join or Recover From Snapshot>`.

include/ccf/node/startup_config.h

@@ -29,6 +29,8 @@ namespace ccf
 
     ccf::ds::SizeString historical_cache_soft_limit = {"512MB"};
 
+    ccf::ds::TimeString pending_node_timeout = {"1h"};
+
     ccf::consensus::Configuration consensus = {};
     ccf::NodeInfoNetwork network;
 

include/ccf/service/node_info.h

@@ -72,6 +72,10 @@ namespace ccf
      * compatibility.
      */
     bool retired_committed = false;
+
+    /** Time when this Pending node last sent a join request, in milliseconds
+     * since the Unix epoch. Optional for backward compatibility. */
+    std::optional<int64_t> pending_last_seen = std::nullopt;
   };
   DECLARE_JSON_TYPE_WITH_BASE_AND_OPTIONAL_FIELDS(NodeInfo, NodeInfoNetwork);
   DECLARE_JSON_REQUIRED_FIELDS(
@@ -84,7 +88,8 @@ namespace ccf
     certificate_signing_request,
     public_key,
     node_data,
-    retired_committed);
+    retired_committed,
+    pending_last_seen);
 }
 
 FMT_BEGIN_NAMESPACE

src/common/configuration.h

@@ -145,6 +145,7 @@ namespace ccf
     attestation,
     snapshots,
     files_cleanup,
+    pending_node_timeout,
     node_to_node_message_limit,
     historical_cache_soft_limit,
     identity_history_fetch);

src/node/node_state.h

@@ -44,6 +44,7 @@
 #include "node/local_sealing.h"
 #include "node/node_inbound_message.h"
 #include "node/node_to_node_channel_manager.h"
+#include "node/pending_node_cleanup.h"
 #include "node/recovery_decision_protocol.h"
 #include "node/recovery_snapshot_ledger.h"
 #include "node/signature_cache_subsystem.h"
@@ -485,6 +486,7 @@ namespace ccf
     // JWT key auto-refresh
     //
     std::shared_ptr<JwtKeyAutoRefresh> jwt_key_auto_refresh;
+    std::shared_ptr<PendingNodeCleanup> pending_node_cleanup;
 
     std::unique_ptr<StartupSnapshotInfo> startup_snapshot_info = nullptr;
     // Set to the snapshot seqno when a node starts from one and remembered for
@@ -3599,6 +3601,12 @@ namespace ccf
         commit_callbacks,
         public_only);
 
+      pending_node_cleanup = std::make_shared<PendingNodeCleanup>(
+        node_client,
+        consensus,
+        std::chrono::milliseconds(config.pending_node_timeout));
+      pending_node_cleanup->start();
+
       network.tables->set_consensus(consensus);
       network.tables->set_snapshotter(snapshotter);
 

src/node/pending_node_cleanup.h

@@ -0,0 +1,92 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "kv/kv_types.h"
+#include "node/node_client.h"
+#include "tasks/basic_task.h"
+#include "tasks/task_system.h"
+
+#include <algorithm>
+#include <chrono>
+#include <memory>
+
+namespace ccf
+{
+  class PendingNodeCleanup
+    : public std::enable_shared_from_this<PendingNodeCleanup>
+  {
+  private:
+    static constexpr auto max_cleanup_interval = std::chrono::minutes(1);
+
+    std::shared_ptr<NodeClient> node_client;
+    std::shared_ptr<ccf::kv::Consensus> consensus;
+    std::chrono::milliseconds cleanup_interval;
+    ccf::tasks::Task periodic_cleanup_task;
+
+    void send_cleanup_request()
+    {
+      ::http::Request request(
+        fmt::format(
+          "/{}/{}",
+          ccf::get_actor_prefix(ccf::ActorsType::nodes),
+          "network/nodes/remove_expired_pending"),
+        HTTP_POST);
+      request.set_header(http::headers::CONTENT_LENGTH, "0");
+
+      node_client->make_request(request);
+    }
+
+  public:
+    PendingNodeCleanup(
+      std::shared_ptr<NodeClient> node_client_,
+      std::shared_ptr<ccf::kv::Consensus> consensus_,
+      std::chrono::milliseconds pending_node_timeout) :
+      node_client(std::move(node_client_)),
+      consensus(std::move(consensus_)),
+      cleanup_interval(std::min(
+        pending_node_timeout,
+        std::chrono::duration_cast<std::chrono::milliseconds>(
+          max_cleanup_interval)))
+    {}
+
+    ~PendingNodeCleanup()
+    {
+      stop();
+    }
+
+    void start()
+    {
+      if (cleanup_interval <= std::chrono::milliseconds::zero())
+      {
+        return;
+      }
+
+      const auto self = weak_from_this();
+      periodic_cleanup_task = ccf::tasks::make_basic_task([self]() {
+        const auto self_sp = self.lock();
+        if (self_sp == nullptr)
+        {
+          return;
+        }
+
+        if (self_sp->consensus->can_replicate())
+        {
+          self_sp->send_cleanup_request();
+        }
+      });
+
+      ccf::tasks::add_periodic_task(
+        periodic_cleanup_task, cleanup_interval, cleanup_interval);
+    }
+
+    void stop()
+    {
+      if (periodic_cleanup_task != nullptr)
+      {
+        periodic_cleanup_task->cancel_task();
+        periodic_cleanup_task = nullptr;
+      }
+    }
+  };
+}