DAOS-11169 control: decouple storage format from engine start

ci/rpm/test_daos_node.sh

@@ -216,6 +216,7 @@ while [[ $line != *started\ on\ rank\ 0* ]] && [ "$SECONDS" -lt "$deadline" ]; d
     stdout+=("$line")
     if [[ $line == SCM\ format\ required\ on\ instance\ * ]]; then
         dmg storage format -l "$HOSTNAME" --force
+        dmg system initialize -l "$HOSTNAME" --timeout=2m
         if ! dmg system query -v; then
             sleep 5
             dmg system query -v || true

docs/QSG/setup_rhel.md

@@ -416,6 +416,9 @@ Examples are available on [github](https://github.com/daos-stack/daos/tree/maste
 		# format storage
 		dmg storage format -l $SERVER_NODES # can use --force if needed
 
+		# start the engines and wait for them to join
+		dmg system initialize -l $SERVER_NODES
+
 3. Verify that all servers have started:
 
 		# system query from ADMIN_NODES

docs/QSG/setup_suse.md

@@ -417,6 +417,9 @@ Examples are available on [github](https://github.com/daos-stack/daos/tree/maste
 		# format storage
 		dmg storage format -l $SERVER_NODES # can use --force if needed
 
+		# start the engines and wait for them to join
+		dmg system initialize -l $SERVER_NODES
+
 3. Verify that all servers have started:
 
 		# system query from ADMIN_NODES

docs/QSG/tour.md

@@ -232,6 +232,9 @@ bring-up DAOS servers and clients.
 	  -----             ----------- ------------
 	  boro-[8,35,52-53] 1           0
 
+	$ dmg system initialize
+	4 engines joined on 4 hosts
+
 	$ dmg pool create --size 10G Pool2
 	$ daos cont create --type POSIX --oclass RP_3G1 --properties rd_fac:2 Pool2 Cont2
 	$ daos pool list-cont Pool2

docs/admin/administration.md

@@ -972,6 +972,16 @@ The output table will indicate action and result.
 While the engines are stopped, the DAOS servers will continue to
 operate and listen on the management network.
 
+A stopped rank leaves the system group at once, so its pools exclude it
+under their self-heal policy just as they would a failed rank. To stop a
+rank without a pool exclusion, disable self-heal exclusion on its pools
+first.
+
+The same applies to a system that is stopped and then only partly started:
+the ranks still stopped are excluded from every pool as soon as the pool
+services come up. A staged restart must disable self-heal exclusion first or
+start every rank.
+
 !!! warning
     All engines monitor each other and pro-actively exclude unresponsive
     members. It is critical to properly stop a DAOS system as with dmg in
@@ -1017,6 +1027,47 @@ specified on the command line:
 If the ranks were excluded from pools (e.g., unclean shutdown), they will need to
 be reintegrated. Please see the pool operation section for more information.
 
+### Engine Startup
+
+A `daos_server` only starts an I/O engine when the system expects it to be
+running. After the first `dmg storage format`, engines wait for
+`dmg system initialize` (or `dmg system extend` for hosts added later). When
+`daos_server` restarts, an engine whose rank was stopped with `dmg system stop`
+stays stopped until `dmg system start`; any other rank that was a member is
+restarted.
+
+- Initialize a System:
+
+```bash
+$ dmg system initialize -l ${storage_nodes} [--timeout=<duration>]
+```
+
+Starts the engines on the listed hosts (or the hosts in the dmg configuration
+file) and waits for them to join. With `--timeout`, hosts whose engines had
+not joined when it expired are named in the output and the command may be
+rerun once they are fixed; without it, the command waits until interrupted.
+
+- Extend a System:
+
+```bash
+$ dmg system extend -l ${new_storage_nodes} [--timeout=<duration>]
+```
+
+The same operation for hosts added to a system that already has pools.
+
+The `restart_engines_on_boot` system property controls whether `daos_server`
+restarts joined ranks on boot. Set it to `false` for sites where a reboot is a
+maintenance step and engines should stay down until `dmg system start`:
+
+```bash
+$ dmg system set-prop restart_engines_on_boot:false
+```
+
+`dmg storage format --start` formats and starts the engines in one step, as
+earlier releases did; on hosts that are already formatted it starts the engines
+still waiting. `dmg storage format --replace` also starts the engine it
+formats, so the rank it replaces rejoins without a further command.
+
 ### Engine Auto-Restart
 
 DAOS automatically restarts engines that self-terminate after being excluded from
@@ -1449,7 +1500,8 @@ stopped by running `dmg system stop` (if necessary with the `--force` flag).
 The erase operation will destroy any pools that may still exist, and will
 unconfigure the storage. It will not stop the daos\_server process, so
 the `dmg` command can still be used. For example, the system can be
-formatted again by running `dmg storage format`.
+formatted again by running `dmg storage format`, followed by
+`dmg system initialize` to start the engines.
 
 !!! note
     Note that `dmg system erase` does not currently reset the SCM.
@@ -1544,19 +1596,20 @@ To add a new server to an existing DAOS system, one should install:
 
 The daos\_control.yml file should also be updated to include the new DAOS server.
 
-Then start the daos\_server via systemd and format the new server via
-dmg as follows:
+Then start the daos\_server via systemd, format the new server and start its
+engines via dmg as follows:
 
 ```
 $ dmg storage format -l ${new_storage_node}
+$ dmg system extend -l ${new_storage_node}
 ```
 
 `new_storage_node` should be replaced with the hostname or the IP address of the
 new storage node (comma separated list or range of hosts for multiple nodes)
 to be added.
 
-Upon completion of the format operation, the new storage nodes will join
-the system (this can be checked with `dmg system query -v`).
+`dmg system extend` returns once the new storage nodes have joined the system
+(this can be checked with `dmg system query -v`).
 
 !!! note
     New pools created after the extension will automatically use the newly added

docs/admin/common_tasks.md

@@ -21,7 +21,8 @@ for more detailed explanations about each step.
 8. Use `dmg config generate` to generate the config file that contains PMEM and NVMe.
 9. Define the certificate files in the server config.
 10. Start server with the generated config file.
-11. Check that the server is waiting for SCM format. Call `dmg storage format`.
+11. Check that the server is waiting for SCM format. Call `dmg storage format`, then
+`dmg system initialize`.
 12. Create a small pool; E.g., ~500MB.
 13. Define the certificate files in the agent config.
 14. Start agent.
@@ -49,8 +50,8 @@ E.g., `mgmt_svc_replicas`, `class`.
 NVMe.
 9. Distribute the config file to `/etc/daos` of all hosts.
 10. Start server on all the hosts.
-11. Check that the server is waiting for SCM format. Call `dmg storage format` against all
-server hosts.
+11. Check that the server is waiting for SCM format. Call `dmg storage format` and then
+`dmg system initialize` against all server hosts.
 12. Check that the servers are running on all the hosts with `dmg system query
 --verbose`.
 13. Create a small pool; E.g., ~500MB.

docs/admin/deployment.md

@@ -12,8 +12,9 @@ After `daos_server` instances have been started on each storage node for the
 first time, `daos_server scm prepare` will set PMem storage into the necessary
 state for use with DAOS when run on each host.
 Then `dmg storage format` formats persistent storage devices (specified in the
-server configuration file) on the storage nodes and writes necessary metadata
-before starting DAOS I/O engine processes that will operate across the fabric.
+server configuration file) on the storage nodes and writes necessary metadata,
+and `dmg system initialize` starts the DAOS I/O engine processes that will
+operate across the fabric.
 
 To sum up, the typical workflow of a DAOS system deployment consists of the
 following steps:
@@ -25,6 +26,8 @@ following steps:
 
 - [Format](#storage-formatting) the DAOS system
 
+- [Initialize](#system-initialization) the DAOS system
+
 - [Set up and start the agent](#agent-setup) on the client nodes
 
 - [Validate](#system-validation) that the DAOS system is operational
@@ -551,27 +554,26 @@ SCM format required on instance 1
 The `daos_server start --auto-format` option has now been added to allow automatic formatting on
 startup in the event that externally triggered format is not desirable. When using this option
 the storage format will occur on `daos_server` startup without any user intervention assuming the
-supplied config file is valid.
+supplied config file is valid. The engines still wait for `dmg system initialize`.
 
 To manually format the storage and start the engine processes, we run the following on a separate
-terminal window and verify that engine processes (ranks) have registered with the system.
-Note the subsequent system query command may not show ranks started immediately after the storage
-format command returns so it is recommended to leave a short delay (~5s) before invoking.
+terminal window. `dmg system initialize` returns once the engine processes (ranks) have registered
+with the system or, with `--timeout`, reports the hosts whose engines did not.
 
 ```bash
 $ dmg storage format -i
 Format Summary:
   Hosts     SCM Devices NVMe Devices
   -----     ----------- ------------
   localhost 2           16
+$ dmg system initialize -i
+2 engines joined on 1 host
 $ dmg system query -i
 Rank  State
 ----  -----
 [0-1] Joined
 ```
 
-Engine I/O processes will be launched by the `daos_server` service following storage format.
-
 ```bash
 Instance 0: starting format of nvme block devices 0000:2b:00.0,0000:2c:00.0,0000:2d:00.0,0000:2e:00.0,0000:63:00.0,0000:64:00.0,0000:65:00.0,0000:66:00.0
 Instance 0: finished format of nvme block devices 0000:2b:00.0,0000:2c:00.0,0000:2d:00.0,0000:2e:00.0,0000:63:00.0,0000:64:00.0,0000:65:00.0,0000:66:00.0
@@ -1584,11 +1586,8 @@ the host for use with DAOS using the parameters defined in the server config fil
 node specifying a hostlist (`-l <host>[,...]`) of storage nodes with SCM/PMem
 modules and NVMe SSDs installed and prepared.
 
-Upon successful format, DAOS Control Servers will start DAOS I/O engines that
-have been specified in the server config file.
-
-Successful start-up is indicated by the following on stdout:
-`DAOS I/O Engine (v2.0.1) process 433456 started on rank 1 with 8 target, 2 helper XS, firstcore 0, host wolf-72.domain.`
+Upon successful format, the DAOS Control Servers hold the DAOS I/O engines
+specified in the server config file until the system is initialized.
 
 ### SCM Format
 
@@ -1632,9 +1631,23 @@ After the `storage format` command is run, the path specified by the server
 configuration file `scm_mount` parameter should be mounted and should contain
 the necessary DAOS metadata indicating that the server has been formatted.
 
-When starting, `daos_server` will skip `maintenance mode` and attempt to start
-I/O engines if valid DAOS metadata is found in `scm_mount`.
+When starting, `daos_server` will skip `maintenance mode` if valid DAOS metadata
+is found in `scm_mount`, and will start the I/O engines that the system expects
+to be running (see [Engine Startup](administration.md#engine-startup)).
+
+## System Initialization
+
+`dmg -l <host>[,...] system initialize` starts the I/O engines on the formatted
+storage nodes and waits for them to join the system. It is run once, after the
+first format, and may be rerun if some hosts did not join. To add storage nodes
+to a system that already has pools, format them and run
+`dmg -l <host>[,...] system extend` instead.
+
+Successful start-up is indicated by the following on stdout:
+`DAOS I/O Engine (v2.0.1) process 433456 started on rank 1 with 8 target, 2 helper XS, firstcore 0, host wolf-72.domain.`
 
+To format and start the engines in one step, as earlier releases did, run
+`dmg storage format --start`; the engines join without `dmg system initialize`.
 
 ## Agent Setup
 

docs/admin/self_healing.md

@@ -208,7 +208,7 @@ Self-heal policy for the system (self_heal) none
 **3.** Add ranks to system by starting storage server DAOS services and
 formatting so that they join.
 
-Use `daos_server start` and `dmg storage format` commands.
+Use `daos_server start`, `dmg storage format` and `dmg system initialize` commands.
 
 **4.** Run system query which will indicate disabled system `self_heal` flags.
 

docs/admin/starting-daos.md

@@ -15,6 +15,10 @@ All commands except the `daos` command should be run as root.
 
 - Start the `daos_server` daemon on all server nodes,
   for example by running `systemctl start daos_server` in a parallel shell.
+  Each server restarts the engines whose ranks were running when the system
+  was last up. Ranks stopped with `dmg system stop` stay stopped until
+  `dmg system start`, and if the `restart_engines_on_boot` system property is
+  `false`, all engines wait for `dmg system start`.
 
 - Wait for the DAOS control plane and DAOS engines to start on all servers.
   Depending on the number and capacity of the NVMe disks in the servers,
@@ -57,7 +61,6 @@ All commands except the `daos` command should be run as root.
   N0203 INFO 2026/08/20 14:04:10 daos_engine:1 DAOS I/O Engine (v2.8.0) process 254652 started on rank 1 with 20 target, 10 helper XS, firstcore 0, host N0203.
   N0203 INFO 2026/08/20 14:04:11 daos_engine:0 DAOS I/O Engine (v2.8.0) process 254706 started on rank 0 with 20 target, 10 helper XS, firstcore 0, host N0203.
   ...
-  DEBUG 2026/08/20 14:04:11.912940 server.go:413: engines have started
 ```
 
 - Verify that all engines have started successfully,

docs/dev/development.md

@@ -392,8 +392,13 @@ This can be triggered in a different shell, using the following command:
 $ docker exec server dmg -i storage format
 ```
 
-Upon successful completion of the format, the storage engine is started, and pools
-can be created using the daos admin tool (see next section).
+Once the storage is formatted, start the engine with:
+
+```bash
+$ docker exec server dmg -i system initialize
+```
+
+Pools can then be created using the daos admin tool (see next section).
 
 For more advanced configurations involving SCM, SSD or a real fabric, please
 refer to the next section.

src/control/cmd/dmg/command_test.go

@@ -149,6 +149,10 @@ func (bci *bridgeConnInvoker) InvokeUnaryRPC(ctx context.Context, uReq control.U
 		resp = control.MockMSResponse("", nil, &mgmtpb.SystemStartResp{})
 	case *control.SystemEraseReq:
 		resp = control.MockMSResponse("", nil, &mgmtpb.SystemEraseResp{})
+	case *control.SystemInitializeReq:
+		resp = control.MockMSResponse("", nil, &mgmtpb.SystemInitializeResp{})
+	case *control.SystemExtendReq:
+		resp = control.MockMSResponse("", nil, &mgmtpb.SystemExtendResp{})
 	case *control.SystemExcludeReq:
 		resp = control.MockMSResponse("", nil, &mgmtpb.SystemExcludeResp{})
 	case *control.SystemDrainReq:

src/control/cmd/dmg/pretty/printers.go

@@ -1,5 +1,6 @@
 //
 // (C) Copyright 2020-2023 Intel Corporation.
+// (C) Copyright 2026 Hewlett Packard Enterprise Development LP
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -37,6 +38,8 @@ type (
 		ShowHostPorts bool
 		// LEDInfoOnly indicates that the output should only include LED related info.
 		LEDInfoOnly bool
+		// WholeSystem indicates that the response was not filtered by rank, host or state.
+		WholeSystem bool
 	}
 
 	// PrintConfigOption defines a config function.
@@ -56,6 +59,13 @@ func PrintWithVerboseOutput(verbose bool) PrintConfigOption {
 	}
 }
 
+// PrintWithWholeSystem indicates that the response covers the whole system.
+func PrintWithWholeSystem(whole bool) PrintConfigOption {
+	return func(cfg *PrintConfig) {
+		cfg.WholeSystem = whole
+	}
+}
+
 // PrintWithHostPorts enables display of host ports in output.
 func PrintWithHostPorts() PrintConfigOption {
 	return func(cfg *PrintConfig) {

src/control/cmd/dmg/pretty/system.go

@@ -131,10 +131,13 @@ func PrintSystemQueryResponse(out, outErr io.Writer, resp *control.SystemQueryRe
 		return errors.Errorf("nil %T", resp)
 	}
 
+	cfg := getPrintConfig(opts...)
 	switch {
+	case len(resp.Members) == 0 && cfg.WholeSystem:
+		fmt.Fprintln(out, "No ranks have joined the system. Formatted engines wait for dmg system initialize.")
 	case len(resp.Members) == 0:
 		fmt.Fprintln(out, "Query matches no ranks in system")
-	case getPrintConfig(opts...).Verbose:
+	case cfg.Verbose:
 		printSystemQueryVerbose(out, resp.Members)
 	default:
 		if err := printSystemQuery(out, resp.Members, &resp.AbsentRanks); err != nil {
@@ -218,6 +221,29 @@ func PrintSystemStartResponse(out, outErr io.Writer, resp *control.SystemStartRe
 	return printSystemResults(out, outErr, resp.Results, &resp.AbsentHosts, &resp.AbsentRanks)
 }
 
+// PrintSystemStartHostsResponse generates a human-readable representation of the
+// supplied SystemStartHostsResp struct and writes it to the supplied io.Writer.
+func PrintSystemStartHostsResponse(out, outErr io.Writer, resp *control.SystemStartHostsResp) {
+	results := resp.Results
+	hosts := make(map[string]struct{})
+	ranks := ranklist.NewRankSet()
+	for _, r := range results {
+		if r.Errored {
+			fmt.Fprintf(outErr, "%s: %s\n", r.Addr, r.Msg)
+			continue
+		}
+		ranks.Add(r.Rank)
+		hosts[r.Addr] = struct{}{}
+	}
+
+	if ranks.Count() == 0 {
+		fmt.Fprintln(out, "No engines joined")
+		return
+	}
+	fmt.Fprintf(out, "%s joined on %s\n", english.Plural(ranks.Count(), "engine", ""),
+		english.Plural(len(hosts), "host", ""))
+}
+
 // PrintSystemStopResponse generates a human-readable representation of the
 // supplied SystemStopResp struct and writes it to the supplied io.Writer.
 func PrintSystemStopResponse(out, outErr io.Writer, resp *control.SystemStopResp) error {