Add lblk cluster mode (Linux block devices via SPDK AIO bdevs) + campaign fixes

simplyblock_cli/cli-reference.yaml

@@ -106,8 +106,31 @@ commands:
             required: false
             type: str
             default: ""
+          - name: "--lblk"
+            help: "Configure the node with Linux block devices (lblk cluster mode) instead of NVMe PCIe devices: eligible unmounted, unheld, unpartitioned whole disks are wrapped in SPDK AIO bdevs. Select devices with --blk-names, --blk-names-exclude or --blk-serials; without a selector, every eligible disk is used."
+            dest: lblk
+            type: bool
+            action: store_true
+          - name: "--blk-names"
+            help: "Comma separated list of block device names to use, like sdb,sdc (requires --lblk). Requested devices must be eligible; a busy device is an error."
+            dest: blk_names
+            required: false
+            type: str
+            default: ""
+          - name: "--blk-names-exclude"
+            help: "Comma separated list of block device names to exclude, like sda (requires --lblk). All other eligible disks are used."
+            dest: blk_names_exclude
+            required: false
+            type: str
+            default: ""
+          - name: "--blk-serials"
+            help: "Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk)."
+            dest: blk_serials
+            required: false
+            type: str
+            default: ""
           - name: "--force"
-            help: "Force format detected or passed nvme pci address to 4K and clean partitions."
+            help: "Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format."
             dest: force
             type: bool
             action: store_true
@@ -182,6 +205,12 @@ commands:
             type: bool
             default: false
             action: store_true
+          - name: "--force-format"
+            help: "lblk cluster mode only: wipe partition tables and filesystem signatures from configured block devices that carry partitions (wipefs). Without this flag, partitioned devices are not eligible."
+            dest: force_format
+            type: bool
+            default: false
+            action: store_true
           - name: "--format-4k"
             help: "Force format nvme devices with 4K."
             dest: format_4k
@@ -1016,6 +1045,14 @@ commands:
             dest: enable_failure_domain
             type: bool
             action: store_true
+          - name: "--device-mode"
+            help: "Storage-device mode for the whole cluster. 'nvme' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. 'lblk': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at 'sn configure' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected."
+            dest: device_mode
+            type: str
+            choices:
+              - nvme
+              - lblk
+            default: nvme
           - name: "--name"
             help: >
               Assigns a name to the newly created cluster.
@@ -1198,6 +1235,14 @@ commands:
             dest: enable_failure_domain
             type: bool
             action: store_true
+          - name: "--device-mode"
+            help: "Storage-device mode for the whole cluster. 'nvme' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. 'lblk': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at 'sn configure' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected."
+            dest: device_mode
+            type: str
+            choices:
+              - nvme
+              - lblk
+            default: nvme
           - name: "--name"
             help: >
               Assigns a name to the newly created cluster.

simplyblock_cli/cli.py

@@ -109,7 +109,11 @@ def init_storage_node__configure(self, subparser):
         subcommand.add_argument('--device-model', help='NVMe SSD model string, example: --model PM1628. Can be used alone to filter by model, or combined with --size-range to further filter by size.', type=str, default='', dest='device_model', required=False)
         subcommand.add_argument('--size-range', help='NVMe SSD device size range separated by -, can be X(m,g,t) or bytes as integer, example: --size-range 50G-1T or --size-range 1232345-67823987. Can be used alone to filter by size, or combined with --device-model to further filter by model.', type=str, default='', dest='size_range', required=False)
         subcommand.add_argument('--nvme-names', help='Comma separated list of nvme namespace names like nvme0n1,nvme1n1.', type=str, default='', dest='nvme_names', required=False)
-        subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions.', dest='force', action='store_true')
+        subcommand.add_argument('--lblk', help='Configure the node with Linux block devices (lblk cluster mode) instead of NVMe PCIe devices: eligible unmounted, unheld, unpartitioned whole disks are wrapped in SPDK AIO bdevs. Select devices with --blk-names, --blk-names-exclude or --blk-serials; without a selector, every eligible disk is used.', dest='lblk', action='store_true')
+        subcommand.add_argument('--blk-names', help='Comma separated list of block device names to use, like sdb,sdc (requires --lblk). Requested devices must be eligible; a busy device is an error.', type=str, default='', dest='blk_names', required=False)
+        subcommand.add_argument('--blk-names-exclude', help='Comma separated list of block device names to exclude, like sda (requires --lblk). All other eligible disks are used.', type=str, default='', dest='blk_names_exclude', required=False)
+        subcommand.add_argument('--blk-serials', help='Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk).', type=str, default='', dest='blk_serials', required=False)
+        subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format.', dest='force', action='store_true')
         subcommand.add_argument('--calculate-hp-only', help='Calculate the minimum required huge pages, it depends on the following params: --cores-percentage, --sockets-to-use, --max-subsys, --nodes-per-socket, --number-of-devices.', dest='calculate_hp_only', action='store_true')
         subcommand.add_argument('--number-of-devices', help='Number of devices that will be used on this host. For calculating huge pages memory only.', type=int, dest='number_of_devices')
 
@@ -131,6 +135,7 @@ def init_storage_node__add_node(self, subparser):
         subcommand.add_argument('ifname', help='The management interface name.', type=str)
         subcommand.add_argument('--journal-partition', help='**Deprecated since: 26.1** Replaced by: --enable-journal-device\n\n1: Auto-create small partitions for journal on nvme devices. 0: use a separate (the smallest) nvme device of the node for journal. The journal needs a maximum of 3 percent of total available raw disk space. Default: `1`.', type=int, dest='partitions', choices=[0,1,])
         subcommand.add_argument('--enable-journal-device', help='Enables the use of a separate (the smallest) NVMe device of the node for the journal. Otherwise, the journal uses a maximum of 3%% of total available raw disk space across all NVMe devices.', default=False, dest='enable_journal_device', action='store_true')
+        subcommand.add_argument('--force-format', help='lblk cluster mode only: wipe partition tables and filesystem signatures from configured block devices that carry partitions (wipefs). Without this flag, partitioned devices are not eligible.', default=False, dest='force_format', action='store_true')
         subcommand.add_argument('--format-4k', help='Force format nvme devices with 4K.', dest='format_4k', action='store_true')
         if self.developer_mode:
             subcommand.add_argument('--jm-percent', help='Number in percent to use for JM from each device. Default: `3`.', type=int, default=3, dest='jm_percent')
@@ -439,6 +444,7 @@ def init_cluster__create(self, subparser):
             subcommand.add_argument('--disable-monitoring', help='Disable monitoring stack, false by default. Default: `false`.', dest='disable_monitoring', action='store_true')
         subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster.', dest='strict_node_anti_affinity', action='store_true')
         subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true')
+        subcommand.add_argument('--device-mode', help='Storage-device mode for the whole cluster. \'nvme\' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. \'lblk\': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at \'sn configure\' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected.', type=str, default='nvme', dest='device_mode', choices=['nvme','lblk',])
         subcommand.add_argument('--name', '-n', help='Assigns a name to the newly created cluster.', type=str, dest='name')
         subcommand.add_argument('--qpair-count', help='The NVMe/TCP transport qpair count per logical volume. Default: `32`.', type=range_type(0, 128), default=32, dest='qpair_count')
         subcommand.add_argument('--client-qpair-count', help='The default NVMe/TCP transport qpair count per logical volume for client. Default: `3`.', type=range_type(0, 128), default=3, dest='client_qpair_count')
@@ -475,6 +481,7 @@ def init_cluster__add(self, subparser):
             subcommand.add_argument('--inflight-io-threshold', help='The number of inflight IOs allowed before the IO queuing starts. Default: `4`.', type=int, default=4, dest='inflight_io_threshold')
         subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster."', dest='strict_node_anti_affinity', action='store_true')
         subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true')
+        subcommand.add_argument('--device-mode', help='Storage-device mode for the whole cluster. \'nvme\' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. \'lblk\': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at \'sn configure\' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected.', type=str, default='nvme', dest='device_mode', choices=['nvme','lblk',])
         subcommand.add_argument('--name', '-n', help='Assigns a name to the newly created cluster.', type=str, dest='name')
         subcommand.add_argument('--client-data-nic', help='Network interface name from client to use for logical volume connection.', type=str, dest='client_data_nic')
         subcommand.add_argument('--use-backup', help='The path to JSON file with S3/MinIO backup configuration.', type=str, dest='use_backup')

simplyblock_cli/clibase.py

@@ -128,15 +128,34 @@ def storage_node__configure(self, sub_command, args):
             pci_blocked = [str(x) for x in args.pci_blocked.split(',')]
         if args.nvme_names:
             nvme_names = [str(x) for x in args.nvme_names.split(',')]
+        lblk = getattr(args, 'lblk', False)
+        blk_names = getattr(args, 'blk_names', None)
+        blk_names_exclude = getattr(args, 'blk_names_exclude', None)
+        blk_serials = getattr(args, 'blk_serials', None)
         use_pci_allowed = bool(args.pci_allowed)
         use_pci_blocked = bool(args.pci_blocked)
         use_model_range = bool(args.device_model or args.size_range)
-        if sum([use_pci_allowed, use_pci_blocked, use_model_range]) > 1:
+        use_lblk = bool(lblk or blk_names or blk_names_exclude or blk_serials)
+        if sum([use_pci_allowed, use_pci_blocked, use_model_range, use_lblk]) > 1:
             self.parser.error(
-                "Choose only one device selection method: --pci-allowed, --pci-blocked, or "
+                "Choose only one device selection method: --pci-allowed, --pci-blocked, "
                 "--device-model/--size-range (--device-model and --size-range may be combined "
-                "with each other, but not with --pci-allowed or --pci-blocked)."
+                "with each other, but not with --pci-allowed or --pci-blocked), or --lblk with "
+                "its --blk-* selectors."
             )
+        lblk_selection = None
+        if use_lblk:
+            if not lblk:
+                self.parser.error("--blk-names/--blk-names-exclude/--blk-serials require --lblk")
+            if sum([bool(blk_names), bool(blk_names_exclude), bool(blk_serials)]) > 1:
+                self.parser.error(
+                    "Choose only one block-device selection method: --blk-names, "
+                    "--blk-names-exclude, or --blk-serials.")
+            lblk_selection = {
+                "names": [str(x) for x in blk_names.split(',')] if blk_names else None,
+                "names_exclude": [str(x) for x in blk_names_exclude.split(',')] if blk_names_exclude else None,
+                "serials": [str(x) for x in blk_serials.split(',')] if blk_serials else None,
+            }
         cores_percentage = int(args.cores_percentage)
         if args.calculate_hp_only:
             if not args.number_of_devices:
@@ -148,7 +167,8 @@ def storage_node__configure(self, sub_command, args):
             args.max_lvol, max_prov, sockets_to_use,args.nodes_per_socket,
             pci_allowed, pci_blocked, force=args.force, device_model=args.device_model,
             size_range=args.size_range, cores_percentage=cores_percentage, nvme_names=nvme_names,
-            calculate_hp_only=args.calculate_hp_only, number_of_devices=number_of_devices)
+            calculate_hp_only=args.calculate_hp_only, number_of_devices=number_of_devices,
+            lblk_selection=lblk_selection)
 
     def storage_node__deploy_cleaner(self, sub_command, args):
         storage_ops.deploy_cleaner()
@@ -218,6 +238,7 @@ def storage_node__add_node(self, sub_command, args):
                 spdk_sys_mem=spdk_sys_mem,
                 expansion=expansion,
                 failure_domain=failure_domain,
+                force_format=getattr(args, 'force_format', False),
             )
         except Exception as e:
             print(e)
@@ -1190,6 +1211,7 @@ def cluster_add(self, args):
         is_single_node = args.is_single_node
         client_data_nic = args.client_data_nic
         enable_failure_domain = getattr(args, 'enable_failure_domain', False)
+        device_mode = getattr(args, 'device_mode', 'nvme')
 
         max_fault_tolerance = min(distr_npcs, 2) if distr_npcs >= 1 else 1
 
@@ -1206,6 +1228,7 @@ def cluster_add(self, args):
             nvmf_base_port=args.nvmf_base_port, rpc_base_port=args.rpc_base_port, snode_api_port=args.snode_api_port,
             hashicorp_vault_settings=HashicorpVaultSettings({"base_url": args.hashicorp_vault_url}) if args.hashicorp_vault_url else None,
             enable_failure_domain=enable_failure_domain,
+            device_mode=device_mode,
         )
 
     def cluster_create(self, args):
@@ -1243,6 +1266,7 @@ def cluster_create(self, args):
         fabric = args.fabric
         client_data_nic = args.client_data_nic
         enable_failure_domain = getattr(args, 'enable_failure_domain', False)
+        device_mode = getattr(args, 'device_mode', 'nvme')
         # Private (developer-mode-only) arg: absent unless sbctl was run with --dev.
         enable_hang_device = getattr(args, "enable_hang_device", False)
 
@@ -1265,6 +1289,7 @@ def cluster_create(self, args):
             nvmf_base_port=args.nvmf_base_port, rpc_base_port=args.rpc_base_port, snode_api_port=args.snode_api_port,
             hashicorp_vault_settings=HashicorpVaultSettings({"base_url": args.hashicorp_vault_url}) if args.hashicorp_vault_url else None,
             enable_failure_domain=enable_failure_domain,
+            device_mode=device_mode,
             enable_hang_device=enable_hang_device,
         )