diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 26f0e71d69..6bf73369e5 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -106,8 +106,31 @@ commands: required: false type: str default: "" + - name: "--lblk" + help: "Configure the node with Linux block devices (lblk cluster mode) instead of NVMe PCIe devices: eligible unmounted, unheld, unpartitioned whole disks are wrapped in SPDK AIO bdevs. Select devices with --blk-names, --blk-names-exclude or --blk-serials; without a selector, every eligible disk is used." + dest: lblk + type: bool + action: store_true + - name: "--blk-names" + help: "Comma separated list of block device names to use, like sdb,sdc (requires --lblk). Requested devices must be eligible; a busy device is an error." + dest: blk_names + required: false + type: str + default: "" + - name: "--blk-names-exclude" + help: "Comma separated list of block device names to exclude, like sda (requires --lblk). All other eligible disks are used." + dest: blk_names_exclude + required: false + type: str + default: "" + - name: "--blk-serials" + help: "Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk)." + dest: blk_serials + required: false + type: str + default: "" - name: "--force" - help: "Force format detected or passed nvme pci address to 4K and clean partitions." + help: "Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format." dest: force type: bool action: store_true @@ -182,6 +205,12 @@ commands: type: bool default: false action: store_true + - name: "--force-format" + help: "lblk cluster mode only: wipe partition tables and filesystem signatures from configured block devices that carry partitions (wipefs). Without this flag, partitioned devices are not eligible." + dest: force_format + type: bool + default: false + action: store_true - name: "--format-4k" help: "Force format nvme devices with 4K." dest: format_4k @@ -1016,6 +1045,14 @@ commands: dest: enable_failure_domain type: bool action: store_true + - name: "--device-mode" + help: "Storage-device mode for the whole cluster. 'nvme' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. 'lblk': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at 'sn configure' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected." + dest: device_mode + type: str + choices: + - nvme + - lblk + default: nvme - name: "--name" help: > Assigns a name to the newly created cluster. @@ -1198,6 +1235,14 @@ commands: dest: enable_failure_domain type: bool action: store_true + - name: "--device-mode" + help: "Storage-device mode for the whole cluster. 'nvme' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. 'lblk': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at 'sn configure' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected." + dest: device_mode + type: str + choices: + - nvme + - lblk + default: nvme - name: "--name" help: > Assigns a name to the newly created cluster. diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 709b031820..248a52c353 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -109,7 +109,11 @@ def init_storage_node__configure(self, subparser): subcommand.add_argument('--device-model', help='NVMe SSD model string, example: --model PM1628. Can be used alone to filter by model, or combined with --size-range to further filter by size.', type=str, default='', dest='device_model', required=False) subcommand.add_argument('--size-range', help='NVMe SSD device size range separated by -, can be X(m,g,t) or bytes as integer, example: --size-range 50G-1T or --size-range 1232345-67823987. Can be used alone to filter by size, or combined with --device-model to further filter by model.', type=str, default='', dest='size_range', required=False) subcommand.add_argument('--nvme-names', help='Comma separated list of nvme namespace names like nvme0n1,nvme1n1.', type=str, default='', dest='nvme_names', required=False) - subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions.', dest='force', action='store_true') + subcommand.add_argument('--lblk', help='Configure the node with Linux block devices (lblk cluster mode) instead of NVMe PCIe devices: eligible unmounted, unheld, unpartitioned whole disks are wrapped in SPDK AIO bdevs. Select devices with --blk-names, --blk-names-exclude or --blk-serials; without a selector, every eligible disk is used.', dest='lblk', action='store_true') + subcommand.add_argument('--blk-names', help='Comma separated list of block device names to use, like sdb,sdc (requires --lblk). Requested devices must be eligible; a busy device is an error.', type=str, default='', dest='blk_names', required=False) + subcommand.add_argument('--blk-names-exclude', help='Comma separated list of block device names to exclude, like sda (requires --lblk). All other eligible disks are used.', type=str, default='', dest='blk_names_exclude', required=False) + subcommand.add_argument('--blk-serials', help='Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk).', type=str, default='', dest='blk_serials', required=False) + subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format.', dest='force', action='store_true') subcommand.add_argument('--calculate-hp-only', help='Calculate the minimum required huge pages, it depends on the following params: --cores-percentage, --sockets-to-use, --max-subsys, --nodes-per-socket, --number-of-devices.', dest='calculate_hp_only', action='store_true') subcommand.add_argument('--number-of-devices', help='Number of devices that will be used on this host. For calculating huge pages memory only.', type=int, dest='number_of_devices') @@ -131,6 +135,7 @@ def init_storage_node__add_node(self, subparser): subcommand.add_argument('ifname', help='The management interface name.', type=str) subcommand.add_argument('--journal-partition', help='**Deprecated since: 26.1** Replaced by: --enable-journal-device\n\n1: Auto-create small partitions for journal on nvme devices. 0: use a separate (the smallest) nvme device of the node for journal. The journal needs a maximum of 3 percent of total available raw disk space. Default: `1`.', type=int, dest='partitions', choices=[0,1,]) subcommand.add_argument('--enable-journal-device', help='Enables the use of a separate (the smallest) NVMe device of the node for the journal. Otherwise, the journal uses a maximum of 3%% of total available raw disk space across all NVMe devices.', default=False, dest='enable_journal_device', action='store_true') + subcommand.add_argument('--force-format', help='lblk cluster mode only: wipe partition tables and filesystem signatures from configured block devices that carry partitions (wipefs). Without this flag, partitioned devices are not eligible.', default=False, dest='force_format', action='store_true') subcommand.add_argument('--format-4k', help='Force format nvme devices with 4K.', dest='format_4k', action='store_true') if self.developer_mode: subcommand.add_argument('--jm-percent', help='Number in percent to use for JM from each device. Default: `3`.', type=int, default=3, dest='jm_percent') @@ -439,6 +444,7 @@ def init_cluster__create(self, subparser): subcommand.add_argument('--disable-monitoring', help='Disable monitoring stack, false by default. Default: `false`.', dest='disable_monitoring', action='store_true') subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster.', dest='strict_node_anti_affinity', action='store_true') subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') + subcommand.add_argument('--device-mode', help='Storage-device mode for the whole cluster. \'nvme\' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. \'lblk\': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at \'sn configure\' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected.', type=str, default='nvme', dest='device_mode', choices=['nvme','lblk',]) subcommand.add_argument('--name', '-n', help='Assigns a name to the newly created cluster.', type=str, dest='name') subcommand.add_argument('--qpair-count', help='The NVMe/TCP transport qpair count per logical volume. Default: `32`.', type=range_type(0, 128), default=32, dest='qpair_count') subcommand.add_argument('--client-qpair-count', help='The default NVMe/TCP transport qpair count per logical volume for client. Default: `3`.', type=range_type(0, 128), default=3, dest='client_qpair_count') @@ -475,6 +481,7 @@ def init_cluster__add(self, subparser): subcommand.add_argument('--inflight-io-threshold', help='The number of inflight IOs allowed before the IO queuing starts. Default: `4`.', type=int, default=4, dest='inflight_io_threshold') subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster."', dest='strict_node_anti_affinity', action='store_true') subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') + subcommand.add_argument('--device-mode', help='Storage-device mode for the whole cluster. \'nvme\' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. \'lblk\': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at \'sn configure\' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected.', type=str, default='nvme', dest='device_mode', choices=['nvme','lblk',]) subcommand.add_argument('--name', '-n', help='Assigns a name to the newly created cluster.', type=str, dest='name') subcommand.add_argument('--client-data-nic', help='Network interface name from client to use for logical volume connection.', type=str, dest='client_data_nic') subcommand.add_argument('--use-backup', help='The path to JSON file with S3/MinIO backup configuration.', type=str, dest='use_backup') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index a8e837c66f..f68fa46d07 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -128,15 +128,34 @@ def storage_node__configure(self, sub_command, args): pci_blocked = [str(x) for x in args.pci_blocked.split(',')] if args.nvme_names: nvme_names = [str(x) for x in args.nvme_names.split(',')] + lblk = getattr(args, 'lblk', False) + blk_names = getattr(args, 'blk_names', None) + blk_names_exclude = getattr(args, 'blk_names_exclude', None) + blk_serials = getattr(args, 'blk_serials', None) use_pci_allowed = bool(args.pci_allowed) use_pci_blocked = bool(args.pci_blocked) use_model_range = bool(args.device_model or args.size_range) - if sum([use_pci_allowed, use_pci_blocked, use_model_range]) > 1: + use_lblk = bool(lblk or blk_names or blk_names_exclude or blk_serials) + if sum([use_pci_allowed, use_pci_blocked, use_model_range, use_lblk]) > 1: self.parser.error( - "Choose only one device selection method: --pci-allowed, --pci-blocked, or " + "Choose only one device selection method: --pci-allowed, --pci-blocked, " "--device-model/--size-range (--device-model and --size-range may be combined " - "with each other, but not with --pci-allowed or --pci-blocked)." + "with each other, but not with --pci-allowed or --pci-blocked), or --lblk with " + "its --blk-* selectors." ) + lblk_selection = None + if use_lblk: + if not lblk: + self.parser.error("--blk-names/--blk-names-exclude/--blk-serials require --lblk") + if sum([bool(blk_names), bool(blk_names_exclude), bool(blk_serials)]) > 1: + self.parser.error( + "Choose only one block-device selection method: --blk-names, " + "--blk-names-exclude, or --blk-serials.") + lblk_selection = { + "names": [str(x) for x in blk_names.split(',')] if blk_names else None, + "names_exclude": [str(x) for x in blk_names_exclude.split(',')] if blk_names_exclude else None, + "serials": [str(x) for x in blk_serials.split(',')] if blk_serials else None, + } cores_percentage = int(args.cores_percentage) if args.calculate_hp_only: if not args.number_of_devices: @@ -148,7 +167,8 @@ def storage_node__configure(self, sub_command, args): args.max_lvol, max_prov, sockets_to_use,args.nodes_per_socket, pci_allowed, pci_blocked, force=args.force, device_model=args.device_model, size_range=args.size_range, cores_percentage=cores_percentage, nvme_names=nvme_names, - calculate_hp_only=args.calculate_hp_only, number_of_devices=number_of_devices) + calculate_hp_only=args.calculate_hp_only, number_of_devices=number_of_devices, + lblk_selection=lblk_selection) def storage_node__deploy_cleaner(self, sub_command, args): storage_ops.deploy_cleaner() @@ -218,6 +238,7 @@ def storage_node__add_node(self, sub_command, args): spdk_sys_mem=spdk_sys_mem, expansion=expansion, failure_domain=failure_domain, + force_format=getattr(args, 'force_format', False), ) except Exception as e: print(e) @@ -1190,6 +1211,7 @@ def cluster_add(self, args): is_single_node = args.is_single_node client_data_nic = args.client_data_nic enable_failure_domain = getattr(args, 'enable_failure_domain', False) + device_mode = getattr(args, 'device_mode', 'nvme') max_fault_tolerance = min(distr_npcs, 2) if distr_npcs >= 1 else 1 @@ -1206,6 +1228,7 @@ def cluster_add(self, args): nvmf_base_port=args.nvmf_base_port, rpc_base_port=args.rpc_base_port, snode_api_port=args.snode_api_port, hashicorp_vault_settings=HashicorpVaultSettings({"base_url": args.hashicorp_vault_url}) if args.hashicorp_vault_url else None, enable_failure_domain=enable_failure_domain, + device_mode=device_mode, ) def cluster_create(self, args): @@ -1243,6 +1266,7 @@ def cluster_create(self, args): fabric = args.fabric client_data_nic = args.client_data_nic enable_failure_domain = getattr(args, 'enable_failure_domain', False) + device_mode = getattr(args, 'device_mode', 'nvme') # Private (developer-mode-only) arg: absent unless sbctl was run with --dev. enable_hang_device = getattr(args, "enable_hang_device", False) @@ -1265,6 +1289,7 @@ def cluster_create(self, args): nvmf_base_port=args.nvmf_base_port, rpc_base_port=args.rpc_base_port, snode_api_port=args.snode_api_port, hashicorp_vault_settings=HashicorpVaultSettings({"base_url": args.hashicorp_vault_url}) if args.hashicorp_vault_url else None, enable_failure_domain=enable_failure_domain, + device_mode=device_mode, enable_hang_device=enable_hang_device, ) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index ffffaac19c..74d5966031 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -256,6 +256,17 @@ def parse_protocols(input_str: str): "rdma": "rdma" in parts, } +def _validated_device_mode(device_mode) -> str: + """Normalize/validate the cluster device mode ("nvme" | "lblk"). + Deploy-time only, like enable_failure_domain.""" + mode = (device_mode or constants.DEVICE_MODE_NVME).lower() + if mode not in (constants.DEVICE_MODE_NVME, constants.DEVICE_MODE_LBLK): + raise ValueError( + f"invalid device_mode {device_mode!r}; must be " + f"'{constants.DEVICE_MODE_NVME}' or '{constants.DEVICE_MODE_LBLK}'") + return mode + + def create_cluster(blk_size, page_size_in_blocks, cli_pass, cap_warn, cap_crit, prov_cap_warn, prov_cap_crit, ifname, mgmt_ip, log_del_interval, metrics_retention_period, contact_point, grafana_endpoint, distr_ndcs, distr_npcs, distr_bs, distr_chunk_bs, ha_type, mode, @@ -265,6 +276,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, nvmf_base_port=4420, rpc_base_port=8080, snode_api_port=50001, container_image_prefix=None, hashicorp_vault_settings : t.Optional[HashicorpVaultSettings] = None, enable_failure_domain=False, + device_mode=constants.DEVICE_MODE_NVME, enable_hang_device=False, ) -> str: @@ -399,6 +411,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.inflight_io_threshold = inflight_io_threshold cluster.strict_node_anti_affinity = strict_node_anti_affinity cluster.enable_failure_domain = enable_failure_domain + cluster.device_mode = _validated_device_mode(device_mode) cluster.contact_point = contact_point cluster.disable_monitoring = disable_monitoring cluster.mode = mode @@ -504,6 +517,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn nvmf_base_port=4420, rpc_base_port=8080, snode_api_port=50001, hashicorp_vault_settings : t.Optional[HashicorpVaultSettings] = None, enable_failure_domain=False, + device_mode=constants.DEVICE_MODE_NVME, ) -> str: """Thin wrapper around _add_cluster_impl() that serializes create calls for the same name behind a ClusterCreateLock. @@ -529,6 +543,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn client_data_nic=client_data_nic, max_fault_tolerance=max_fault_tolerance, backup_config=backup_config, nvmf_base_port=nvmf_base_port, rpc_base_port=rpc_base_port, snode_api_port=snode_api_port, hashicorp_vault_settings=hashicorp_vault_settings, enable_failure_domain=enable_failure_domain, + device_mode=device_mode, ) if not name: return _add_cluster_impl(**kwargs) @@ -552,6 +567,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca nvmf_base_port=4420, rpc_base_port=8080, snode_api_port=50001, hashicorp_vault_settings : t.Optional[HashicorpVaultSettings] = None, enable_failure_domain=False, + device_mode=constants.DEVICE_MODE_NVME, ) -> str: clusters = db_controller.get_clusters() @@ -590,6 +606,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca cluster.secret = SecretStr(utils.generate_string(20)) cluster.strict_node_anti_affinity = strict_node_anti_affinity cluster.enable_failure_domain = enable_failure_domain + cluster.device_mode = _validated_device_mode(device_mode) if clusters: cfg = db_controller.get_deploy_config() @@ -1548,6 +1565,27 @@ def _set_node_ana(node_id) -> None: # want headroom for an unplanned failure concurrent with a rollout.) utils.set_storage_mcp_max_unavailable(cl_id, cluster.max_fault_tolerance) + # JM mesh gate (2026-08-05 incident: nodes joined via add-node retries + # activated with peers missing their remote_jm controllers — the cluster + # reported healthy while a third of the journal mesh was unreachable, + # and the first journal load collapsed n_safe_jms into a cluster-wide + # JCERR). FRESH activation must not complete over such a hole; a + # RE-ACTIVATION is a recovery path that may legitimately run with one + # or two nodes unhealthy, so it repairs best-effort and only warns — + # the verifier already skips JMs whose owner node is not ONLINE. + if cluster.ha_type == "ha": + jm_problems = storage_node_ops.verify_jm_mesh_coverage(cl_id, repair=True) + if jm_problems: + if is_fresh_activation: + set_cluster_status(cl_id, ols_status) + raise ValueError( + "Failed to activate cluster: JM mesh coverage incomplete " + "(journal quorum would silently run degraded): " + + "; ".join(jm_problems)) + logger.warning( + "JM mesh coverage incomplete on re-activation (continuing — " + "recovery path): %s", "; ".join(jm_problems)) + set_cluster_status(cl_id, Cluster.STATUS_ACTIVE) logger.info("Cluster activated successfully") diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 62b58e967d..aaf3582939 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -62,6 +62,31 @@ def get_config_var(name, default=None): CACHED_LVOL_STAT_COLLECTOR_INTERVAL_SEC = 15 DEV_DISCOVERY_INTERVAL_SEC = 60 +# --- lblk cluster mode (Linux block devices via SPDK AIO bdevs) --- +DEVICE_MODE_NVME = "nvme" +DEVICE_MODE_LBLK = "lblk" +# DPDK PCI allowlist placeholder used when starting SPDK in lblk mode: an +# empty allowlist means "allow all" to DPDK (the k8s launch path passes an +# empty PCI_ALLOWED today), which would let SPDK's nvme driver claim +# kernel-owned NVMe disks. 0000:00:00.0 is the host bridge — syntactically a +# valid BDF, never a storage device, no DPDK driver binds it. +LBLK_PCI_ALLOWED_PLACEHOLDER = "0000:00:00.0" +# Queue-depth sampling period enabled on every AIO base bdev so +# bdev_get_iostat reports queue_depth (feeds the hung-IO watchdog). +AIO_QD_SAMPLING_PERIOD_US = 100000 # 100 ms +# Hung-IO watchdog: consecutive device_monitor polls (DEV_MONITOR_INTERVAL_SEC +# apart) with queue_depth > 0 and zero completion progress before the device +# is declared stalled (3 x 10s = 30s — deliberately above kernel SCSI/NVMe +# timeouts, which convert most stalls into EIO for us via the distrib +# error_* events; the watchdog only catches what the kernel never times out). +AIO_HUNG_IO_STALL_POLLS = 3 +# Consecutive polls a configured block device may be absent from the host's +# lsblk before it is treated as hot-removed (REMOVAL semantics). +AIO_DEVICE_ABSENT_POLLS = 2 +# Kernel block devices never eligible for lblk data placement. +LBLK_EXCLUDED_NAME_PREFIXES = ("ram", "loop", "sr", "fd", "zram", "nbd", + "md", "dm-", "drbd") + PMEM_DIR = '/tmp/pmem' NVME_PROGRAM_FAIL_COUNT = 50 diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index fc3871dbb6..3d218a7165 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -4,7 +4,7 @@ import logging import uuid -from simplyblock_core import distr_controller, utils, storage_node_ops +from simplyblock_core import constants, distr_controller, utils, storage_node_ops from simplyblock_core.controllers import device_events, tasks_controller from simplyblock_core.db_controller import DBController from simplyblock_core.models.nvme_device import NVMeDevice, JMDevice @@ -563,7 +563,34 @@ def restart_device(device_id, force=False): except Exception as e: logger.error(f"Failed to log teardown-warning event for {device_id}: {e}") - if not snode.rpc_client().bdev_nvme_controller_list(device_obj.nvme_controller): + if device_obj.bdev_type == "aio": + # lblk mode: the base bdev is an AIO bdev over a kernel block device. + # Re-resolve serial-first (kernel names shift), recreate if gone. + if not snode.rpc_client().get_bdevs(device_obj.nvme_bdev): + try: + filename = device_obj.by_id_path or device_obj.device_path + try: + inventory, _ = snode.client(timeout=30, retry=1).get_blockdevices() + for blk in inventory or []: + if blk.get("serial") == device_obj.serial_number: + filename = blk.get("by_id_path") or blk.get("device_path") + device_obj.device_path = blk.get("device_path", device_obj.device_path) + device_obj.by_id_path = blk.get("by_id_path", device_obj.by_id_path) + break + except Exception as e: + logger.warning(f"blockdevices inventory failed, using stored path: {e}") + if not filename: + logger.error(f"No block device path known for {device_id}") + return False + snode.rpc_client().bdev_aio_create(device_obj.nvme_bdev, filename) + snode.rpc_client().bdev_examine(device_obj.nvme_bdev) + snode.rpc_client().bdev_wait_for_examine() + snode.rpc_client().bdev_set_qd_sampling_period( + device_obj.nvme_bdev, constants.AIO_QD_SAMPLING_PERIOD_US) + except Exception as e: + logger.error(e) + return False + elif not snode.rpc_client().bdev_nvme_controller_list(device_obj.nvme_controller): try: ret = snode.client(timeout=30, retry=1).bind_device_to_spdk(device_obj.pcie_address) logger.debug(ret) @@ -983,12 +1010,24 @@ def reset_storage_device(dev_id): logger.info("Resetting device") rpc_client = snode.rpc_client() - controller_name = device.nvme_controller - response = rpc_client.reset_device(controller_name) - if not response: - logger.error(f"Failed to reset NVMe BDev {controller_name}") - return False - time.sleep(3) + if device.bdev_type == "aio": + # No controller-reset primitive for AIO bdevs, and deleting/ + # recreating the bdev here would cascade a REMOVE through the + # alceml stack. Liveness-probe instead: bdev present => clear the + # error state below (device_set_online also forgives flaps); + # bdev gone => fail so the tasks framework escalates to + # restart_device, whose full stack rebuild is the real recovery. + if not rpc_client.get_bdevs(device.nvme_bdev): + logger.error(f"AIO bdev {device.nvme_bdev} is gone; reset cannot " + f"recover it — restart the device instead") + return False + else: + controller_name = device.nvme_controller + response = rpc_client.reset_device(controller_name) + if not response: + logger.error(f"Failed to reset NVMe BDev {controller_name}") + return False + time.sleep(3) # set io_error flag False device_set_io_error(dev_id, False) @@ -1333,18 +1372,47 @@ def new_device_from_failed(device_id): logger.error("Device is already added back from failed") return False - if not device_node.rpc_client().bdev_nvme_controller_list(device.nvme_controller): - try: - ret = device_node.client(timeout=30, retry=1).bind_device_to_spdk(device.pcie_address) - logger.debug(ret) - device_node.rpc_client().bdev_nvme_controller_attach(device.nvme_controller, device.pcie_address) - except Exception as e: - logger.error(e) + if device.bdev_type == "aio": + # lblk mode: ensure the AIO bdev exists again (serial-first + # re-resolution against the live host; stored path as fallback). + if not device_node.rpc_client().get_bdevs(device.nvme_bdev): + try: + filename = device.by_id_path or device.device_path + try: + inventory, _ = device_node.client(timeout=30, retry=1).get_blockdevices() + for blk in inventory or []: + if blk.get("serial") == device.serial_number: + filename = blk.get("by_id_path") or blk.get("device_path") + break + except Exception as e: + logger.warning(f"blockdevices inventory failed, using stored path: {e}") + if not filename: + logger.error(f"No block device path known for {device_id}") + return False + device_node.rpc_client().bdev_aio_create(device.nvme_bdev, filename) + device_node.rpc_client().bdev_examine(device.nvme_bdev) + device_node.rpc_client().bdev_wait_for_examine() + device_node.rpc_client().bdev_set_qd_sampling_period( + device.nvme_bdev, constants.AIO_QD_SAMPLING_PERIOD_US) + except Exception as e: + logger.error(e) + return False + if not device_node.rpc_client().get_bdevs(device.nvme_bdev): + logger.error(f"Failed to find AIO bdev {device.nvme_bdev}") return False + else: + if not device_node.rpc_client().bdev_nvme_controller_list(device.nvme_controller): + try: + ret = device_node.client(timeout=30, retry=1).bind_device_to_spdk(device.pcie_address) + logger.debug(ret) + device_node.rpc_client().bdev_nvme_controller_attach(device.nvme_controller, device.pcie_address) + except Exception as e: + logger.error(e) + return False - if not device_node.rpc_client().bdev_nvme_controller_list(device.nvme_controller): - logger.error(f"Failed to find device nvme controller {device.nvme_controller}") - return False + if not device_node.rpc_client().bdev_nvme_controller_list(device.nvme_controller): + logger.error(f"Failed to find device nvme controller {device.nvme_controller}") + return False new_device = NVMeDevice(device.to_dict()) new_device.uuid = str(uuid.uuid4()) @@ -1379,6 +1447,16 @@ def get_device_health_info(device_id): logger.error(e) return False + if device.bdev_type == "aio": + # SMART is not reachable through SPDK for AIO bdevs; host-side + # smartctl via the node agent is a possible follow-up. + return json.dumps({ + "bdev_type": "aio", + "device_path": device.device_path, + "smart": None, + "message": "SMART data is not available through SPDK for AIO devices", + }, indent=2) + rpc_client = snode.rpc_client() ret = rpc_client.bdev_nvme_get_controller_health_info(device.nvme_controller) return json.dumps(ret, indent=2) \ No newline at end of file diff --git a/simplyblock_core/env_var b/simplyblock_core/env_var index 5abdecd440..0ad369b78b 100644 --- a/simplyblock_core/env_var +++ b/simplyblock_core/env_var @@ -2,4 +2,4 @@ SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev SIMPLY_BLOCK_VERSION=19.2.34 SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main -SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:main-latest +SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:md-journal-latest diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 82401eba93..4bb6143c65 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -184,6 +184,13 @@ def is_topology_owned(self) -> bool: # Deploy-time only — set at cluster create/add, never toggled at runtime; # an existing cluster must be redeployed to gain the feature. enable_failure_domain: bool = False + # Storage-device mode for the whole cluster. "nvme" (default): NVMe PCIe + # controllers auto-detected and attached through the SPDK nvme bdev. + # "lblk": arbitrary Linux block devices wrapped in SPDK AIO bdevs (one + # per device); everything from alceml upward is identical. Deploy-time + # only — set at cluster create/add, never toggled at runtime. Inter-node + # fabric (nvme-tcp/rdma) is unaffected by this mode. + device_mode: str = "nvme" snapshot_replication_target_cluster: str = "" snapshot_replication_target_pool: str = "" snapshot_replication_timeout: int = 60*10 diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index e36a80c700..fd6a8c0def 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -74,6 +74,19 @@ class NVMeDevice(BaseModel): # Passthrough bdev UUID for cross-node nvme bdev identification, # meaning that remote bdev to this bdev would share the same uuid. pt_bdev_uuid: str = "" + # Base-bdev type discriminator: "nvme" (SPDK nvme bdev over a PCIe + # controller) or "aio" (SPDK AIO bdev over a Linux block device, lblk + # cluster mode). For "aio" devices, pcie_address and nvme_controller stay + # empty and nvme_bdev holds the AIO bdev name; identity is serial_number + # (lsblk SERIAL/WWN or a synthetic stable id), with device_path / + # by_id_path re-resolved from the live host on every restart. + bdev_type: str = "nvme" + # Current kernel device path (e.g. /dev/sdb) — informational; re-learned + # each restart, never used as identity when a serial is available. + device_path: str = "" + # Stable /dev/disk/by-id/... symlink when the device has one; preferred + # as the AIO bdev filename so udev renames cannot bite mid-flight. + by_id_path: str = "" def __change_dev_connection_to(self, connecting_from_node): # Targeted single-record write. The previous implementation scanned diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 095848fc16..5fafd3db58 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -109,6 +109,11 @@ class StorageNode(BaseNodeObject): partitions_count: int = 0 # Unused poller_cpu_cores: List[int] = [] ssd_pcie: List = [] + # lblk cluster mode: the configured block-device selection for this node, + # entries {name, serial, by_id, size, numa}. Parallel to ssd_pcie (which + # stays empty in lblk mode). Persisted so restart re-resolves devices + # (serial-first) without depending on the host config file. + lblk_devices: List[dict] = [] pollers_mask: str = "" primary_ip: str = "" raid: str = "" diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index f36fe067a6..52abe70f96 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1206,6 +1206,36 @@ def bdev_examine(self, name): def bdev_wait_for_examine(self): return self._request("bdev_wait_for_examine") + def bdev_aio_create(self, name, filename, block_size=0): + """Create an SPDK AIO bdev over a Linux block device (lblk cluster + mode). ``filename`` is the device path — prefer the stable + /dev/disk/by-id symlink. ``block_size`` 0 lets SPDK use the device's + logical block size.""" + params = {"name": name, "filename": filename} + if block_size: + params["block_size"] = block_size + return self._request("bdev_aio_create", params) + + def bdev_aio_delete(self, name): + return self._request("bdev_aio_delete", {"name": name}) + + def bdev_aio_rescan(self, name): + """Re-read the backing device's size (device grow pickup).""" + return self._request("bdev_aio_rescan", {"name": name}) + + def bdev_set_qd_sampling_period(self, name, period_us): + """Enable queue-depth sampling on a bdev so bdev_get_iostat reports + queue_depth/io_time — the hung-IO watchdog's signal for AIO base + bdevs (period 0 disables).""" + params = {"name": name, "period": period_us} + return self._request("bdev_set_qd_sampling_period", params) + + def get_bdevs_2(self, name): + """(ret, err) probe variant of bdev_get_bdevs, mirroring + bdev_nvme_controller_list_2 — used where the caller must distinguish + 'bdev gone' from RPC failure without raising.""" + return self._request2("bdev_get_bdevs", {"name": name}) + def bdev_enable_histogram(self, name, enable=True, opc=None): # opc filters to a single I/O type (e.g. "read"/"write"); requires # SPDK >= 24.01. Toggling disable->enable clears the collected data, diff --git a/simplyblock_core/services/device_monitor.py b/simplyblock_core/services/device_monitor.py index e564cbf88c..0b5ed80c9d 100644 --- a/simplyblock_core/services/device_monitor.py +++ b/simplyblock_core/services/device_monitor.py @@ -3,6 +3,7 @@ from simplyblock_core import constants, db_controller, utils from simplyblock_core.controllers import tasks_controller, device_controller +from simplyblock_core.controllers.device_controller import CAUSE_LOCAL_FAILURE from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.storage_node import StorageNode @@ -15,6 +16,151 @@ db = db_controller.DBController() +# --- lblk (AIO) watchdog state ------------------------------------------- +# AIO bdevs have no bdev_nvme-style IO timeout (timeout_us + +# action_on_timeout=reset is what converts hung IO into failed IO — and +# thereby distrib error_* events — on the nvme path). For AIO devices the +# control plane compensates here: queue-depth sampling is enabled on every +# aio bdev at creation, so bdev_get_iostat reports queue_depth; a device +# with inflight IO and zero completion progress across +# AIO_HUNG_IO_STALL_POLLS consecutive sweeps is declared stalled and fed +# into the exact same machinery an erroring nvme device hits +# (io_error + UNAVAILABLE with a countable LOCAL_FAILURE cause → +# auto-restart budget → flap limit → FAILED → migration). +# +# _aio_progress: device_id -> (last_total_completed_ops, consecutive_stalls) +# _aio_absent: device_id -> consecutive polls missing from the host lsblk +_aio_progress: dict = {} +_aio_absent: dict = {} + + +def _aio_total_ops(stat: dict) -> int: + return (int(stat.get("num_read_ops") or 0) + + int(stat.get("num_write_ops") or 0) + + int(stat.get("num_unmap_ops") or 0)) + + +def _check_aio_hung_io(node, rpc_client) -> list: + """Return the node's ONLINE aio devices whose IO is stalled past the + threshold. An RPC failure or missing queue_depth counts as UNKNOWN — + the stall counter is frozen, not advanced: a wedged SPDK reactor slows + the RPC path itself, and mgmt-plane slowness must not be converted + into device failures (cf. constants NVME_TIMEOUT_US rationale).""" + stalled = [] + for dev in node.nvme_devices: + if dev.bdev_type != "aio" or dev.status != NVMeDevice.STATUS_ONLINE: + _aio_progress.pop(dev.get_id(), None) + continue + try: + ret = rpc_client.get_lvol_stats(dev.nvme_bdev) + except Exception as e: + logger.debug(f"iostat failed for {dev.nvme_bdev}: {e}") + continue # unknown — freeze + bdevs = (ret or {}).get("bdevs") or [] + if not bdevs: + continue # unknown — freeze + stat = bdevs[0] + queue_depth = stat.get("queue_depth") + if queue_depth is None: + # qd-sampling not active (fork without the fields, or sampling + # lost across an SPDK restart) — re-arm it and skip this poll. + try: + rpc_client.bdev_set_qd_sampling_period( + dev.nvme_bdev, constants.AIO_QD_SAMPLING_PERIOD_US) + except Exception: + pass + continue + total = _aio_total_ops(stat) + last_total, stalls = _aio_progress.get(dev.get_id(), (None, 0)) + # A stall tick requires inflight IO on THIS poll and zero completion + # progress since the previous one; any progress resets the window. + if last_total is not None and total == last_total and queue_depth > 0: + stalls += 1 + else: + stalls = 0 + _aio_progress[dev.get_id()] = (total, stalls) + if stalls >= constants.AIO_HUNG_IO_STALL_POLLS: + stalled.append(dev) + return stalled + + +def _check_aio_device_presence(node) -> list: + """Return the node's aio devices whose backing block device has been + absent from the host inventory for AIO_DEVICE_ABSENT_POLLS consecutive + sweeps (hot-removal). Inventory failure = unknown — counters freeze.""" + aio_devs = [dev for dev in node.nvme_devices + if dev.bdev_type == "aio" + and dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, + NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]] + if not aio_devs: + return [] + try: + inventory, _ = node.client(timeout=10, retry=1).get_blockdevices() + except Exception as e: + logger.debug(f"blockdevices inventory failed for node {node.get_id()}: {e}") + return [] + if not inventory: + return [] + serials = {d.get("serial") for d in inventory} + names = {d.get("name") for d in inventory} + gone = [] + for dev in aio_devs: + if dev.serial_number in serials or dev.device_name in names: + _aio_absent.pop(dev.get_id(), None) + continue + absent = _aio_absent.get(dev.get_id(), 0) + 1 + _aio_absent[dev.get_id()] = absent + if absent >= constants.AIO_DEVICE_ABSENT_POLLS: + gone.append(dev) + return gone + + +def _sweep_aio_devices(node) -> None: + """lblk failure parity: hot-removal → device_remove (the treatment + SPDK_BDEV_EVENT_REMOVE gets), hung IO → io_error + UNAVAILABLE with a + countable cause. Node-level pattern (>=2 devices stalled at once — + reactor stall, controller, expander) escalates to a node auto-restart + instead of failing devices one by one, mirroring the >=2 rule of the + io_error auto-restart path below.""" + gone = _check_aio_device_presence(node) + for dev in gone: + logger.warning(f"AIO device {dev.get_id()} ({dev.device_name}, serial " + f"{dev.serial_number}) disappeared from host inventory; removing") + _aio_absent.pop(dev.get_id(), None) + _aio_progress.pop(dev.get_id(), None) + try: + device_controller.device_remove(dev.get_id(), cause=CAUSE_LOCAL_FAILURE) + except Exception as e: + logger.error(f"device_remove failed for {dev.get_id()}: {e}") + + try: + rpc_client = node.rpc_client() + stalled = _check_aio_hung_io(node, rpc_client) + except Exception as e: + logger.debug(f"hung-IO sweep failed for node {node.get_id()}: {e}") + return + if not stalled: + return + for dev in stalled: + _aio_progress.pop(dev.get_id(), None) + if len(stalled) >= 2: + logger.warning(f"{len(stalled)} AIO devices stalled simultaneously on " + f"node {node.get_id()}; treating as node-level and " + f"queueing node auto-restart") + tasks_controller.add_node_to_auto_restart(node) + return + dev = stalled[0] + logger.warning(f"AIO device {dev.get_id()} ({dev.nvme_bdev}) has inflight IO " + f"with no completion progress for " + f"{constants.AIO_HUNG_IO_STALL_POLLS * constants.DEV_MONITOR_INTERVAL_SEC}s; " + f"marking unavailable") + try: + device_controller.device_set_io_error(dev.get_id(), True) + device_controller.device_set_unavailable(dev.get_id(), cause=CAUSE_LOCAL_FAILURE) + except Exception as e: + logger.error(f"failed to mark stalled device {dev.get_id()}: {e}") + + def main(): logger.info("Starting Device monitor...") while True: @@ -35,6 +181,12 @@ def main(): if node.status != StorageNode.STATUS_ONLINE: logger.warning(f"Node status is not online, id: {node.get_id()}, status: {node.status}") continue + + if cluster.device_mode == constants.DEVICE_MODE_LBLK: + _sweep_aio_devices(node) + # Re-read: the sweep may have changed device statuses. + node = db.get_storage_node_by_id(node.get_id()) + for dev in node.nvme_devices: if dev.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]: diff --git a/simplyblock_core/services/main_distr_event_collector.py b/simplyblock_core/services/main_distr_event_collector.py index 2f2acf31fd..a8eb1851fd 100644 --- a/simplyblock_core/services/main_distr_event_collector.py +++ b/simplyblock_core/services/main_distr_event_collector.py @@ -236,8 +236,15 @@ def process_device_event(event, logger): logger.info(f"event was fired {time_delta.total_seconds()} seconds ago, target remote controller ok, skipping") event.status = f'skipping_late_by_{int(time_delta.total_seconds())}s_but_controller_ok' return - ret, err = event_node_obj.rpc_client().bdev_nvme_controller_list_2(device_obj.nvme_controller) - if err and err['code'] == 22: + if device_obj.bdev_type == "aio": + # AIO devices have no nvme controller — probe the base + # bdev instead: bdev gone => the late event is real. + ret, err = event_node_obj.rpc_client().get_bdevs_2(device_obj.nvme_bdev) + controller_missing = bool(err) or not ret + else: + ret, err = event_node_obj.rpc_client().bdev_nvme_controller_list_2(device_obj.nvme_controller) + controller_missing = bool(err) and err['code'] == 22 + if controller_missing: logger.info(f"event was fired {time_delta.total_seconds()} seconds ago, checking controller filed") event.status = f'late_by_{int(time_delta.total_seconds())}s' else: diff --git a/simplyblock_core/snode_client.py b/simplyblock_core/snode_client.py index 0872b0a6bc..17783ee558 100644 --- a/simplyblock_core/snode_client.py +++ b/simplyblock_core/snode_client.py @@ -187,6 +187,15 @@ def join_swarm(self, cluster_ip, join_token, db_connection, cluster_id): def spdk_process_kill(self, rpc_port, cluster_id=None): return self._request("GET", "spdk_process_kill", {"rpc_port": rpc_port, "cluster_id": cluster_id}) + def spdk_process_cleanup(self, rpc_port, cluster_id=None): + """Slow, authoritative SPDK teardown: restart policy cleared, remove + synchronous, success only when the containers/pod are verifiably + GONE. Use on failure-cleanup paths (spdk_process_kill is the fast + peer-termination sibling whose detached remove can lose against a + restart policy).""" + return self._request("GET", "spdk_process_cleanup", + {"rpc_port": rpc_port, "cluster_id": cluster_id}) + def leave_swarm(self): return True # return self._request("GET", "leave_swarm") @@ -216,6 +225,16 @@ def bind_device_to_spdk(self, device_pci): params = {"device_pci": device_pci} return self._request("POST", "bind_device_to_spdk", params) + def get_blockdevices(self): + """Whole-disk inventory for the lblk cluster mode.""" + return self._request("GET", "blockdevices") + + def wipe_block_device(self, device_name): + """--force-format for lblk add-node: wipe partition/FS signatures + from a whole disk (refused when busy).""" + return self._request("POST", "wipe_block_device", + {"device_name": device_name}) + def spdk_process_is_up(self, rpc_port, cluster_id): params = {"rpc_port": rpc_port, "cluster_id": cluster_id} return self._request("GET", "spdk_process_is_up", params) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 16ce871d92..00f88b5566 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -147,6 +147,27 @@ def _kill_spdk_until_dead(snode, max_attempts=3, poll_per_attempt_sec=5, # poll_per_attempt_sec of CPU per attempt. rounds_per_attempt = max(1, int(poll_per_attempt_sec / poll_interval)) for attempt in range(1, max_attempts + 1): + # Prefer the authoritative container-level cleanup: it clears the + # restart policy and removes synchronously, so success means the + # container is verifiably GONE — not merely "RPC socket down". + # spdk_process_is_up probes the RPC Unix socket, which false- + # negatives an SPDK that booted but never brought its RPC up; its + # zombie container then squats the host's hugepages and starves + # every subsequent add/restart attempt (2026-08-05 incident, and + # the resurrection race: kill's detached remove vs restart policy). + try: + ret, _err = snode_api.spdk_process_cleanup(snode.rpc_port, snode.cluster_id) + if ret: + logger.info( + "SPDK on %s cleaned up (container-level, attempt %d/%d)", + snode.get_id(), attempt, max_attempts, + ) + return True + except Exception as e: + # Older agent without the endpoint, or transient failure — + # fall back to the legacy kill + socket-poll below. + logger.debug("spdk_process_cleanup unavailable on %s: %s", + snode.get_id(), e) try: snode_api.spdk_process_kill(snode.rpc_port, snode.cluster_id) except Exception as e: @@ -2000,6 +2021,97 @@ def _peer_reachable_via_jm_quorum(target_node_id, this_node, peer_probe_timeout= return not probed +def verify_jm_mesh_coverage(cluster_id, repair=True): + """Verify the JC journal mesh: every ONLINE node must hold a live remote + bdev for every remote JM it references (``jm_ids``) whose OWNER node is + itself ONLINE. Returns a list of problem strings (empty = healthy). + + Rationale (2026-08-05 incident): two nodes joined through add-node + retries and the peers never attached their ``remote_jm_*`` controllers; + the cluster activated and reported healthy while a third of the journal + mesh was unreachable. First journal load excluded those JMs, n_safe_jms + collapsed and JCERR cascaded cluster-wide. This check is the activation + gate for exactly that hole. + + ``repair=True`` re-runs _connect_to_remote_jm_devs once for nodes with + missing coverage before reporting. JMs whose owner is not ONLINE are + skipped — a re-activation with one or two unhealthy nodes must never be + blocked by their (legitimately absent) journals. + """ + db_controller = DBController() + nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id) + jm_owner_by_id = {} + for n in nodes: + if n.jm_device and n.jm_device.get_id(): + jm_owner_by_id[n.jm_device.get_id()] = n + + problems = [] + for node in nodes: + if node.status != StorageNode.STATUS_ONLINE or not node.enable_ha_jm: + continue + expected = {} + for entry in (node.remote_jm_devices or []): + owner_id = entry.node_id + if entry.remote_bdev: + expected[owner_id] = entry.remote_bdev + missing = [] + for jm_id in (node.jm_ids or []): + owner = jm_owner_by_id.get(jm_id) + if owner is None or owner.get_id() == node.get_id(): + continue + if owner.status != StorageNode.STATUS_ONLINE: + continue # recovery tolerance: absent owner, absent journal + remote_bdev = expected.get(owner.get_id()) + if not remote_bdev: + missing.append((jm_id, owner.get_id(), "")) + rpc_client = node.rpc_client(timeout=10, retry=2) + for owner_id, remote_bdev in expected.items(): + owner = None + for n in nodes: + if n.get_id() == owner_id: + owner = n + break + if owner is not None and owner.status != StorageNode.STATUS_ONLINE: + continue + try: + present = bool(rpc_client.get_bdevs(remote_bdev)) + except Exception: + present = False + if not present: + missing.append(("", owner_id, remote_bdev)) + + if missing and repair: + logger.warning( + f"JM mesh: node {node.get_id()} missing {len(missing)} remote " + f"JM bdev(s); attempting reconnect") + try: + fresh = db_controller.get_storage_node_by_id(node.get_id()) + fresh.remote_jm_devices = _connect_to_remote_jm_devs(fresh) + fresh.write_to_db(db_controller.kv_store) + still = [] + rpc_client = fresh.rpc_client(timeout=10, retry=2) + for jm_id, owner_id, remote_bdev in missing: + fixed = False + for entry in (fresh.remote_jm_devices or []): + if entry.node_id == owner_id and entry.remote_bdev: + try: + fixed = bool(rpc_client.get_bdevs(entry.remote_bdev)) + except Exception: + fixed = False + break + if not fixed: + still.append((jm_id, owner_id, remote_bdev)) + missing = still + except Exception as e: + logger.error(f"JM mesh repair failed for {node.get_id()}: {e}") + + for jm_id, owner_id, remote_bdev in missing: + problems.append( + f"node {node.get_id()}: unreachable remote JM of node " + f"{owner_id} (bdev {remote_bdev})") + return problems + + def _connect_to_remote_jm_devs(this_node, jm_ids=None, only_node_id=None): """Connect ``this_node`` to remote JM devices and return the refreshed remote-JM records. @@ -2349,7 +2461,8 @@ def _cluster_add_lock_heartbeat(db_controller, cluster_id, owner, stop_event): return -def _classify_existing_endpoint_record(db_controller, cluster_id, node_addr, ssd_pcie): +def _classify_existing_endpoint_record(db_controller, cluster_id, node_addr, ssd_pcie, + lblk_serials=None): """Classify a pre-existing storage-node record for ``node_addr`` that owns one of the joining node's SSDs, before an add-node proceeds. @@ -2358,6 +2471,9 @@ def _classify_existing_endpoint_record(db_controller, cluster_id, node_addr, ssd their channels were reset mid-command); the caller's retry then finds the record already present and must not fail permanently on it. + Device ownership is tested by PCIe overlap (nvme mode) or by configured + block-device serial overlap (lblk mode, ``lblk_serials``). + Returns one of: - (None, None): no record for this endpoint owns any of these SSDs. - ("already_added", node): record is ONLINE — the earlier add completed; @@ -2367,10 +2483,14 @@ def _classify_existing_endpoint_record(db_controller, cluster_id, node_addr, ssd - ("conflict", node): record in any other status — refuse; the operator must delete or restart that node explicitly. """ + lblk_serials = set(lblk_serials or []) for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): if node.api_endpoint != node_addr: continue - if not any(ssd in node.ssd_pcie for ssd in ssd_pcie): + pcie_overlap = any(ssd in node.ssd_pcie for ssd in ssd_pcie or []) + serial_overlap = bool(lblk_serials and lblk_serials.intersection( + e.get("serial") for e in (node.lblk_devices or []))) + if not pcie_overlap and not serial_overlap: continue if node.status == StorageNode.STATUS_ONLINE: return "already_added", node @@ -2386,7 +2506,8 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, num_partitions_per_dev=0, jm_percent=0, enable_test_device=False, namespace=None, enable_ha_jm=False, cr_name=None, cr_namespace=None, cr_plural=None, id_device_by_nqn=False, partition_size="", ha_jm_count=None, format_4k=False, - spdk_proxy_image=None, spdk_sys_mem=None, expansion=False, failure_domain=None): + spdk_proxy_image=None, spdk_sys_mem=None, expansion=False, failure_domain=None, + force_format=False): snode_api = SNodeClient(node_addr) node_info, _ = snode_api.info() if node_info.get("nodes_config") and node_info["nodes_config"].get("nodes"): @@ -2535,10 +2656,37 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, minimum_sys_memory = node_config.get("sys_memory") max_lvol = node_config.get("max_lvol") ssd_pcie = node_config.get("ssd_pcis") + lblk_configured = node_config.get("lblk_devices") or [] - if ssd_pcie: + lblk_mode = cluster.device_mode == constants.DEVICE_MODE_LBLK + if lblk_mode: + if not lblk_configured: + logger.error( + "This cluster runs in lblk device mode but the node config " + "carries no 'lblk_devices'; run 'sn configure --lblk ...' first.") + return False + if ssd_pcie: + logger.error("lblk device mode: the node config must not carry " + "'ssd_pcis' entries") + return False + # Phase 1: lblk requires journal-on-device (the GPT-partition JM + # mode detaches/re-attaches NVMe controllers to re-examine). + if num_partitions_per_dev != 0 and jm_percent != 0: + logger.error("lblk device mode requires --enable-journal-device " + "(journal on a dedicated device); partitioned " + "journal mode is not supported") + return False + elif lblk_configured and not ssd_pcie: + logger.error( + "The node config carries 'lblk_devices' but this cluster runs " + f"in {cluster.device_mode} device mode; re-run 'sn configure' " + "without --lblk or create the cluster with --device-mode lblk.") + return False + + if ssd_pcie or lblk_configured: action, existing = _classify_existing_endpoint_record( - db_controller, cluster_id, node_addr, ssd_pcie) + db_controller, cluster_id, node_addr, ssd_pcie, + lblk_serials=[e.get("serial") for e in lblk_configured]) if action == "cleanup": # Repeated partial attempts can leave several stale records # for the same endpoint; we clean one per task retry. @@ -2740,11 +2888,41 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, results = None l_cores = node_config.get("l-cores") spdk_cpu_mask = node_config.get("cpu_mask") - for ssd in ssd_pcie: - if format_4k: - snode_api.format_device_with_4k(ssd) + lblk_resolved = [] + if lblk_mode: + # No driver rebind in lblk mode — the AIO bdev needs the device + # on its kernel driver. Resolve the configured selection against + # the live host (serial-first), then --force-format wipes + # partitioned disks (re-validated host-side: busy => refused). + blk_inventory, err = snode_api.get_blockdevices() + if not blk_inventory: + logger.error(f"Failed to list block devices on {node_addr}: {err}") + return False + lblk_resolved, missing = utils.resolve_lblk_entries( + lblk_configured, blk_inventory) + if missing: + logger.error( + f"Configured block device(s) not found on host: " + f"{[(e.get('name'), e.get('serial')) for e in missing]}") + return False + for blk_info in lblk_resolved: + if blk_info.get("has_partitions"): + if not force_format: + logger.error( + f"Block device {blk_info['name']} is partitioned; " + f"pass --force-format to wipe it, or exclude it") + return False + ret, err = snode_api.wipe_block_device(blk_info["name"]) + if not ret: + logger.error(f"Failed to wipe block device " + f"{blk_info['name']}: {err}") + return False + else: + for ssd in ssd_pcie: + if format_4k: + snode_api.format_device_with_4k(ssd) + snode_api.bind_device_to_spdk(ssd) snode_api.bind_device_to_spdk(ssd) - snode_api.bind_device_to_spdk(ssd) if not spdk_proxy_image: spdk_proxy_image = cluster.container_image_prefix + constants.SIMPLY_BLOCK_DOCKER_IMAGE @@ -2760,7 +2938,14 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, namespace, mgmt_ip, rpc_port, rpc_user, rpc_pass, multi_threading_enabled=constants.SPDK_PROXY_MULTI_THREADING_ENABLED, timeout=constants.SPDK_PROXY_TIMEOUT, - ssd_pcie=ssd_pcie, total_mem=total_mem, system_mem=minimum_sys_memory, cluster_mode=cluster.mode, + # lblk mode: never pass an empty PCI allowlist — DPDK treats + # it as allow-all and SPDK's nvme driver could claim kernel + # NVMe disks (the k8s path passes PCI_ALLOWED="" today). The + # host-bridge placeholder is a valid BDF that never matches a + # storage device. + ssd_pcie=(ssd_pcie if not lblk_mode + else [constants.LBLK_PCI_ALLOWED_PLACEHOLDER]), + total_mem=total_mem, system_mem=minimum_sys_memory, cluster_mode=cluster.mode, socket=node_socket, cluster_id=cluster_id, spdk_proxy_image=spdk_proxy_image, mcp_max_unavailable=mcp_max_unavailable) time.sleep(5) @@ -2871,7 +3056,10 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, snode.cr_name = cr_name snode.cr_namespace = cr_namespace snode.cr_plural = cr_plural - snode.ssd_pcie = ssd_pcie + snode.ssd_pcie = ssd_pcie if not lblk_mode else [] + # lblk mode: persist the configured selection (with serial identity) + # so restart can re-resolve devices without the host config file. + snode.lblk_devices = lblk_configured if lblk_mode else [] snode.hostname = hostname snode.host_nqn = subsystem_nqn snode.subsystem = subsystem_nqn @@ -3106,13 +3294,16 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, # snode.ssd_pcie = node_info['spdk_pcie_list'] # snode.write_to_db() # discover devices - if not snode.ssd_pcie: - node_info, _ = snode_api.info() - ssds = node_info['spdk_pcie_list'] + if lblk_mode: + nvme_devs = utils.addAioDevices(rpc_client, snode, lblk_resolved) else: - ssds = snode.ssd_pcie + if not snode.ssd_pcie: + node_info, _ = snode_api.info() + ssds = node_info['spdk_pcie_list'] + else: + ssds = snode.ssd_pcie - nvme_devs = addNvmeDevices(rpc_client, snode, ssds) + nvme_devs = addNvmeDevices(rpc_client, snode, ssds) if nvme_devs: for nvme in nvme_devs: @@ -3868,6 +4059,8 @@ def _finalize_node_removal(removed_node): snode_api.leave_swarm() pci_address = [] for dev in removed_node.nvme_devices: + if dev.bdev_type == "aio": + continue # no PCIe identity; lblk devices are wiped via wipe_block_device if dev.pcie_address not in pci_address: ret = snode_api.delete_dev_gpt_partitions(dev.pcie_address) logger.debug(ret) @@ -4296,6 +4489,11 @@ def _restart_storage_node_impl( snode.hostname = node_info['hostname'] if snode.num_partitions_per_dev == 0 and reattach_volume: + if snode.lblk_devices: + # EBS re-homing + PCI rebinding is nvme-mode machinery; + # lblk devices re-resolve by serial at discovery below. + logger.error("--reattach-volume is not supported on lblk-mode nodes") + return False new_cloud_instance_id = node_info['cloud_instance']['id'] detached_volumes = node_utils.detach_ebs_volumes(snode.cloud_instance_id) if not detached_volumes: @@ -4441,9 +4639,16 @@ def _restart_storage_node_impl( if not snode.spdk_proxy_image: snode.spdk_proxy_image = cluster.container_image_prefix + constants.SIMPLY_BLOCK_DOCKER_IMAGE + lblk_mode = cluster.device_mode == constants.DEVICE_MODE_LBLK + results = None try: if new_ssd_pcie and type(new_ssd_pcie) is list: + if lblk_mode: + # Phase 1: growing an lblk node's device set goes through + # `sn configure --lblk` + re-add, not restart-time PCI binds. + logger.error("--ssd-pcie is not supported on lblk-mode clusters") + return False for new_ssd in new_ssd_pcie: if new_ssd not in snode.ssd_pcie: try: @@ -4460,7 +4665,11 @@ def _restart_storage_node_impl( snode.l_cores, snode.spdk_mem, snode.spdk_image, spdk_debug, cluster_ip, fdb_connection, snode.namespace, snode.mgmt_ip, snode.rpc_port, snode.rpc_username, snode.rpc_password, multi_threading_enabled=constants.SPDK_PROXY_MULTI_THREADING_ENABLED, timeout=constants.SPDK_PROXY_TIMEOUT, - ssd_pcie=snode.ssd_pcie, total_mem=total_mem, system_mem=minimum_sys_memory, cluster_mode=cluster.mode, + # lblk: placeholder allowlist — an empty list means allow-all to + # DPDK and SPDK's nvme driver could claim kernel NVMe disks. + ssd_pcie=(snode.ssd_pcie if not lblk_mode + else [constants.LBLK_PCI_ALLOWED_PLACEHOLDER]), + total_mem=total_mem, system_mem=minimum_sys_memory, cluster_mode=cluster.mode, socket=snode.socket, cluster_id=snode.cluster_id, spdk_proxy_image=snode.spdk_proxy_image) @@ -4609,18 +4818,38 @@ def _restart_storage_node_impl( return False node_info, _ = snode_api.info() - if not snode.ssd_pcie: - ssds = node_info['spdk_pcie_list'] + if lblk_mode: + # Re-resolve the persisted selection against the live host, + # SERIAL-FIRST (kernel names shift across reboots; the stored name + # is only the fallback for serial-less devices), then rebuild the + # AIO bdevs. A missing device degrades to STATUS_REMOVED in the + # reconcile below — same semantics as a missing NVMe controller. + blk_inventory, blk_err = snode_api.get_blockdevices() + if not blk_inventory: + logger.error(f"Failed to list block devices: {blk_err}") + return False + lblk_resolved, lblk_missing = utils.resolve_lblk_entries( + snode.lblk_devices, blk_inventory) + for entry in lblk_missing: + logger.warning(f"Configured block device {entry.get('name')} " + f"(serial {entry.get('serial')}) not found on host") + nvme_devs = utils.addAioDevices(rpc_client, snode, lblk_resolved) + if not nvme_devs: + logger.error("No eligible block devices were found!") + return False else: - ssds = [] - for ssd in snode.ssd_pcie: - if ssd in node_info['spdk_pcie_list']: - ssds.append(ssd) - - nvme_devs = addNvmeDevices(rpc_client, snode, ssds) - if not nvme_devs: - logger.error("No NVMe devices was found!") - return False + if not snode.ssd_pcie: + ssds = node_info['spdk_pcie_list'] + else: + ssds = [] + for ssd in snode.ssd_pcie: + if ssd in node_info['spdk_pcie_list']: + ssds.append(ssd) + + nvme_devs = addNvmeDevices(rpc_client, snode, ssds) + if not nvme_devs: + logger.error("No NVMe devices was found!") + return False logger.info(f"Devices found: {len(nvme_devs)}") logger.debug(nvme_devs) @@ -4645,8 +4874,15 @@ def _restart_storage_node_impl( if not db_dev.is_partition and not found_dev.is_partition: db_dev.device_name = found_dev.device_name db_dev.nvme_bdev = found_dev.nvme_bdev - db_dev.nvme_controller = found_dev.nvme_controller - db_dev.pcie_address = found_dev.pcie_address + if found_dev.bdev_type == "aio": + # AIO devices have no controller/PCIe identity; refresh + # the re-resolved kernel path instead. + db_dev.bdev_type = "aio" + db_dev.device_path = found_dev.device_path + db_dev.by_id_path = found_dev.by_id_path + else: + db_dev.nvme_controller = found_dev.nvme_controller + db_dev.pcie_address = found_dev.pcie_address # if db_dev.status in [ NVMeDevice.STATUS_ONLINE]: # db_dev.status = NVMeDevice.STATUS_UNAVAILABLE @@ -5078,7 +5314,9 @@ def list_storage_devices(node_id): "Name": device.alceml_name, "Size": utils.humanbytes(device.size), "Serial Number": device.serial_number, - "PCIe": device.pcie_address, + # lblk (aio) devices have no PCIe identity — show the kernel path. + "PCIe": (device.pcie_address if device.bdev_type != "aio" + else device.device_path), "Status": device.status, "IO Err": device.io_error, # Device health is only meaningful when its node is ONLINE/DOWN. @@ -5712,6 +5950,8 @@ def shutdown_storage_node(node_id, force=False, keep_auto_restart=False, return False pci_address = [] for dev in snode.nvme_devices: + if dev.bdev_type == "aio": + continue # lblk devices never left their kernel driver if dev.pcie_address not in pci_address: try: ret = snode.client(timeout=30, retry=1).bind_device_to_nvme(dev.pcie_address) @@ -6015,7 +6255,7 @@ def upgrade_automated_deployment_config(): def generate_automated_deployment_config(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_allowed, pci_blocked, cores_percentage=0, force=False, device_model="", size_range="", nvme_names=None, k8s=False, - calculate_hp_only=False, number_of_devices=0): + calculate_hp_only=False, number_of_devices=0, lblk_selection=None): if calculate_hp_only: minimum_hp_memory = utils.calculate_hp_only(max_lvol, number_of_devices, sockets_to_use, nodes_per_socket, cores_percentage) hp_number = math.ceil(minimum_hp_memory / 2) @@ -6027,13 +6267,16 @@ def generate_automated_deployment_config(max_lvol, max_prov, sockets_to_use, nod if total_cores < 6: raise ValueError("Error: Not enough CPU cores to deploy storage node. Minimum 6 cores required.") - # load vfio_pci and uio_pci_generic - utils.load_kernel_module("vfio_pci") - utils.load_kernel_module("uio_pci_generic") + if lblk_selection is None: + # load vfio_pci and uio_pci_generic (nvme mode only — lblk keeps + # devices on their kernel driver, SPDK accesses them via AIO) + utils.load_kernel_module("vfio_pci") + utils.load_kernel_module("uio_pci_generic") nodes_config, system_info = utils.generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_allowed, pci_blocked, cores_percentage, force=force, - device_model=device_model, size_range=size_range, nvme_names=nvme_names) + device_model=device_model, size_range=size_range, nvme_names=nvme_names, + lblk_selection=lblk_selection) if not nodes_config or not nodes_config.get("nodes"): return False utils.store_config_file(nodes_config, constants.NODES_CONFIG_FILE, create_read_only_file=True) diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index d15547c9aa..8630bbdb40 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1,5 +1,6 @@ # coding=utf-8 import glob +import hashlib import json import logging import math @@ -1395,6 +1396,107 @@ def addNvmeDevices(rpc_client, snode, devs): return devices +def aio_bdev_name_for_serial(serial: str) -> str: + """Stable AIO bdev name derived from the device's serial identity — the + lblk analogue of the PCI-derived nvme controller name. Survives kernel + device renames across reboots. Whenever sanitization loses information + (special chars replaced, or truncation), a short hash of the ORIGINAL + serial is appended so distinct serials can never collide.""" + sanitized = re.sub(r"[^A-Za-z0-9_]", "_", serial) + if sanitized != serial or len(sanitized) > 40: + digest = hashlib.sha1(serial.encode()).hexdigest()[:6] + sanitized = f"{sanitized[:40]}_{digest}" + return f"aio_{sanitized}" + + +def resolve_lblk_entries(configured_entries, host_devices): + """Match the node's configured lblk devices against the live host + inventory, SERIAL-FIRST: kernel names shift across reboots, so the stored + name is only a fallback for devices without a resolvable serial. Returns + ``(resolved, missing)`` where resolved entries carry the CURRENT + name/path/by-id.""" + by_serial = {d["serial"]: d for d in host_devices} + by_name = {d["name"]: d for d in host_devices} + resolved, missing = [], [] + for entry in configured_entries: + live = by_serial.get(entry.get("serial")) or by_name.get(entry.get("name")) + if live is None: + missing.append(entry) + continue + resolved.append({ + "name": live["name"], + "current_path": live["device_path"], + "serial": entry.get("serial") or live["serial"], + "by_id": live.get("by_id_path") or entry.get("by_id", ""), + "size": int(live.get("size") or entry.get("size") or 0), + "numa": int(live.get("numa_node", entry.get("numa", -1))), + "model": live.get("model", ""), + "has_partitions": bool(live.get("has_partitions")), + }) + return resolved, missing + + +def addAioDevices(rpc_client, snode, blk_entries): + """lblk-mode sibling of addNvmeDevices: create one SPDK AIO bdev per + resolved block device and model it as an NVMeDevice with + bdev_type="aio". Idempotent — an already-present bdev (restart path) is + reused. Everything above the base bdev (alceml, PT, subsystems) is + built by the same code as for nvme devices.""" + devices = [] + next_physical_label = snode.physical_label + for entry in blk_entries: + bdev_name = aio_bdev_name_for_serial(entry["serial"]) + ret = rpc_client.get_bdevs(bdev_name) + if not ret: + # Prefer the by-id path as the filename so a udev rename between + # resolution and create cannot swap devices under us. + filename = entry.get("by_id") or entry["current_path"] + ret = rpc_client.bdev_aio_create(bdev_name, filename) + if not ret: + raise Exception( + f"bdev_aio_create failed for {bdev_name} ({filename}) " + f"on {rpc_client.host}") + rpc_client.bdev_examine(bdev_name) + rpc_client.bdev_wait_for_examine() + + ret = rpc_client.get_bdevs(bdev_name) + if not ret: + raise Exception(f"AIO bdev {bdev_name} not found after create on {rpc_client.host}") + bdev = ret[0] + total_size = bdev['block_size'] * bdev['num_blocks'] + if total_size == 0: + logger.warning(f"Skipping zero-size block device {entry['name']} ({bdev_name})") + continue + + # Queue-depth sampling feeds the control-plane hung-IO watchdog + # (AIO has no bdev_nvme-style timeout/action_on_timeout). + try: + rpc_client.bdev_set_qd_sampling_period( + bdev_name, constants.AIO_QD_SAMPLING_PERIOD_US) + except Exception as e: + logger.warning(f"qd-sampling enable failed on {bdev_name}: {e}") + + devices.append( + NVMeDevice({ + 'uuid': str(uuid.uuid4()), + 'device_name': entry["name"], + 'size': total_size, + 'physical_label': next_physical_label, + 'pcie_address': "", + 'model_id': entry.get("model", ""), + 'serial_number': entry["serial"], + 'nvme_bdev': bdev_name, + 'nvme_controller': "", + 'bdev_type': "aio", + 'device_path': entry["current_path"], + 'by_id_path': entry.get("by_id", ""), + 'node_id': snode.get_id(), + 'cluster_id': snode.cluster_id, + 'status': NVMeDevice.STATUS_ONLINE + })) + return devices + + def get_random_snapshot_vuid(all_lvols=None, all_snapshots=None): # Monotonic allocation via DBController.next_vuid — shares the single vuid # sequence with lvols/clones (one numeric space, so no cross-collision). @@ -1566,6 +1668,142 @@ def detect_nvmes(pci_allowed, pci_blocked, device_model, size_range, nvme_names) return nvmes +def filter_eligible_block_devices(devices, include_names=None, exclude_names=None, + include_serials=None, force_format=False): + """Eligibility filter for the lblk cluster mode (pure — unit-testable). + + ``devices`` is the list produced by node_utils.get_block_devices_info(). + A device is eligible iff it is a whole disk, not a special device + (LBLK_EXCLUDED_NAME_PREFIXES), carries no mountpoint anywhere in its + subtree, has no holders (LVM/md/dm-crypt), does not back the root + filesystem, is not read-only, has a non-zero size, and is unpartitioned + unless ``force_format`` (the actual wipe happens at add-node). + + Selection is one of: ``include_names`` (explicitly requested names must + exist AND be eligible — a busy requested device is a hard error), + ``exclude_names`` (all eligible minus these), ``include_serials`` + (matched against the serial/WWN identity). Without a selection, every + eligible disk is taken. + + Returns ``(eligible_devices, rejected)`` where rejected is a list of + ``(device_dict, reason)``. Raises ValueError on a requested-but- + ineligible name/serial or on duplicate serials among the selection. + """ + include_names = set(include_names or []) + exclude_names = set(exclude_names or []) + include_serials = set(include_serials or []) + + def _ineligible_reason(dev): + if dev.get("type") != "disk": + return "not a whole disk" + if dev["name"].startswith(constants.LBLK_EXCLUDED_NAME_PREFIXES): + return "special device type" + if dev.get("mounted_in_subtree"): + return "mounted (busy)" + if dev.get("holders"): + return f"held by {dev['holders']} (busy)" + if dev.get("is_root_disk"): + return "backs the root filesystem" + if dev.get("ro"): + return "read-only" + if not dev.get("size"): + return "zero size" + if dev.get("has_partitions") and not force_format: + return "partitioned (pass --force to format at add-node)" + return None + + eligible, rejected = [], [] + for dev in devices: + reason = _ineligible_reason(dev) + if reason: + rejected.append((dev, reason)) + else: + eligible.append(dev) + + by_name = {d["name"]: d for d in eligible} + rejected_by_name = {d["name"]: r for d, r in rejected} + if include_names: + missing = include_names - set(by_name) + if missing: + details = {n: rejected_by_name.get(n, "not present") for n in sorted(missing)} + raise ValueError(f"requested block devices are not eligible: {details}") + selected = [by_name[n] for n in sorted(include_names)] + elif include_serials: + by_serial = {d["serial"]: d for d in eligible} + missing_serials = include_serials - set(by_serial) + if missing_serials: + raise ValueError( + f"no eligible block device found for serial(s): {sorted(missing_serials)}") + selected = [by_serial[s] for s in sorted(include_serials)] + else: + selected = [d for d in eligible if d["name"] not in exclude_names] + + serials = [d["serial"] for d in selected] + dupes = {s for s in serials if serials.count(s) > 1} + if dupes: + raise ValueError( + f"duplicate serial number(s) among selected block devices: {sorted(dupes)}; " + f"device identity requires unique serials per node") + return selected, rejected + + +def detect_lblk_devices(include_names=None, exclude_names=None, + include_serials=None, force_format=False): + """Local-host block-device detection for `sn configure --lblk`. + Returns ``{name: config_entry}`` where config_entry is the shape stored + in the node config file's ``lblk_devices`` list.""" + devices = node_utils.get_block_devices_info() + selected, rejected = filter_eligible_block_devices( + devices, include_names=include_names, exclude_names=exclude_names, + include_serials=include_serials, force_format=force_format) + for dev, reason in rejected: + logger.debug(f"block device {dev['name']} skipped: {reason}") + result = {} + for dev in selected: + if dev.get("serial_synthetic"): + logger.warning( + f"block device {dev['name']} has no hardware serial/WWN; using " + f"synthetic identity {dev['serial']} (stable across reboots " + f"only while size and by-id path are unchanged)") + result[dev["name"]] = { + "name": dev["name"], + "serial": dev["serial"], + "by_id": dev.get("by_id_path", ""), + "size": int(dev["size"]), + "numa": int(dev.get("numa_node", -1)), + } + return result + + +def node_config_device_count(node) -> int: + """Number of storage devices a node-config entry carries — ssd_pcis for + nvme mode, lblk_devices for lblk mode.""" + return len(node.get("lblk_devices") or []) or len(node.get("ssd_pcis") or []) + + +# Sys-memory sizing intent (see generate_automated_deployment_config): +# "RAM 4GB min. Plus 0.2% of the storage." The nvme path nominally adds the +# FULL device capacity but in practice always measures 0 — capacity is read +# via `nvme list` AFTER the devices were unbound from the kernel driver. The +# lblk path knows the real sizes, so it applies the documented 0.2% factor +# (2026-08-05 AWS run: summing full capacity demanded 102 GiB sys memory for +# 2x50G EBS volumes on 32 GiB hosts and failed every `sn configure --lblk`). +SYS_MEMORY_STORAGE_FACTOR = 0.002 + + +def node_config_min_sys_memory(node) -> int: + """Minimum system memory for a node-config entry: 2 GiB + 0.2% of total + device capacity. lblk entries carry their sizes; nvme goes through + nvme-cli.""" + lblk = node.get("lblk_devices") or [] + if lblk: + capacity = sum(int(e.get("size") or 0) for e in lblk) + total = 2147483648 + int(capacity * SYS_MEMORY_STORAGE_FACTOR) + logger.debug(f"Minimum system memory is {humanbytes(total)}") + return int(total) + return calculate_minimum_sys_memory(node.get("ssd_pcis") or []) + + def get_total_capacity_of_nvme_devices(pci_lst): json_string = get_nvme_list_verbose() data = json.loads(json_string) @@ -1791,9 +2029,9 @@ def regenerate_config(new_config, old_config, force=False): if old_config["nodes"][i]["socket"] != new_config["nodes"][i]["socket"]: logger.error("The socket is changed, please rerun sbcli configure without upgrade firstly") return False - number_of_alcemls = len(new_config["nodes"][i]["ssd_pcis"]) + number_of_alcemls = node_config_device_count(new_config["nodes"][i]) if (old_config["nodes"][i]["cpu_mask"] != new_config["nodes"][i]["cpu_mask"] or - len(old_config["nodes"][i]["ssd_pcis"]) != len(new_config["nodes"][i]["ssd_pcis"]) or force): + node_config_device_count(old_config["nodes"][i]) != number_of_alcemls or force): try: isolated_cores = hexa_to_cpu_list(new_config["nodes"][i]["cpu_mask"]) except ValueError: @@ -1826,6 +2064,8 @@ def regenerate_config(new_config, old_config, force=False): number_of_distribs = 12 old_config["nodes"][i]["number_of_distribs"] = number_of_distribs old_config["nodes"][i]["ssd_pcis"] = new_config["nodes"][i]["ssd_pcis"] + if new_config["nodes"][i].get("lblk_devices") is not None: + old_config["nodes"][i]["lblk_devices"] = new_config["nodes"][i]["lblk_devices"] old_config["nodes"][i]["nic_ports"] = new_config["nodes"][i]["nic_ports"] for nic in old_config["nodes"][i]["nic_ports"]: if nic not in all_nics: @@ -1843,7 +2083,7 @@ def regenerate_config(new_config, old_config, force=False): old_config["nodes"][i]["small_pool_count"] = small_pool_count old_config["nodes"][i]["large_pool_count"] = large_pool_count old_config["nodes"][i]["huge_page_memory"] = minimum_hp_memory - minimum_sys_memory = calculate_minimum_sys_memory(old_config["nodes"][i]["ssd_pcis"]) + minimum_sys_memory = node_config_min_sys_memory(old_config["nodes"][i]) old_config["nodes"][i]["sys_memory"] = minimum_sys_memory memory_details = node_utils.get_memory_details() @@ -1853,7 +2093,7 @@ def regenerate_config(new_config, old_config, force=False): total_required_memory = 0 all_isolated_cores = set() for node in old_config["nodes"]: - if len(node["ssd_pcis"]) == 0: + if node_config_device_count(node) == 0: logger.error(f"There are no enough SSD devices on numa node {node['socket']}") return False total_required_memory += node["huge_page_memory"] + node["sys_memory"] @@ -1868,7 +2108,8 @@ def regenerate_config(new_config, old_config, force=False): def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_allowed, pci_blocked, - cores_percentage=0, force=False, device_model="", size_range="", nvme_names=None): + cores_percentage=0, force=False, device_model="", size_range="", nvme_names=None, + lblk_selection=None): system_info = {} nodes_config: dict = {"nodes": []} @@ -1876,12 +2117,36 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a validate_sockets(sockets_to_use, cores_by_numa) logger.debug(f"Cores by numa {cores_by_numa}") nics = detect_nics() - nvmes = detect_nvmes(pci_allowed, pci_blocked, device_model, size_range, nvme_names) - if not nvmes: - logger.error( - "There are no enough SSD devices on system, you may run 'sbctl sn clean-devices', to clean devices stored in /etc/simplyblock/sn_config_file") - return False, False - if force: + lblk_mode = lblk_selection is not None + lblk_entries: dict = {} + if lblk_mode: + # lblk cluster mode: eligible Linux block devices instead of NVMe + # PCIe controllers. No driver unbind, no formatting here (--force + # only marks partitioned disks eligible; the wipe happens at + # add-node). Reuse the NVMe NUMA-distribution scaffolding by + # presenting the same {name: {"numa_node": ...}} shape. + try: + lblk_entries = detect_lblk_devices( + include_names=lblk_selection.get("names"), + exclude_names=lblk_selection.get("names_exclude"), + include_serials=lblk_selection.get("serials"), + force_format=force) + except ValueError as e: + logger.error(str(e)) + return False, False + nvmes = {name: {"pci_address": "", "numa_node": entry["numa"]} + for name, entry in lblk_entries.items()} + if not nvmes: + logger.error("No eligible Linux block devices found on this system " + "(devices must be unmounted, unheld, unpartitioned whole disks)") + return False, False + else: + nvmes = detect_nvmes(pci_allowed, pci_blocked, device_model, size_range, nvme_names) + if not nvmes: + logger.error( + "There are no enough SSD devices on system, you may run 'sbctl sn clean-devices', to clean devices stored in /etc/simplyblock/sn_config_file") + return False, False + if force and not lblk_mode: nvme_devices = " ".join([f"/dev/{d}n1" for d in nvmes.keys()]) logger.warning(f"Formating Nvme devices {nvme_devices}") answer = input("Type YES/Y to continue: ").strip().lower() @@ -1913,11 +2178,15 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a for nvme, val in nvmes.items(): pci = val["pci_address"] numa = int(val["numa_node"]) - pci_utils.unbind_driver(pci) + if not lblk_mode: + # lblk keeps the kernel driver — the AIO bdev needs the block + # device usable by the kernel, the exact opposite of DPDK claim. + pci_utils.unbind_driver(pci) + dev_ref = pci if not lblk_mode else nvme if numa in sockets_to_use: - system_info[numa]["nvmes"].append(pci) + system_info[numa]["nvmes"].append(dev_ref) else: - system_info.setdefault(numa, {"cores": [], "nics": [], "nvmes": []})["nvmes"].append(pci) + system_info.setdefault(numa, {"cores": [], "nics": [], "nvmes": []})["nvmes"].append(dev_ref) nvme_by_numa: dict = {nid: [] for nid in sockets_to_use} nvme_numa_neg1 = [] @@ -1984,11 +2253,18 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a node_info["number_of_distribs"] = number_of_distribs nvme_neg1_list = all_nvmes_neg1_per_node[node_index] - for nvme_name in nvme_neg1_list: - node_info["ssd_pcis"].append(nvmes[nvme_name]["pci_address"]) - for nvme_name in nvme_per_core_group[idx]: - node_info["ssd_pcis"].append(nvmes[nvme_name]["pci_address"]) - number_of_alcemls = len(node_info["ssd_pcis"]) + if lblk_mode: + node_info["lblk_devices"] = [] + for dev_name in nvme_neg1_list: + node_info["lblk_devices"].append(lblk_entries[dev_name]) + for dev_name in nvme_per_core_group[idx]: + node_info["lblk_devices"].append(lblk_entries[dev_name]) + else: + for nvme_name in nvme_neg1_list: + node_info["ssd_pcis"].append(nvmes[nvme_name]["pci_address"]) + for nvme_name in nvme_per_core_group[idx]: + node_info["ssd_pcis"].append(nvmes[nvme_name]["pci_address"]) + number_of_alcemls = node_config_device_count(node_info) node_info["number_of_alcemls"] = number_of_alcemls small_pool_count, large_pool_count = calculate_pool_count(number_of_alcemls, 2 * number_of_distribs, len(core_group["isolated"]), @@ -2002,7 +2278,7 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a node_info["max_lvol"] = max_lvol node_info["max_size"] = max_prov node_info["huge_page_memory"] = max(minimum_hp_memory, max_prov) - minimum_sys_memory = calculate_minimum_sys_memory(node_info["ssd_pcis"]) + minimum_sys_memory = node_config_min_sys_memory(node_info) node_info["sys_memory"] = minimum_sys_memory all_nodes.append(node_info) node_index += 1 @@ -2013,7 +2289,7 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a total_required_memory = 0 all_isolated_cores = set() for node in all_nodes: - if len(node["ssd_pcis"]) == 0: + if node_config_device_count(node) == 0: logger.error(f"There are no enough SSD devices on numa node {node['socket']}") return False, False total_required_memory += node["huge_page_memory"] + node["sys_memory"] @@ -2251,12 +2527,31 @@ def validate_node_config(node): logger.error(f"Missing required distribution field '{field}' in node: {node.get('socket')}") return False + # Exactly one device source: PCIe SSDs (nvme mode) or Linux block + # devices (lblk mode). Both empty, or both populated, is a broken config. + lblk_devices = node.get("lblk_devices") or [] + if bool(node["ssd_pcis"]) == bool(lblk_devices): + logger.error( + f"Node config must carry exactly one non-empty device list of " + f"'ssd_pcis' / 'lblk_devices' in node: {node.get('socket')}") + return False + # Check ssd_pcis fields for ssd in node["ssd_pcis"]: if not is_valid_pci_address(ssd): logger.error(f"Missing required SSD field '{ssd}' in node: {node.get('socket')}") return False + # Check lblk_devices entries (manually editable — validate shape). + for entry in lblk_devices: + if not isinstance(entry, dict) or not entry.get("name") or not entry.get("serial"): + logger.error(f"lblk_devices entry missing 'name'/'serial' in node: {node.get('socket')}") + return False + if not isinstance(entry.get("size"), int) or entry["size"] <= 0: + logger.error(f"lblk_devices entry '{entry.get('name')}' needs a positive integer " + f"'size' in node: {node.get('socket')}") + return False + if not node["isolated"]: logger.error(f"'isolated' list is empty in node: {node.get('socket')}") return False diff --git a/simplyblock_web/api/internal/storage_node/docker.py b/simplyblock_web/api/internal/storage_node/docker.py index d8fa0018fb..efd7581f8a 100644 --- a/simplyblock_web/api/internal/storage_node/docker.py +++ b/simplyblock_web/api/internal/storage_node/docker.py @@ -303,6 +303,70 @@ def _remove_one(container): return utils.get_response(True) +@api.get('/spdk_process_cleanup', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'boolean' + })}}}, +}) +def spdk_process_cleanup(query: utils.RPCPortParams): + """Synchronous, resurrection-proof teardown of ``spdk_`` and its + proxy, VERIFIED at the container level. + + spdk_process_kill is deliberately fast (detached remove) for the peer- + termination paths — but that leaves two gaps for the add-node/restart + FAILURE cleanup: the containers run with a restart policy that can + resurrect them after the SIGKILL if the detached remove loses the race + against a loaded dockerd, and spdk_process_is_up probes the RPC Unix + socket, so an SPDK that never brought its RPC up reads as "down" while + its container lives on holding all hugepages (2026-08-05 incident: the + zombie starved every add-node retry on the host). This endpoint is the + slow, authoritative sibling: disable the restart policy first, remove + synchronously, and only report success when the containers are GONE. + """ + from docker.errors import NotFound + + client = get_docker_client() + names = [f"/spdk_{query.rpc_port}", f"/spdk_proxy_{query.rpc_port}"] + ok = True + for name in names: + try: + container = client.containers.get(name) + except NotFound: + continue + except Exception as exc: + logger.error("cleanup: resolving %s failed: %s", name, exc) + ok = False + continue + try: + # No restart policy => dockerd cannot resurrect it between the + # kill and the (synchronous) remove below. + client.api.update_container(container.id, + restart_policy={"Name": "no"}) + except Exception as exc: + logger.warning("cleanup: clearing restart policy on %s failed: %s", + container.id[:12], exc) + try: + container.remove(force=True) + except NotFound: + pass + except Exception as exc: + logger.error("cleanup: remove(%s) failed: %s", container.id[:12], exc) + ok = False + # Verification: success means the names resolve to nothing. + for name in names: + try: + client.containers.get(name) + ok = False + logger.error("cleanup: %s still present after remove", name) + except NotFound: + pass + except Exception: + ok = False + if not ok: + return utils.get_response(None, "spdk container cleanup incomplete") + return utils.get_response(True) + + # Tight client timeout for the dockerd fall-through in spdk_process_is_up. # The docker-py default is 60s, which under post-outage Swarm reconciliation # (incident 2026-04-24, vm205) caused this endpoint to take 76-80s. The @@ -473,6 +537,36 @@ def get_node_lsblk(): return data +@api.get('/blockdevices', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'array', + 'items': {'type': 'object', 'additionalProperties': True}, + })}}}, +}) +def get_blockdevices(): + """Whole-disk inventory for the lblk cluster mode (eligibility fields, + serial/WWN identity, by-id path, NUMA).""" + return utils.get_response(node_utils.get_block_devices_info()) + + +class _WipeBlockDeviceParams(BaseModel): + device_name: str + + +@api.post('/wipe_block_device', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'boolean' + })}}}, +}) +def wipe_block_device(body: _WipeBlockDeviceParams): + """--force-format for lblk add-node: wipe partition/FS signatures from a + whole disk. Refuses busy devices (mounts/holders/root disk).""" + ok, reason = node_utils.wipe_block_device_signatures(body.device_name) + if not ok: + return utils.get_response(None, reason) + return utils.get_response(True) + + def get_nodes_config(): logger.debug("function:get_nodes_config start") file_path = constants.NODES_CONFIG_FILE diff --git a/simplyblock_web/api/internal/storage_node/kubernetes.py b/simplyblock_web/api/internal/storage_node/kubernetes.py index 7845c2c69a..67bc40709d 100644 --- a/simplyblock_web/api/internal/storage_node/kubernetes.py +++ b/simplyblock_web/api/internal/storage_node/kubernetes.py @@ -138,6 +138,36 @@ def get_info(): }) +@api.get('/blockdevices', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'array', + 'items': {'type': 'object', 'additionalProperties': True}, + })}}}, +}) +def get_blockdevices(): + """Whole-disk inventory for the lblk cluster mode (eligibility fields, + serial/WWN identity, by-id path, NUMA).""" + return utils.get_response(node_utils.get_block_devices_info()) + + +class _WipeBlockDeviceParams(BaseModel): + device_name: str + + +@api.post('/wipe_block_device', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'boolean' + })}}}, +}) +def wipe_block_device(body: _WipeBlockDeviceParams): + """--force-format for lblk add-node: wipe partition/FS signatures from a + whole disk. Refuses busy devices (mounts/holders/root disk).""" + ok, reason = node_utils.wipe_block_device_signatures(body.device_name) + if not ok: + return utils.get_response(None, reason) + return utils.get_response(True) + + @api.post('/join_swarm', responses={ 200: {'content': {'application/json': {'schema': utils.response_schema({ 'type': 'boolean' @@ -597,6 +627,20 @@ def spdk_process_kill(query: utils.RPCPortParams): return utils.get_response(True) +@api.get('/spdk_process_cleanup', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'boolean' + })}}}, +}) +def spdk_process_cleanup(query: utils.RPCPortParams): + """Authoritative SPDK teardown for failure-cleanup paths. Pod deletion in + this deployment mode is already synchronous and verified (see + spdk_process_kill's poll-until-gone), so this is an alias kept for parity + with the docker agent, where kill (fast, detached remove) and cleanup + (slow, verified remove) are distinct.""" + return spdk_process_kill(query) + + def _is_pod_up(rpc_port, cluster_id): k8s_core_v1 = core_utils.get_k8s_core_client() pod_name = f"snode-spdk-pod-{rpc_port}-{cluster_id}" diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index 0e40fbf1d4..5c1d663c4d 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -112,6 +112,7 @@ class ClusterDTO(BaseModel): node_affinity: bool anti_affinity: bool enable_failure_domain: bool + device_mode: str secret: SecretStr tls_enabled: bool max_fault_tolerance: int @@ -141,6 +142,7 @@ def from_model(model: Cluster, stat_obj: Optional[StatsObject] = None): node_affinity=model.enable_node_affinity, anti_affinity=model.strict_node_anti_affinity, enable_failure_domain=model.enable_failure_domain, + device_mode=model.device_mode, secret=model.secret, tls_enabled=model.tls, max_fault_tolerance=model.max_fault_tolerance, @@ -159,6 +161,8 @@ class DeviceDTO(BaseModel): serial_number: str nvme_controller: str pcie_address: str + bdev_type: str = "nvme" + device_path: str = "" status: str # None => health check not applicable (owning node not ONLINE/DOWN) health_check: Optional[bool] @@ -182,6 +186,8 @@ def from_model(model: NVMeDevice, storage_node_id: str, stat_obj: Optional[Stats serial_number=model.serial_number, nvme_controller=model.nvme_controller, pcie_address=model.pcie_address, + bdev_type=model.bdev_type, + device_path=model.device_path, status=model.status, health_check=model.health_check, retries_exhausted=model.retries_exhausted, diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index 3f9a9aabb5..96e220b4d2 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -85,6 +85,7 @@ class ClusterParams(BaseModel): backup_config: Optional[BackupConfigParams] = None hashicorp_vault_settings: Optional[HashicorpVaultSettings] = None enable_failure_domain: bool = False + device_mode: Literal["nvme", "lblk"] = "nvme" @api.get('/', name='clusters:list') diff --git a/simplyblock_web/api/v2/cluster/storage_node/__init__.py b/simplyblock_web/api/v2/cluster/storage_node/__init__.py index 1f68785187..3448b73518 100644 --- a/simplyblock_web/api/v2/cluster/storage_node/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_node/__init__.py @@ -55,6 +55,7 @@ class StorageNodeParams(BaseModel): spdk_sys_mem: Optional[str] = None failure_domain: Optional[int] = None expand: bool = False + force_format: bool = False @api.post('/', name='clusters:storage-nodes:create', status_code=201, responses={201: {"content": None}}) @@ -86,6 +87,7 @@ def add(request: Request, cluster: Cluster, parameters: StorageNodeParams, respo "spdk_sys_mem": parameters.spdk_sys_mem, "failure_domain": parameters.failure_domain, "expansion": parameters.expand, + "force_format": parameters.force_format, } ) if not task_id_or_false: diff --git a/simplyblock_web/node_configure.py b/simplyblock_web/node_configure.py index 42a13f05f5..1df3d113b5 100755 --- a/simplyblock_web/node_configure.py +++ b/simplyblock_web/node_configure.py @@ -158,6 +158,39 @@ def parse_arguments() -> argparse.Namespace: dest='nvme_names', required=False ) + parser.add_argument( + '--lblk', + help='Configure the node with Linux block devices (lblk cluster mode) instead of ' + 'NVMe PCIe devices: eligible unmounted, unheld, unpartitioned whole disks are ' + 'wrapped in SPDK AIO bdevs', + action='store_true', + dest='lblk', + required=False + ) + parser.add_argument( + '--blk-names', + help='Comma separated list of block device names to use, like sdb,sdc (requires --lblk)', + type=str, + default='', + dest='blk_names', + required=False + ) + parser.add_argument( + '--blk-names-exclude', + help='Comma separated list of block device names to exclude, like sda (requires --lblk)', + type=str, + default='', + dest='blk_names_exclude', + required=False + ) + parser.add_argument( + '--blk-serials', + help='Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk)', + type=str, + default='', + dest='blk_serials', + required=False + ) return parser.parse_args() @@ -194,6 +227,18 @@ def validate_arguments(args: argparse.Namespace) -> None: "pci-allowed and pci-blocked cannot be both specified" ) + use_lblk = bool(args.lblk or args.blk_names or args.blk_names_exclude or args.blk_serials) + if use_lblk and not args.lblk: + raise argparse.ArgumentError( + None, "--blk-names/--blk-names-exclude/--blk-serials require --lblk") + if use_lblk and (args.pci_allowed or args.pci_blocked or args.device_model + or args.size_range or args.nvme_names): + raise argparse.ArgumentError( + None, "--lblk cannot be combined with NVMe device selection options") + if sum([bool(args.blk_names), bool(args.blk_names_exclude), bool(args.blk_serials)]) > 1: + raise argparse.ArgumentError( + None, "Choose only one of --blk-names, --blk-names-exclude, --blk-serials") + max_prov = utils.parse_size(args.max_prov, assume_unit='G') if max_prov < 0: raise argparse.ArgumentError( @@ -254,6 +299,14 @@ def main() -> None: if args.nvme_names: nvme_names = [nvme_name.strip() for nvme_name in args.nvme_names.split(',') if nvme_name.strip()] + lblk_selection = None + if args.lblk: + lblk_selection = { + "names": [x.strip() for x in args.blk_names.split(',') if x.strip()] or None, + "names_exclude": [x.strip() for x in args.blk_names_exclude.split(',') if x.strip()] or None, + "serials": [x.strip() for x in args.blk_serials.split(',') if x.strip()] or None, + } + # Generate the deployment configuration generate_automated_deployment_config( max_lvol=int(args.max_lvol), @@ -267,7 +320,8 @@ def main() -> None: device_model=args.device_model, size_range=args.size_range, nvme_names=nvme_names, - k8s=True + k8s=True, + lblk_selection=lblk_selection ) except argparse.ArgumentError as e: diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index f902285983..9a4ebd40ae 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -148,6 +148,184 @@ def get_spdk_devices(): return [] +def _read_sysfs(path: str) -> str: + try: + with open(path, "r") as f: + return f.read().strip() + except OSError: + return "" + + +def _disk_holders(name: str) -> List[str]: + """Union of /sys/block//holders and every partition's holders — + catches LVM PVs, md members and dm-crypt without a mountpoint.""" + import os + holders: List[str] = [] + base = f"/sys/block/{name}" + try: + holders.extend(os.listdir(f"{base}/holders")) + except OSError: + pass + try: + for entry in os.listdir(base): + if entry.startswith(name): + try: + holders.extend(os.listdir(f"{base}/{entry}/holders")) + except OSError: + pass + except OSError: + pass + return sorted(set(holders)) + + +def _disk_by_id_path(name: str) -> str: + """Preferred stable /dev/disk/by-id symlink for a whole disk: wwn-* first, + then any other non-partition link. Empty when none exists.""" + import os + by_id_dir = "/dev/disk/by-id" + target = f"/dev/{name}" + candidates: List[str] = [] + try: + for entry in os.listdir(by_id_dir): + if "-part" in entry: + continue + path = os.path.join(by_id_dir, entry) + try: + if os.path.realpath(path) == target: + candidates.append(path) + except OSError: + continue + except OSError: + return "" + if not candidates: + return "" + candidates.sort(key=lambda p: (0 if "/wwn-" in p.replace("\\", "/") else 1, p)) + return candidates[0] + + +def _root_disk_names() -> List[str]: + """Kernel names of the disk(s) backing the root filesystem.""" + out, _, rc = shell_utils.run_command("findmnt -no SOURCE /") + if rc != 0 or not out.strip(): + return [] + source = out.strip().splitlines()[0] + # Walk PKNAME upwards (handles /dev/sda2, dm/LVM roots, etc.). + out, _, rc = shell_utils.run_command(f"lsblk -no PKNAME,NAME {source}") + names = set() + if rc == 0: + for line in out.splitlines(): + for token in line.split(): + names.add(token.strip()) + if source.startswith("/dev/"): + names.add(source[len("/dev/"):]) + return sorted(n for n in names if n) + + +def _subtree_mounted(dev: dict) -> bool: + if dev.get("mountpoint"): + return True + return any(_subtree_mounted(child) for child in dev.get("children") or []) + + +def get_block_devices_info() -> List[dict]: + """Inventory of whole-disk block devices for the lblk cluster mode. + + One dict per lsblk TYPE=disk entry, carrying everything the control + plane needs for eligibility filtering, identity (serial-first) and AIO + bdev creation. Sizes are bytes (lsblk -b). Serial falls back to WWN; + devices with neither get a synthetic-stable id derived from + hostname|by-id-or-name|size so identity survives reboots. + """ + import hashlib + import socket + + logger.debug("function:get_block_devices_info start") + out, err, rc = shell_utils.run_command( + "lsblk -J -b -o NAME,TYPE,SIZE,SERIAL,WWN,MOUNTPOINT,MODEL,ROTA,RO,VENDOR,PKNAME") + if rc != 0: + logger.error("Error running lsblk: %s", err) + return [] + try: + data = json.loads(out) + except json.JSONDecodeError as e: + logger.error("Failed to parse lsblk output: %s", e) + return [] + + root_disks = _root_disk_names() + hostname = socket.gethostname() + devices: List[dict] = [] + for dev in data.get("blockdevices", []): + if dev.get("type") != "disk": + continue + name = dev.get("name", "") + children = dev.get("children") or [] + by_id_path = _disk_by_id_path(name) + serial = (dev.get("serial") or "").strip() + wwn = (dev.get("wwn") or "").strip() + if not serial: + serial = wwn + synthetic = False + if not serial: + seed = f"{hostname}|{by_id_path or name}|{dev.get('size') or 0}" + serial = "SYN-" + hashlib.sha1(seed.encode()).hexdigest()[:16] + synthetic = True + devices.append({ + "name": name, + "device_path": f"/dev/{name}", + "type": dev.get("type"), + "size": int(dev.get("size") or 0), + "serial": serial, + "serial_synthetic": synthetic, + "wwn": wwn, + "model": (dev.get("model") or "").strip(), + "vendor": (dev.get("vendor") or "").strip(), + "rota": bool(dev.get("rota")), + "ro": bool(dev.get("ro")), + "has_partitions": any(c.get("type") == "part" for c in children), + "mounted_in_subtree": _subtree_mounted(dev), + "holders": _disk_holders(name), + "is_root_disk": name in root_disks, + "by_id_path": by_id_path, + "numa_node": int(_read_sysfs(f"/sys/block/{name}/device/numa_node") or -1), + }) + logger.debug("function:get_block_devices_info end") + return devices + + +def wipe_block_device_signatures(device_name: str) -> Tuple[bool, str]: + """Wipe partition-table / filesystem signatures from a whole disk + (`--force-format` on lblk add-node). Re-validates that the device is not + busy before touching it: any mountpoint in the subtree or any holder + refuses the wipe. Wipes partitions first, then the disk itself.""" + import re as _re + if not _re.match(r"^[a-zA-Z0-9_\-]+$", device_name): + return False, f"invalid device name {device_name!r}" + for dev in get_block_devices_info(): + if dev["name"] == device_name: + if dev["mounted_in_subtree"]: + return False, f"device {device_name} has mounted filesystems" + if dev["holders"]: + return False, (f"device {device_name} is held by " + f"{dev['holders']}") + if dev["is_root_disk"]: + return False, f"device {device_name} backs the root filesystem" + break + else: + return False, f"device {device_name} not found" + + out, _, rc = shell_utils.run_command( + f"lsblk -nro NAME -x NAME /dev/{device_name}") + if rc != 0: + return False, f"lsblk failed for {device_name}" + # Children (partitions) first, whole disk last. + names = [n for n in out.split() if n and n != device_name] + for name in names + [device_name]: + _, err, rc = shell_utils.run_command(f"wipefs -a /dev/{name}") + if rc != 0: + return False, f"wipefs /dev/{name} failed: {err}" + return True, "" + + def _get_mem_info(): logger.debug("function:_get_mem_info start") out, err, rc = shell_utils.run_command("cat /proc/meminfo") diff --git a/tests/integration/test_lblk_device_lifecycle.py b/tests/integration/test_lblk_device_lifecycle.py new file mode 100644 index 0000000000..725e2e789b --- /dev/null +++ b/tests/integration/test_lblk_device_lifecycle.py @@ -0,0 +1,358 @@ +# coding=utf-8 +"""Integration tests for the lblk (Linux block device / SPDK AIO) device +mode against a real FoundationDB (testcontainer via tests/integration/ +conftest.py). + +What runs REAL here: the FDB persistence layer (model round-trips), the +device_controller state machine (device_set_state flap accounting, forced +FAILED, device_remove) and the device_monitor watchdog logic. What is +faked: SPDK RPC (per-call mocks), the node agent (blockdevices inventory) +and the distr/event fan-out (patched at the consuming module). + +Scenarios: + 1. Model round-trip — cluster.device_mode, node.lblk_devices and the + per-device aio identity fields survive FDB serialization. + 2. Restart identity contract — resolve_lblk_entries + addAioDevices + against a renamed-device inventory produce records whose serials match + the DB reconcile keys (serial-first restart survival). + 3. Watchdog stall — real device_set_unavailable/io_error transitions in + FDB after the hung-IO threshold, with a countable flap. + 4. Flap limit — repeated LOCAL_FAILURE transitions force STATUS_FAILED + and queue failed-device migration. + 5. Disappearance — the presence sweep drives the real device_remove to + STATUS_REMOVED. + 6. reset_storage_device — aio liveness probe against a real DB record. +""" + +import uuid as uuid_mod +from unittest.mock import MagicMock, patch + +import pytest + +from simplyblock_core import constants, utils +from simplyblock_core.controllers import device_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import device_monitor + + +CLUSTER_ID = "11111111-1111-1111-1111-111111111111" + + +def _seed_cluster(db, device_mode="lblk", status=Cluster.STATUS_ACTIVE): + cluster = Cluster() + cluster.uuid = CLUSTER_ID + cluster.status = status + cluster.device_mode = device_mode + cluster.ha_type = "ha" + cluster.write_to_db(db.kv_store) + return cluster + + +def _aio_device(serial="S1", name="sdb", status=NVMeDevice.STATUS_ONLINE): + dev = NVMeDevice() + dev.uuid = str(uuid_mod.uuid4()) + dev.cluster_id = CLUSTER_ID + dev.status = status + dev.bdev_type = "aio" + dev.serial_number = serial + dev.device_name = name + dev.device_path = f"/dev/{name}" + dev.by_id_path = f"/dev/disk/by-id/wwn-{serial}" + dev.nvme_bdev = utils.aio_bdev_name_for_serial(serial) + dev.size = 100 << 30 + dev.cluster_device_order = 0 + return dev + + +def _seed_node(db, devices, node_id=None, status=StorageNode.STATUS_ONLINE): + node = StorageNode() + node.uuid = node_id or str(uuid_mod.uuid4()) + node.cluster_id = CLUSTER_ID + node.status = status + node.mgmt_ip = "10.0.0.1" + node.api_endpoint = "10.0.0.1:5000" + node.lblk_devices = [ + {"name": d.device_name, "serial": d.serial_number, + "by_id": d.by_id_path, "size": d.size, "numa": 0} + for d in devices + ] + for d in devices: + d.node_id = node.uuid + node.nvme_devices = devices + node.write_to_db(db.kv_store) + return node + + +@pytest.fixture() +def db(): + return DBController() + + +# --------------------------------------------------------------------------- +# 1. Model round-trips +# --------------------------------------------------------------------------- + +class TestModelRoundTrip: + + def test_cluster_device_mode_persists(self, db): + _seed_cluster(db, device_mode="lblk") + read = db.get_cluster_by_id(CLUSTER_ID) + assert read.device_mode == "lblk" + + def test_cluster_device_mode_defaults_nvme(self, db): + cluster = Cluster() + cluster.uuid = CLUSTER_ID + cluster.status = Cluster.STATUS_ACTIVE + cluster.write_to_db(db.kv_store) + assert db.get_cluster_by_id(CLUSTER_ID).device_mode == "nvme" + + def test_node_and_device_fields_persist(self, db): + _seed_cluster(db) + dev = _aio_device(serial="S3Z8NX0M600123", name="sdb") + node = _seed_node(db, [dev]) + + read_node = db.get_storage_node_by_id(node.get_id()) + assert read_node.lblk_devices == [{ + "name": "sdb", "serial": "S3Z8NX0M600123", + "by_id": "/dev/disk/by-id/wwn-S3Z8NX0M600123", + "size": 100 << 30, "numa": 0, + }] + read_dev = read_node.nvme_devices[0] + assert read_dev.bdev_type == "aio" + assert read_dev.device_path == "/dev/sdb" + assert read_dev.by_id_path == "/dev/disk/by-id/wwn-S3Z8NX0M600123" + assert read_dev.nvme_bdev == utils.aio_bdev_name_for_serial("S3Z8NX0M600123") + assert read_dev.pcie_address == "" + assert read_dev.nvme_controller == "" + + def test_nvme_device_records_unaffected(self, db): + _seed_cluster(db, device_mode="nvme") + dev = NVMeDevice() + dev.uuid = str(uuid_mod.uuid4()) + dev.cluster_id = CLUSTER_ID + dev.status = NVMeDevice.STATUS_ONLINE + dev.pcie_address = "0000:00:1e.0" + dev.nvme_controller = "nvme_1e" + node = _seed_node(db, [dev]) + read_dev = db.get_storage_node_by_id(node.get_id()).nvme_devices[0] + assert read_dev.bdev_type == "nvme" + assert read_dev.pcie_address == "0000:00:1e.0" + + +# --------------------------------------------------------------------------- +# 2. Restart identity contract (serial-first over renamed devices) +# --------------------------------------------------------------------------- + +class TestRestartIdentityContract: + + def test_renamed_devices_resolve_to_same_reconcile_keys(self, db): + _seed_cluster(db) + d1, d2 = _aio_device("S1", "sdb"), _aio_device("S2", "sdc") + node = _seed_node(db, [d1, d2]) + node = db.get_storage_node_by_id(node.get_id()) + + # Reboot renamed sdb->sdd and sdc->sdb (a swap-adjacent shuffle). + live_inventory = [ + {"name": "sdd", "device_path": "/dev/sdd", "serial": "S1", + "by_id_path": "/dev/disk/by-id/wwn-S1", "size": 100 << 30, + "numa_node": 0, "model": "M"}, + {"name": "sdb", "device_path": "/dev/sdb", "serial": "S2", + "by_id_path": "/dev/disk/by-id/wwn-S2", "size": 100 << 30, + "numa_node": 0, "model": "M"}, + ] + resolved, missing = utils.resolve_lblk_entries(node.lblk_devices, live_inventory) + assert missing == [] + + rpc = MagicMock() + rpc.host = "t" + rpc.get_bdevs.return_value = None + created = {} + + def _create(name, filename, block_size=0): + created[name] = filename + rpc.get_bdevs.return_value = [ + {"name": name, "block_size": 4096, "num_blocks": 100}] + return name + + rpc.bdev_aio_create.side_effect = _create + discovered = utils.addAioDevices(rpc, node, resolved) + + # The reconcile at restart keys on serial_number: every discovered + # serial must match a DB record, with the CURRENT (renamed) path. + db_by_serial = {d.serial_number: d for d in node.nvme_devices} + for found in discovered: + assert found.serial_number in db_by_serial + by_serial = {d.serial_number: d for d in discovered} + assert by_serial["S1"].device_name == "sdd" + assert by_serial["S2"].device_name == "sdb" + # Stable bdev names: identical to what add-node created. + assert by_serial["S1"].nvme_bdev == db_by_serial["S1"].nvme_bdev + # AIO filename used the stable by-id path, not the volatile name. + assert created[by_serial["S1"].nvme_bdev] == "/dev/disk/by-id/wwn-S1" + + def test_missing_device_flagged_for_removal_semantics(self, db): + _seed_cluster(db) + node = _seed_node(db, [_aio_device("S1", "sdb"), _aio_device("S2", "sdc")]) + node = db.get_storage_node_by_id(node.get_id()) + live_inventory = [ + {"name": "sdb", "device_path": "/dev/sdb", "serial": "S1", + "by_id_path": "", "size": 1, "numa_node": 0, "model": "M"}, + ] + resolved, missing = utils.resolve_lblk_entries(node.lblk_devices, live_inventory) + assert [e["serial"] for e in resolved] == ["S1"] + assert [e["serial"] for e in missing] == ["S2"] + + +# --------------------------------------------------------------------------- +# 3-5. Watchdog + real device_controller state machine +# --------------------------------------------------------------------------- + +def _patched_fanout(): + """Patch the SPDK/event fan-out that device_set_state / device_remove + perform, leaving the FDB state machine real.""" + return [ + patch.object(device_controller, "distr_controller", MagicMock()), + patch.object(device_controller, "device_events", MagicMock()), + patch.object(StorageNode, "rpc_client", + lambda self, **kw: MagicMock()), + ] + + +class TestWatchdogAgainstRealStateMachine: + + def setup_method(self, _method): + device_monitor._aio_progress.clear() + device_monitor._aio_absent.clear() + + def test_stall_marks_device_unavailable_with_flap(self, db): + _seed_cluster(db) + dev = _aio_device("S1", "sdb") + node = _seed_node(db, [dev]) + node = db.get_storage_node_by_id(node.get_id()) + + stall_rpc = MagicMock() + stall_rpc.get_lvol_stats.return_value = {"bdevs": [{ + "num_read_ops": 100, "num_write_ops": 0, "num_unmap_ops": 0, + "queue_depth": 4}]} + inventory = [{"name": "sdb", "serial": "S1"}] + agent = MagicMock() + agent.get_blockdevices.return_value = (inventory, None) + + patches = _patched_fanout() + [ + patch.object(StorageNode, "client", lambda self, **kw: agent), + ] + for p in patches: + p.start() + try: + with patch.object(StorageNode, "rpc_client", + lambda self, **kw: stall_rpc): + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS + 1): + node = db.get_storage_node_by_id(node.get_id()) + device_monitor._sweep_aio_devices(node) + finally: + for p in patches: + p.stop() + + read = db.get_storage_device_by_id(dev.get_id()) + assert read.status == NVMeDevice.STATUS_UNAVAILABLE + assert read.io_error is True + assert read.flap_count == 1 # ONLINE -> UNAVAILABLE, LOCAL_FAILURE, node ONLINE + + def test_flap_limit_forces_failed_and_queues_migration(self, db): + _seed_cluster(db) + dev = _aio_device("S1", "sdb") + _seed_node(db, [dev]) + + patches = _patched_fanout() + [ + patch.object(device_controller, "DEVICE_FLAP_DEBOUNCE_SEC", 0.0), + # re-online between flaps queues FN_DEV_MIG — irrelevant noise here + patch.object(device_controller.tasks_controller, + "add_device_mig_task_for_node", return_value=None), + patch.object(device_controller.tasks_controller, + "add_device_failed_mig_task"), + ] + started = [p.start() for p in patches] + mig_task = started[-1] + try: + for _ in range(device_controller.DEVICE_FLAP_LIMIT + 1): + device_controller.device_set_unavailable( + dev.get_id(), cause=device_controller.CAUSE_LOCAL_FAILURE) + read = db.get_storage_device_by_id(dev.get_id()) + if read.status == NVMeDevice.STATUS_FAILED: + break + device_controller.device_set_online(dev.get_id()) + finally: + for p in patches: + p.stop() + + read = db.get_storage_device_by_id(dev.get_id()) + assert read.status == NVMeDevice.STATUS_FAILED + mig_task.assert_called_once_with(dev.get_id()) + + def test_disappearance_drives_real_device_remove(self, db): + _seed_cluster(db) + dev = _aio_device("S1", "sdb") + node = _seed_node(db, [dev]) + + agent = MagicMock() + agent.get_blockdevices.return_value = ( + [{"name": "other", "serial": "ZZZ"}], None) + idle_rpc = MagicMock() + idle_rpc.get_lvol_stats.return_value = {"bdevs": [{ + "num_read_ops": 0, "num_write_ops": 0, "num_unmap_ops": 0, + "queue_depth": 0}]} + + patches = _patched_fanout() + [ + patch.object(StorageNode, "client", lambda self, **kw: agent), + ] + for p in patches: + p.start() + try: + with patch.object(StorageNode, "rpc_client", + lambda self, **kw: idle_rpc): + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS): + fresh = db.get_storage_node_by_id(node.get_id()) + device_monitor._sweep_aio_devices(fresh) + finally: + for p in patches: + p.stop() + + read = db.get_storage_device_by_id(dev.get_id()) + assert read.status == NVMeDevice.STATUS_REMOVED + + +# --------------------------------------------------------------------------- +# 6. reset_storage_device against a real DB record +# --------------------------------------------------------------------------- + +class TestResetAgainstDb: + + def test_reset_aio_liveness_probe_recovers_unavailable_device(self, db): + _seed_cluster(db) + dev = _aio_device("S1", "sdb", status=NVMeDevice.STATUS_UNAVAILABLE) + _seed_node(db, [dev]) + + rpc = MagicMock() + rpc.get_bdevs.return_value = [{"name": dev.nvme_bdev}] + + patches = _patched_fanout() + for p in patches: + p.start() + try: + with patch.object(StorageNode, "rpc_client", lambda self, **kw: rpc), \ + patch.object(device_controller.tasks_controller, + "get_active_dev_restart_task", return_value=None), \ + patch.object(device_controller.tasks_controller, + "add_device_mig_task_for_node", return_value=None): + assert device_controller.reset_storage_device(dev.get_id()) + finally: + for p in patches: + p.stop() + + read = db.get_storage_device_by_id(dev.get_id()) + assert read.status == NVMeDevice.STATUS_ONLINE + assert read.io_error is False + rpc.reset_device.assert_not_called() diff --git a/tests/unit/test_api_dto_secrets.py b/tests/unit/test_api_dto_secrets.py index 95e7685f1a..be07e95e33 100644 --- a/tests/unit/test_api_dto_secrets.py +++ b/tests/unit/test_api_dto_secrets.py @@ -52,6 +52,7 @@ def _build_cluster_dto(): node_affinity=False, anti_affinity=False, enable_failure_domain=False, + device_mode="nvme", secret=SecretStr("CLUSTER-SECRET"), tls_enabled=False, max_fault_tolerance=1, diff --git a/tests/unit/test_jm_mesh_and_spdk_cleanup.py b/tests/unit/test_jm_mesh_and_spdk_cleanup.py new file mode 100644 index 0000000000..079043ee17 --- /dev/null +++ b/tests/unit/test_jm_mesh_and_spdk_cleanup.py @@ -0,0 +1,199 @@ +# coding=utf-8 +"""Unit tests for the two 2026-08-05 incident fixes: + +1. verify_jm_mesh_coverage — the activation JM-mesh gate: every ONLINE + node must hold live remote bdevs for the remote JMs it references, + with owner-offline tolerance (re-activation with unhealthy nodes must + never be blocked) and a one-shot reconnect repair. + +2. _kill_spdk_until_dead — failure-path SPDK teardown must prefer the + container-level spdk_process_cleanup (verified-gone semantics) over + the RPC-socket liveness probe that false-negatives a booted-but- + RPC-dead SPDK (the hugepage-squatting zombie that starved add-node + retries). +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import storage_node_ops +from simplyblock_core.models.nvme_device import JMDevice, RemoteJMDevice +from simplyblock_core.models.storage_node import StorageNode + + +def _node(uuid, status=StorageNode.STATUS_ONLINE, enable_ha_jm=True, + jm_dev_id=None, jm_ids=(), remote_jms=()): + n = StorageNode() + n.uuid = uuid + n.status = status + n.enable_ha_jm = enable_ha_jm + n.cluster_id = "cluster-1" + if jm_dev_id: + jm = JMDevice() + jm.uuid = jm_dev_id + n.jm_device = jm + n.jm_ids = list(jm_ids) + n.remote_jm_devices = list(remote_jms) + return n + + +def _rjm(owner_id, remote_bdev): + r = RemoteJMDevice() + r.node_id = owner_id + r.remote_bdev = remote_bdev + r.jm_bdev = f"jm_{owner_id}" + return r + + +class TestJmMeshCoverage(unittest.TestCase): + + def _run(self, nodes, bdevs_by_node, repair=False, reconnect_result=None): + """bdevs_by_node: {node_uuid: set(bdev names present on that node)}""" + db = MagicMock() + db.get_storage_nodes_by_cluster_id.return_value = nodes + db.get_storage_node_by_id.side_effect = lambda nid: next( + n for n in nodes if n.get_id() == nid) + + def _rpc_for(node_self, **kw): + rpc = MagicMock() + present = bdevs_by_node.get(node_self.get_id(), set()) + rpc.get_bdevs.side_effect = lambda name: ( + [{"name": name}] if name in present else None) + return rpc + + patches = [ + patch.object(storage_node_ops, "DBController", return_value=db), + patch.object(StorageNode, "rpc_client", _rpc_for), + ] + if reconnect_result is not None: + patches.append(patch.object( + storage_node_ops, "_connect_to_remote_jm_devs", + return_value=reconnect_result)) + for p in patches: + p.start() + try: + return storage_node_ops.verify_jm_mesh_coverage("cluster-1", repair=repair) + finally: + for p in patches: + p.stop() + + def _two_nodes(self): + a = _node("node-a", jm_dev_id="jm-a", jm_ids=["jm-a", "jm-b"], + remote_jms=[_rjm("node-b", "remote_jm_node-bn1")]) + b = _node("node-b", jm_dev_id="jm-b", jm_ids=["jm-b", "jm-a"], + remote_jms=[_rjm("node-a", "remote_jm_node-an1")]) + return a, b + + def test_healthy_mesh(self): + a, b = self._two_nodes() + problems = self._run([a, b], { + "node-a": {"remote_jm_node-bn1"}, + "node-b": {"remote_jm_node-an1"}, + }) + self.assertEqual(problems, []) + + def test_missing_remote_bdev_reported(self): + a, b = self._two_nodes() + problems = self._run([a, b], { + "node-a": set(), # a cannot see b's JM + "node-b": {"remote_jm_node-an1"}, + }) + self.assertEqual(len(problems), 1) + self.assertIn("node-a", problems[0]) + self.assertIn("node-b", problems[0]) + + def test_missing_record_reported(self): + # node-a references jm-b in jm_ids but has NO remote record at all — + # the exact 2026-08-05 hole. + a = _node("node-a", jm_dev_id="jm-a", jm_ids=["jm-a", "jm-b"], + remote_jms=[]) + b = _node("node-b", jm_dev_id="jm-b", jm_ids=["jm-b"]) + problems = self._run([a, b], {"node-a": set(), "node-b": set()}) + self.assertTrue(any("node-a" in p and "node-b" in p for p in problems)) + + def test_offline_owner_tolerated(self): + # Re-activation rule: a JM whose owner is not ONLINE is skipped. + a, b = self._two_nodes() + b.status = StorageNode.STATUS_OFFLINE + problems = self._run([a, b], {"node-a": set()}) + self.assertEqual(problems, []) + + def test_offline_referencing_node_skipped(self): + a, b = self._two_nodes() + a.status = StorageNode.STATUS_OFFLINE + problems = self._run([a, b], { + "node-b": {"remote_jm_node-an1"}, + }) + self.assertEqual(problems, []) + + def test_repair_fixes_coverage(self): + a, b = self._two_nodes() + # Initially missing on node-a; after reconnect the returned record's + # bdev IS present. + problems = self._run( + [a, b], + {"node-a": {"remote_jm_node-bn1_new"}, + "node-b": {"remote_jm_node-an1"}}, + repair=True, + reconnect_result=[_rjm("node-b", "remote_jm_node-bn1_new")], + ) + self.assertEqual(problems, []) + + def test_repair_failure_still_reported(self): + a, b = self._two_nodes() + problems = self._run( + [a, b], + {"node-a": set(), "node-b": {"remote_jm_node-an1"}}, + repair=True, + reconnect_result=[_rjm("node-b", "remote_jm_node-bn1")], + ) + self.assertEqual(len(problems), 1) + + +class TestKillSpdkUntilDead(unittest.TestCase): + + def _snode(self): + snode = MagicMock() + snode.get_id.return_value = "node-1" + snode.rpc_port = 4423 + snode.cluster_id = "cluster-1" + snode.mgmt_ip = "10.0.0.1" + return snode + + def test_cleanup_success_short_circuits(self): + snode = self._snode() + api = snode.client.return_value + api.spdk_process_cleanup.return_value = (True, None) + self.assertTrue(storage_node_ops._kill_spdk_until_dead(snode)) + api.spdk_process_cleanup.assert_called_once_with(4423, "cluster-1") + api.spdk_process_kill.assert_not_called() + + def test_cleanup_unavailable_falls_back_to_kill(self): + snode = self._snode() + api = snode.client.return_value + api.spdk_process_cleanup.side_effect = Exception("404 not found") + api.spdk_process_is_up.return_value = (False, None) + self.assertTrue(storage_node_ops._kill_spdk_until_dead( + snode, max_attempts=1, poll_per_attempt_sec=1)) + api.spdk_process_kill.assert_called_once() + + def test_cleanup_incomplete_falls_back(self): + snode = self._snode() + api = snode.client.return_value + api.spdk_process_cleanup.return_value = (None, "cleanup incomplete") + api.spdk_process_is_up.return_value = (False, None) + self.assertTrue(storage_node_ops._kill_spdk_until_dead( + snode, max_attempts=1, poll_per_attempt_sec=1)) + api.spdk_process_kill.assert_called_once() + + def test_all_paths_fail_returns_false(self): + snode = self._snode() + api = snode.client.return_value + api.spdk_process_cleanup.return_value = (None, "nope") + api.spdk_process_is_up.return_value = (True, None) + self.assertFalse(storage_node_ops._kill_spdk_until_dead( + snode, max_attempts=1, poll_per_attempt_sec=0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_device_controller.py b/tests/unit/test_lblk_device_controller.py new file mode 100644 index 0000000000..50b0da2fd2 --- /dev/null +++ b/tests/unit/test_lblk_device_controller.py @@ -0,0 +1,263 @@ +# coding=utf-8 +"""Unit tests for the lblk (aio) branches in controllers/device_controller.py +and the mode-aware late-event gate in services/main_distr_event_collector.py. + +Covered: + - reset_storage_device: aio liveness-probe semantics — bdev present clears + the error state (no nvme controller reset issued); bdev gone returns + False so the tasks framework escalates to restart_device. + - get_device_health_info: aio SMART stub (never calls the nvme RPC). + - new_device_from_failed: aio path recreates the AIO bdev serial-first + from the live inventory instead of bind_device_to_spdk + controller + attach. + - restart_device: aio path recreates the missing AIO bdev (with qd + sampling re-armed) instead of the PCIe attach sequence. + - late-event gate: for aio devices the "controller gone?" probe is + get_bdevs_2 on the base bdev; a present bdev skips the late event. +""" + +import json +import unittest +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +from simplyblock_core.controllers import device_controller +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.services import main_distr_event_collector as collector + + +def _aio_dev(uid="dev-1", status=NVMeDevice.STATUS_ONLINE): + d = NVMeDevice() + d.uuid = uid + d.node_id = "node-1" + d.cluster_id = "cluster-1" + d.status = status + d.bdev_type = "aio" + d.serial_number = "S1" + d.nvme_bdev = "aio_S1" + d.device_path = "/dev/sdb" + d.by_id_path = "/dev/disk/by-id/wwn-1" + return d + + +class TestResetStorageDeviceAio(unittest.TestCase): + + def _run(self, bdev_present): + device = _aio_dev(status=NVMeDevice.STATUS_UNAVAILABLE) + snode = MagicMock() + snode.cluster_id = "cluster-1" + rpc = MagicMock() + rpc.get_bdevs.return_value = [{"name": "aio_S1"}] if bdev_present else None + snode.rpc_client.return_value = rpc + + db = MagicMock() + db.get_storage_device_by_id.return_value = device + db.get_storage_node_by_id.return_value = snode + + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(device_controller.tasks_controller, + "get_active_dev_restart_task", return_value=None), \ + patch.object(device_controller, "device_set_unavailable") as set_unavail, \ + patch.object(device_controller, "device_set_io_error") as set_io_err, \ + patch.object(device_controller, "device_set_retries_exhausted") as set_retries, \ + patch.object(device_controller, "device_set_online") as set_online, \ + patch.object(device_controller, "device_events"): + result = device_controller.reset_storage_device("dev-1") + return result, rpc, set_unavail, set_io_err, set_retries, set_online + + def test_bdev_present_clears_error_state(self): + result, rpc, _, set_io_err, set_retries, set_online = self._run(True) + self.assertTrue(result) + set_io_err.assert_called_once_with("dev-1", False) + set_retries.assert_called_once_with("dev-1", False) + set_online.assert_called_once() + rpc.reset_device.assert_not_called() + + def test_bdev_gone_fails_for_escalation(self): + result, rpc, _, set_io_err, _, set_online = self._run(False) + self.assertFalse(result) + set_io_err.assert_not_called() + set_online.assert_not_called() + rpc.reset_device.assert_not_called() + + +class TestHealthInfoAio(unittest.TestCase): + + def test_aio_returns_stub_without_nvme_rpc(self): + device = _aio_dev() + snode = MagicMock() + db = MagicMock() + db.get_storage_device_by_id.return_value = device + db.get_storage_node_by_id.return_value = snode + with patch.object(device_controller, "DBController", return_value=db): + ret = device_controller.get_device_health_info("dev-1") + data = json.loads(ret) + self.assertEqual(data["bdev_type"], "aio") + self.assertIsNone(data["smart"]) + snode.rpc_client.assert_not_called() + + +class TestNewDeviceFromFailedAio(unittest.TestCase): + + def _run(self, bdev_present_initially, inventory=None, create_ok=True): + device = _aio_dev(status=NVMeDevice.STATUS_FAILED_AND_MIGRATED) + node = MagicMock() + node.get_id.return_value = "node-1" + node.nvme_devices = [device] + + rpc = MagicMock() + state = {"present": bdev_present_initially} + + def _get_bdevs(name): + return [{"name": name}] if state["present"] else None + + def _aio_create(name, filename, block_size=0): + if create_ok: + state["present"] = True + return name + return None + + rpc.get_bdevs.side_effect = _get_bdevs + rpc.bdev_aio_create.side_effect = _aio_create + node.rpc_client.return_value = rpc + + client = MagicMock() + client.get_blockdevices.return_value = (inventory or [], None) + node.client.return_value = client + + db = MagicMock() + db.get_storage_nodes.return_value = [node] + with patch.object(device_controller, "DBController", return_value=db): + result = device_controller.new_device_from_failed("dev-1") + return result, rpc, db + + def test_bdev_already_present_no_create(self): + result, rpc, db = self._run(True) + self.assertTrue(result) + rpc.bdev_aio_create.assert_not_called() + db.atomic_update.assert_called_once() + + def test_recreates_bdev_serial_first_from_inventory(self): + inventory = [{"name": "sdx", "serial": "S1", + "device_path": "/dev/sdx", + "by_id_path": "/dev/disk/by-id/wwn-NEW"}] + result, rpc, _ = self._run(False, inventory=inventory) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-NEW") + rpc.bdev_set_qd_sampling_period.assert_called_once() + + def test_falls_back_to_stored_path_when_inventory_empty(self): + result, rpc, _ = self._run(False, inventory=[]) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1") + + def test_create_failure_returns_false(self): + result, _, db = self._run(False, inventory=[], create_ok=False) + self.assertFalse(result) + db.atomic_update.assert_not_called() + + +class TestRestartDeviceAio(unittest.TestCase): + + def _run(self, bdev_present): + device = _aio_dev(status=NVMeDevice.STATUS_REMOVED) + device.nvmf_nqn = "" + device.alceml_bdev = "" + snode = MagicMock() + snode.cluster_id = "cluster-1" + snode.nvme_devices = [device] + snode.jm_device = None + + rpc = MagicMock() + state = {"present": bdev_present} + rpc.get_bdevs.side_effect = ( + lambda name: [{"name": name}] if state["present"] else None) + + def _aio_create(name, filename, block_size=0): + state["present"] = True + return name + + rpc.bdev_aio_create.side_effect = _aio_create + snode.rpc_client.return_value = rpc + + client = MagicMock() + client.get_blockdevices.return_value = ([], None) + snode.client.return_value = client + + db = MagicMock() + db.get_storage_device_by_id.return_value = device + db.get_storage_node_by_id.return_value = snode + + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(device_controller.tasks_controller, + "get_active_dev_restart_task", return_value=None), \ + patch.object(device_controller, "device_set_retries_exhausted"), \ + patch.object(device_controller, "device_set_unavailable"), \ + patch.object(device_controller, "_def_create_device_stack", + return_value=True) as create_stack, \ + patch.object(device_controller, "device_set_io_error") as set_io_err, \ + patch.object(device_controller, "device_set_online") as set_online, \ + patch.object(device_controller, "device_events"): + result = device_controller.restart_device("dev-1") + return result, rpc, create_stack, set_io_err, set_online + + def test_missing_aio_bdev_recreated_before_stack(self): + result, rpc, create_stack, set_io_err, set_online = self._run(False) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1") + rpc.bdev_set_qd_sampling_period.assert_called_once() + create_stack.assert_called_once() + set_io_err.assert_called_once_with("dev-1", False) + set_online.assert_called_once() + # never the nvme path + rpc.bdev_nvme_controller_attach.assert_not_called() + + def test_present_aio_bdev_not_recreated(self): + result, rpc, create_stack, _, _ = self._run(True) + self.assertTrue(result) + rpc.bdev_aio_create.assert_not_called() + create_stack.assert_called_once() + + +class TestLateEventGateAio(unittest.TestCase): + + def test_present_aio_bdev_skips_late_event(self): + device = _aio_dev() + device.cluster_device_order = 7 + + home_node = MagicMock() + home_node.get_id.return_value = "node-1" + home_node.nvme_devices = [device] + + event_node = MagicMock() + event_node.get_id.return_value = "node-2" + rpc = MagicMock() + rpc.get_bdevs_2.return_value = ([{"name": "aio_S1"}], None) + event_node.rpc_client.return_value = rpc + + event = MagicMock() + event.message = "error_read" + event.node_id = "node-2" + event.storage_id = 7 + stale = datetime.now() - timedelta(seconds=30) + event.object_dict = {"timestamp": stale.strftime('%Y-%m-%dT%H:%M:%S.%fZ')} + + db = MagicMock() + db.get_storage_node_by_id.return_value = event_node + db.get_storage_nodes.return_value = [home_node] + + with patch.object(collector, "db", db), \ + patch.object(collector, "_is_target_remote_controller_healthy", + return_value=False): + collector.process_device_event(event, collector.logger) + + rpc.get_bdevs_2.assert_called_once_with("aio_S1") + rpc.bdev_nvme_controller_list_2.assert_not_called() + self.assertIn("skipping", event.status) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_eligibility.py b/tests/unit/test_lblk_eligibility.py new file mode 100644 index 0000000000..f1fc1db984 --- /dev/null +++ b/tests/unit/test_lblk_eligibility.py @@ -0,0 +1,333 @@ +# coding=utf-8 +"""Unit tests for lblk-mode device eligibility, detection, identity and +node-config schema (pure helpers in simplyblock_core.utils). + +Covered: + - filter_eligible_block_devices: every rejection reason, all three + selection methods (names / names-exclude / serials), hard errors on + requested-but-ineligible devices and duplicate serials, force_format. + - detect_lblk_devices: config-entry mapping + synthetic-serial warning. + - aio_bdev_name_for_serial: stability, sanitization, collision-freedom. + - resolve_lblk_entries: serial-first resolution (rename survival), stored + name fallback, missing devices, field refresh. + - node_config_device_count / node_config_min_sys_memory. + - validate_node_config: exactly-one-device-source rule + lblk entry shape. +""" + +import unittest +from unittest.mock import patch + +from simplyblock_core import utils + + +def _blk(name, serial="", size=100 << 30, mounted=False, holders=None, + root=False, ro=False, parts=False, dtype="disk", by_id="", + numa=0, synthetic=False, model="MODEL-X", wwn=""): + return { + "name": name, + "device_path": f"/dev/{name}", + "type": dtype, + "size": size, + "serial": serial or f"SER-{name}", + "serial_synthetic": synthetic, + "wwn": wwn, + "model": model, + "vendor": "ACME", + "rota": False, + "ro": ro, + "has_partitions": parts, + "mounted_in_subtree": mounted, + "holders": holders or [], + "is_root_disk": root, + "by_id_path": by_id, + "numa_node": numa, + } + + +class TestEligibility(unittest.TestCase): + + def _reasons(self, devs, **kwargs): + _, rejected = utils.filter_eligible_block_devices(devs, **kwargs) + return {d["name"]: r for d, r in rejected} + + def test_clean_disk_is_eligible(self): + sel, rej = utils.filter_eligible_block_devices([_blk("sdb")]) + self.assertEqual([d["name"] for d in sel], ["sdb"]) + self.assertEqual(rej, []) + + def test_partition_type_rejected(self): + reasons = self._reasons([_blk("sdb1", dtype="part")]) + self.assertIn("not a whole disk", reasons["sdb1"]) + + def test_special_prefixes_rejected(self): + for name in ("ram0", "loop3", "sr0", "zram1", "nbd0", "md127", "dm-0", "drbd0", "fd0"): + reasons = self._reasons([_blk(name, serial=f"S-{name}")]) + self.assertIn("special", reasons[name], name) + + def test_mounted_subtree_rejected(self): + reasons = self._reasons([_blk("sdb", mounted=True)]) + self.assertIn("busy", reasons["sdb"]) + + def test_holders_rejected(self): + reasons = self._reasons([_blk("sdb", holders=["dm-0"])]) + self.assertIn("held by", reasons["sdb"]) + + def test_root_disk_rejected(self): + reasons = self._reasons([_blk("sda", root=True)]) + self.assertIn("root", reasons["sda"]) + + def test_read_only_rejected(self): + reasons = self._reasons([_blk("sdb", ro=True)]) + self.assertIn("read-only", reasons["sdb"]) + + def test_zero_size_rejected(self): + reasons = self._reasons([_blk("sdb", size=0)]) + self.assertIn("zero size", reasons["sdb"]) + + def test_partitioned_rejected_without_force(self): + reasons = self._reasons([_blk("sdb", parts=True)]) + self.assertIn("partitioned", reasons["sdb"]) + + def test_partitioned_eligible_with_force(self): + sel, _ = utils.filter_eligible_block_devices( + [_blk("sdb", parts=True)], force_format=True) + self.assertEqual([d["name"] for d in sel], ["sdb"]) + + def test_nvme_kernel_devices_remain_eligible(self): + # "arbitrary Linux block devices" includes kernel-driver NVMe disks + sel, _ = utils.filter_eligible_block_devices([_blk("nvme0n1")]) + self.assertEqual([d["name"] for d in sel], ["nvme0n1"]) + + # --- selection methods ------------------------------------------------ + + def test_include_names_selects_only_requested(self): + devs = [_blk("sdb"), _blk("sdc"), _blk("sdd")] + sel, _ = utils.filter_eligible_block_devices(devs, include_names=["sdb", "sdd"]) + self.assertEqual(sorted(d["name"] for d in sel), ["sdb", "sdd"]) + + def test_include_names_busy_device_is_hard_error(self): + devs = [_blk("sdb", mounted=True)] + with self.assertRaises(ValueError) as ctx: + utils.filter_eligible_block_devices(devs, include_names=["sdb"]) + self.assertIn("busy", str(ctx.exception)) + + def test_include_names_absent_device_is_hard_error(self): + with self.assertRaises(ValueError) as ctx: + utils.filter_eligible_block_devices([_blk("sdb")], include_names=["sdz"]) + self.assertIn("not present", str(ctx.exception)) + + def test_exclude_names(self): + devs = [_blk("sdb"), _blk("sdc")] + sel, _ = utils.filter_eligible_block_devices(devs, exclude_names=["sdb"]) + self.assertEqual([d["name"] for d in sel], ["sdc"]) + + def test_include_serials(self): + devs = [_blk("sdb", serial="S1"), _blk("sdc", serial="S2")] + sel, _ = utils.filter_eligible_block_devices(devs, include_serials=["S2"]) + self.assertEqual([d["name"] for d in sel], ["sdc"]) + + def test_include_serials_missing_is_hard_error(self): + with self.assertRaises(ValueError) as ctx: + utils.filter_eligible_block_devices( + [_blk("sdb", serial="S1")], include_serials=["S9"]) + self.assertIn("S9", str(ctx.exception)) + + def test_duplicate_serials_hard_error(self): + devs = [_blk("sdb", serial="DUP"), _blk("sdc", serial="DUP")] + with self.assertRaises(ValueError) as ctx: + utils.filter_eligible_block_devices(devs) + self.assertIn("DUP", str(ctx.exception)) + + def test_no_selection_takes_all_eligible(self): + devs = [_blk("sdb"), _blk("sda", root=True, mounted=True), _blk("sdc")] + sel, _ = utils.filter_eligible_block_devices(devs) + self.assertEqual(sorted(d["name"] for d in sel), ["sdb", "sdc"]) + + +class TestDetectLblkDevices(unittest.TestCase): + + def test_maps_config_entry_shape(self): + devs = [_blk("sdb", serial="S1", by_id="/dev/disk/by-id/wwn-0x1", + size=42, numa=1)] + with patch.object(utils.node_utils, "get_block_devices_info", return_value=devs): + result = utils.detect_lblk_devices() + self.assertEqual(result, { + "sdb": {"name": "sdb", "serial": "S1", + "by_id": "/dev/disk/by-id/wwn-0x1", "size": 42, "numa": 1}, + }) + + def test_synthetic_serial_warns_but_passes(self): + devs = [_blk("sdb", serial="SYN-abc123", synthetic=True)] + with patch.object(utils.node_utils, "get_block_devices_info", return_value=devs), \ + patch.object(utils, "logger") as mock_logger: + result = utils.detect_lblk_devices() + self.assertIn("sdb", result) + self.assertTrue(mock_logger.warning.called) + + +class TestAioBdevName(unittest.TestCase): + + def test_plain_serial(self): + self.assertEqual(utils.aio_bdev_name_for_serial("S3Z8NX0M600123"), + "aio_S3Z8NX0M600123") + + def test_stable(self): + self.assertEqual(utils.aio_bdev_name_for_serial("ABC_1"), + utils.aio_bdev_name_for_serial("ABC_1")) + + def test_special_chars_never_collide(self): + a = utils.aio_bdev_name_for_serial("S1:A") + b = utils.aio_bdev_name_for_serial("S1;A") + self.assertNotEqual(a, b) + for name in (a, b): + self.assertRegex(name, r"^aio_[A-Za-z0-9_]+$") + + def test_long_serial_truncated_with_hash(self): + serial = "X" * 100 + name = utils.aio_bdev_name_for_serial(serial) + self.assertLessEqual(len(name), len("aio_") + 40 + 7) + self.assertNotEqual(name, utils.aio_bdev_name_for_serial("X" * 99)) + + +class TestResolveLblkEntries(unittest.TestCase): + + CONFIGURED = [ + {"name": "sdb", "serial": "S1", "by_id": "/dev/disk/by-id/wwn-1", + "size": 100, "numa": 0}, + {"name": "sdc", "serial": "S2", "by_id": "", "size": 200, "numa": 1}, + ] + + def test_serial_first_survives_rename(self): + # After reboot S1 moved sdb->sdx; stored name must NOT win. + host = [_blk("sdx", serial="S1", by_id="/dev/disk/by-id/wwn-1"), + _blk("sdc", serial="S2")] + resolved, missing = utils.resolve_lblk_entries(self.CONFIGURED, host) + self.assertEqual(missing, []) + by_serial = {e["serial"]: e for e in resolved} + self.assertEqual(by_serial["S1"]["name"], "sdx") + self.assertEqual(by_serial["S1"]["current_path"], "/dev/sdx") + + def test_name_fallback_when_serial_unknown(self): + # Host reports a different serial for sdb (e.g. synthetic drift); + # the stored name is the last-resort match. + host = [_blk("sdb", serial="OTHER"), _blk("sdc", serial="S2")] + resolved, missing = utils.resolve_lblk_entries(self.CONFIGURED, host) + self.assertEqual(missing, []) + names = {e["name"] for e in resolved} + self.assertEqual(names, {"sdb", "sdc"}) + + def test_missing_device_reported(self): + host = [_blk("sdc", serial="S2")] + resolved, missing = utils.resolve_lblk_entries(self.CONFIGURED, host) + self.assertEqual(len(resolved), 1) + self.assertEqual(missing[0]["serial"], "S1") + + def test_live_fields_refresh(self): + host = [_blk("sdb", serial="S1", by_id="/dev/disk/by-id/wwn-NEW", + size=999, numa=1, parts=True), + _blk("sdc", serial="S2")] + resolved, _ = utils.resolve_lblk_entries(self.CONFIGURED, host) + entry = next(e for e in resolved if e["serial"] == "S1") + self.assertEqual(entry["by_id"], "/dev/disk/by-id/wwn-NEW") + self.assertEqual(entry["size"], 999) + self.assertEqual(entry["numa"], 1) + self.assertTrue(entry["has_partitions"]) + + +class TestNodeConfigHelpers(unittest.TestCase): + + def test_device_count_lblk(self): + node = {"ssd_pcis": [], "lblk_devices": [{"name": "sdb"}, {"name": "sdc"}]} + self.assertEqual(utils.node_config_device_count(node), 2) + + def test_device_count_nvme(self): + node = {"ssd_pcis": ["0000:00:1e.0"], "lblk_devices": []} + self.assertEqual(utils.node_config_device_count(node), 1) + + def test_device_count_missing_keys(self): + self.assertEqual(utils.node_config_device_count({}), 0) + + def test_min_sys_memory_lblk_uses_capacity_factor(self): + node = {"lblk_devices": [{"size": 50 << 30}, {"size": 50 << 30}]} + expected = 2147483648 + int((100 << 30) * utils.SYS_MEMORY_STORAGE_FACTOR) + self.assertEqual(utils.node_config_min_sys_memory(node), expected) + # ~2.2 GiB total — NOT 2 GiB + full capacity (the bug the first AWS + # lblk deploy hit: 102 GiB demanded on 32 GiB hosts). + self.assertLess(utils.node_config_min_sys_memory(node), 3 << 30) + + def test_min_sys_memory_nvme_delegates(self): + node = {"ssd_pcis": ["0000:00:1e.0"], "lblk_devices": []} + with patch.object(utils, "calculate_minimum_sys_memory", return_value=7) as m: + self.assertEqual(utils.node_config_min_sys_memory(node), 7) + m.assert_called_once_with(["0000:00:1e.0"]) + + +class TestValidateNodeConfig(unittest.TestCase): + + def _node(self, ssd_pcis=None, lblk_devices=None): + return { + "socket": 0, + "cpu_mask": "0x3", + "isolated": [0, 1], + "l-cores": "0@0,1@1", + "number_of_alcemls": 1, + "distribution": { + "app_thread_core": [0], "jm_cpu_core": [0], + "poller_cpu_cores": [1], "alceml_cpu_cores": [1], + "distrib_cpu_cores": [1], "jc_singleton_core": [0], + }, + "ssd_pcis": ssd_pcis if ssd_pcis is not None else [], + "lblk_devices": lblk_devices if lblk_devices is not None else [], + "nic_ports": ["eth0"], + "number_of_distribs": 2, + "small_pool_count": 1, + "large_pool_count": 1, + "max_lvol": 10, + "max_size": 1 << 30, + "huge_page_memory": 1 << 30, + "sys_memory": 1 << 31, + } + + def test_valid_nvme_config(self): + self.assertTrue(utils.validate_node_config(self._node(ssd_pcis=["0000:00:1e.0"]))) + + def test_valid_lblk_config(self): + node = self._node(lblk_devices=[{"name": "sdb", "serial": "S1", "size": 100}]) + self.assertTrue(utils.validate_node_config(node)) + + def test_nvme_config_without_lblk_key_still_valid(self): + node = self._node(ssd_pcis=["0000:00:1e.0"]) + del node["lblk_devices"] + self.assertTrue(utils.validate_node_config(node)) + + def test_both_sources_rejected(self): + node = self._node(ssd_pcis=["0000:00:1e.0"], + lblk_devices=[{"name": "sdb", "serial": "S1", "size": 1}]) + self.assertFalse(utils.validate_node_config(node)) + + def test_neither_source_rejected(self): + self.assertFalse(utils.validate_node_config(self._node())) + + def test_lblk_entry_missing_serial_rejected(self): + node = self._node(lblk_devices=[{"name": "sdb", "size": 100}]) + self.assertFalse(utils.validate_node_config(node)) + + def test_lblk_entry_missing_name_rejected(self): + node = self._node(lblk_devices=[{"serial": "S1", "size": 100}]) + self.assertFalse(utils.validate_node_config(node)) + + def test_lblk_entry_bad_size_rejected(self): + for size in (0, -5, "100", None): + node = self._node(lblk_devices=[{"name": "sdb", "serial": "S1", "size": size}]) + self.assertFalse(utils.validate_node_config(node), f"size={size!r}") + + def test_lblk_entry_not_a_dict_rejected(self): + node = self._node(lblk_devices=["sdb"]) + self.assertFalse(utils.validate_node_config(node)) + + def test_invalid_pci_still_rejected(self): + self.assertFalse(utils.validate_node_config(self._node(ssd_pcis=["/dev/sdb"]))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_onboarding.py b/tests/unit/test_lblk_onboarding.py new file mode 100644 index 0000000000..407ca1d449 --- /dev/null +++ b/tests/unit/test_lblk_onboarding.py @@ -0,0 +1,247 @@ +# coding=utf-8 +"""Unit tests for lblk-mode device onboarding. + +Covered: + - utils.addAioDevices: fresh-create vs reuse (restart idempotency), + by-id-preferred filename, examine + qd-sampling wiring, zero-size skip, + create-failure raise, full NVMeDevice field population. + - storage_node_ops._classify_existing_endpoint_record: serial-based + overlap detection for lblk nodes (add-node idempotency). + - cluster_ops._validated_device_mode. + - constants sanity (placeholder BDF shape, excluded prefixes are a tuple + usable with str.startswith). +""" + +import re +import unittest +from unittest.mock import MagicMock + +from simplyblock_core import cluster_ops, constants, utils +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.storage_node_ops import _classify_existing_endpoint_record + + +def _entry(name="sdb", serial="S1", by_id="/dev/disk/by-id/wwn-1", + size=100 << 30, numa=0, model="MODEL-X"): + return {"name": name, "serial": serial, "by_id": by_id, "size": size, + "numa": numa, "model": model, "current_path": f"/dev/{name}", + "has_partitions": False} + + +def _snode(node_id="node-1", cluster_id="cluster-1", physical_label=3): + n = StorageNode() + n.uuid = node_id + n.cluster_id = cluster_id + n.physical_label = physical_label + return n + + +class _FakeRpc: + """Minimal SPDK RPC fake for addAioDevices: get_bdevs answers from an + internal registry; bdev_aio_create registers; every call is recorded.""" + + def __init__(self, existing=None, create_ok=True, block_size=4096, + num_blocks=1000): + self.host = "test-host" + self.bdevs = dict(existing or {}) + self.create_ok = create_ok + self.block_size = block_size + self.num_blocks = num_blocks + self.calls = [] + + def get_bdevs(self, name): + self.calls.append(("get_bdevs", name)) + if name in self.bdevs: + return [self.bdevs[name]] + return None + + def bdev_aio_create(self, name, filename, block_size=0): + self.calls.append(("bdev_aio_create", name, filename)) + if not self.create_ok: + return None + self.bdevs[name] = {"name": name, "block_size": self.block_size, + "num_blocks": self.num_blocks} + return name + + def bdev_examine(self, name): + self.calls.append(("bdev_examine", name)) + return True + + def bdev_wait_for_examine(self): + self.calls.append(("bdev_wait_for_examine",)) + return True + + def bdev_set_qd_sampling_period(self, name, period): + self.calls.append(("qd_sampling", name, period)) + return True + + def _called(self, method): + return [c for c in self.calls if c[0] == method] + + +class TestAddAioDevices(unittest.TestCase): + + def test_fresh_create_full_field_population(self): + rpc = _FakeRpc() + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(len(devs), 1) + dev = devs[0] + self.assertIsInstance(dev, NVMeDevice) + self.assertEqual(dev.bdev_type, "aio") + self.assertEqual(dev.nvme_bdev, utils.aio_bdev_name_for_serial("S1")) + self.assertEqual(dev.serial_number, "S1") + self.assertEqual(dev.device_name, "sdb") + self.assertEqual(dev.device_path, "/dev/sdb") + self.assertEqual(dev.by_id_path, "/dev/disk/by-id/wwn-1") + self.assertEqual(dev.pcie_address, "") + self.assertEqual(dev.nvme_controller, "") + self.assertEqual(dev.model_id, "MODEL-X") + self.assertEqual(dev.size, 4096 * 1000) + self.assertEqual(dev.physical_label, 3) + self.assertEqual(dev.node_id, "node-1") + self.assertEqual(dev.cluster_id, "cluster-1") + self.assertEqual(dev.status, NVMeDevice.STATUS_ONLINE) + + def test_filename_prefers_by_id(self): + rpc = _FakeRpc() + utils.addAioDevices(rpc, _snode(), [_entry()]) + create = rpc._called("bdev_aio_create")[0] + self.assertEqual(create[2], "/dev/disk/by-id/wwn-1") + + def test_filename_falls_back_to_current_path(self): + rpc = _FakeRpc() + utils.addAioDevices(rpc, _snode(), [_entry(by_id="")]) + create = rpc._called("bdev_aio_create")[0] + self.assertEqual(create[2], "/dev/sdb") + + def test_reuse_existing_bdev_no_create(self): + name = utils.aio_bdev_name_for_serial("S1") + rpc = _FakeRpc(existing={name: {"name": name, "block_size": 4096, + "num_blocks": 10}}) + devs = utils.addAioDevices(rpc, _snode(), [_entry()]) + self.assertEqual(len(devs), 1) + self.assertEqual(rpc._called("bdev_aio_create"), []) + + def test_examine_and_qd_sampling_wired(self): + rpc = _FakeRpc() + utils.addAioDevices(rpc, _snode(), [_entry()]) + self.assertTrue(rpc._called("bdev_examine")) + self.assertTrue(rpc._called("bdev_wait_for_examine")) + qd = rpc._called("qd_sampling")[0] + self.assertEqual(qd[2], constants.AIO_QD_SAMPLING_PERIOD_US) + + def test_zero_size_skipped(self): + rpc = _FakeRpc(num_blocks=0) + devs = utils.addAioDevices(rpc, _snode(), [_entry()]) + self.assertEqual(devs, []) + + def test_create_failure_raises(self): + rpc = _FakeRpc(create_ok=False) + with self.assertRaises(Exception): + utils.addAioDevices(rpc, _snode(), [_entry()]) + + def test_multiple_devices(self): + rpc = _FakeRpc() + devs = utils.addAioDevices(rpc, _snode(), [ + _entry(name="sdb", serial="S1"), _entry(name="sdc", serial="S2")]) + self.assertEqual([d.serial_number for d in devs], ["S1", "S2"]) + self.assertEqual(len({d.nvme_bdev for d in devs}), 2) + + +class TestClassifyEndpointRecordLblk(unittest.TestCase): + + def _db_with(self, node): + db = MagicMock() + db.get_storage_nodes_by_cluster_id.return_value = [node] + return db + + def _lblk_node(self, status, serials=("S1",)): + n = StorageNode() + n.uuid = "existing" + n.api_endpoint = "1.2.3.4:5000" + n.status = status + n.ssd_pcie = [] + n.lblk_devices = [{"name": f"sd{i}", "serial": s} + for i, s in enumerate(serials)] + return n + + def test_serial_overlap_online_is_already_added(self): + node = self._lblk_node(StorageNode.STATUS_ONLINE) + action, found = _classify_existing_endpoint_record( + self._db_with(node), "c1", "1.2.3.4:5000", [], lblk_serials=["S1"]) + self.assertEqual(action, "already_added") + self.assertIs(found, node) + + def test_serial_overlap_in_creation_is_cleanup(self): + node = self._lblk_node(StorageNode.STATUS_IN_CREATION) + action, _ = _classify_existing_endpoint_record( + self._db_with(node), "c1", "1.2.3.4:5000", [], lblk_serials=["S1"]) + self.assertEqual(action, "cleanup") + + def test_serial_overlap_other_status_is_conflict(self): + node = self._lblk_node(StorageNode.STATUS_OFFLINE) + action, _ = _classify_existing_endpoint_record( + self._db_with(node), "c1", "1.2.3.4:5000", [], lblk_serials=["S1"]) + self.assertEqual(action, "conflict") + + def test_no_serial_overlap_no_match(self): + node = self._lblk_node(StorageNode.STATUS_ONLINE, serials=("OTHER",)) + action, found = _classify_existing_endpoint_record( + self._db_with(node), "c1", "1.2.3.4:5000", [], lblk_serials=["S1"]) + self.assertIsNone(action) + self.assertIsNone(found) + + def test_different_endpoint_ignored(self): + node = self._lblk_node(StorageNode.STATUS_ONLINE) + action, _ = _classify_existing_endpoint_record( + self._db_with(node), "c1", "9.9.9.9:5000", [], lblk_serials=["S1"]) + self.assertIsNone(action) + + def test_nvme_pcie_overlap_still_works(self): + node = StorageNode() + node.uuid = "existing" + node.api_endpoint = "1.2.3.4:5000" + node.status = StorageNode.STATUS_ONLINE + node.ssd_pcie = ["0000:00:1e.0"] + action, _ = _classify_existing_endpoint_record( + self._db_with(node), "c1", "1.2.3.4:5000", ["0000:00:1e.0"]) + self.assertEqual(action, "already_added") + + +class TestDeviceModeValidation(unittest.TestCase): + + def test_accepts_both_modes_case_insensitive(self): + self.assertEqual(cluster_ops._validated_device_mode("nvme"), "nvme") + self.assertEqual(cluster_ops._validated_device_mode("LBLK"), "lblk") + + def test_none_defaults_to_nvme(self): + self.assertEqual(cluster_ops._validated_device_mode(None), "nvme") + + def test_rejects_unknown(self): + with self.assertRaises(ValueError): + cluster_ops._validated_device_mode("scsi") + + +class TestLblkConstants(unittest.TestCase): + + def test_placeholder_is_valid_bdf_and_never_a_device(self): + self.assertTrue(re.fullmatch( + r"[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]", + constants.LBLK_PCI_ALLOWED_PLACEHOLDER)) + self.assertEqual(constants.LBLK_PCI_ALLOWED_PLACEHOLDER, "0000:00:00.0") + + def test_excluded_prefixes_usable_with_startswith(self): + self.assertIsInstance(constants.LBLK_EXCLUDED_NAME_PREFIXES, tuple) + self.assertTrue("loop7".startswith(constants.LBLK_EXCLUDED_NAME_PREFIXES)) + self.assertFalse("sdb".startswith(constants.LBLK_EXCLUDED_NAME_PREFIXES)) + + def test_watchdog_thresholds_positive(self): + self.assertGreater(constants.AIO_HUNG_IO_STALL_POLLS, 0) + self.assertGreater(constants.AIO_DEVICE_ABSENT_POLLS, 0) + self.assertGreater(constants.AIO_QD_SAMPLING_PERIOD_US, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_watchdog.py b/tests/unit/test_lblk_watchdog.py new file mode 100644 index 0000000000..30e56614f6 --- /dev/null +++ b/tests/unit/test_lblk_watchdog.py @@ -0,0 +1,265 @@ +# coding=utf-8 +"""Unit tests for the lblk hung-IO watchdog and device-disappearance sweep +in services/device_monitor.py. + +The watchdog is the control-plane replacement for bdev_nvme's +timeout_us/action_on_timeout (which AIO bdevs lack): queue-depth-sampled +iostat with no completion progress across N polls => the device is fed into +the SAME machinery an erroring nvme device hits (io_error + UNAVAILABLE, +countable LOCAL_FAILURE cause). Disappearance from the host inventory => +device_remove, the SPDK_BDEV_EVENT_REMOVE treatment. + +Covered: + - stall accumulation requires inflight IO on EVERY poll AND zero progress + - any completion progress resets the window + - RPC failure / missing bdevs / missing queue_depth freeze (never count) + - missing queue_depth re-arms qd-sampling + - threshold trip returns the device + - non-ONLINE devices are ignored and their tracking state cleared + - nvme (bdev_type != aio) devices are never touched + - presence sweep: absent-debounce, recovery clears the counter, inventory + failure freezes, serial OR name match counts as present + - action dispatch: 1 stalled -> io_error+UNAVAILABLE(LOCAL_FAILURE); + >=2 stalled -> node-level auto-restart; gone -> device_remove +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.services import device_monitor +from simplyblock_core.services.device_monitor import ( + _check_aio_device_presence, + _check_aio_hung_io, + _sweep_aio_devices, +) + + +def _aio_dev(uid="dev-1", status=NVMeDevice.STATUS_ONLINE, serial="S1", + name="sdb", bdev_type="aio"): + d = NVMeDevice() + d.uuid = uid + d.status = status + d.serial_number = serial + d.device_name = name + d.bdev_type = bdev_type + d.nvme_bdev = f"aio_{serial}" + return d + + +def _node(devs, node_id="node-1"): + n = MagicMock() + n.get_id.return_value = node_id + n.nvme_devices = devs + return n + + +def _rpc_with_stats(stats_by_bdev): + rpc = MagicMock() + + def _stats(name): + entry = stats_by_bdev.get(name) + if entry is None: + return {"bdevs": []} + if isinstance(entry, Exception): + raise entry + return {"bdevs": [entry]} + + rpc.get_lvol_stats.side_effect = _stats + return rpc + + +def _stat(total_ops, queue_depth): + return {"num_read_ops": total_ops, "num_write_ops": 0, + "num_unmap_ops": 0, "queue_depth": queue_depth} + + +class WatchdogBase(unittest.TestCase): + def setUp(self): + device_monitor._aio_progress.clear() + device_monitor._aio_absent.clear() + + +class TestHungIoDetection(WatchdogBase): + + def test_first_poll_never_stalls(self): + dev = _aio_dev() + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + self.assertEqual(_check_aio_hung_io(_node([dev]), rpc), []) + + def test_stall_requires_threshold_consecutive_polls(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + # poll 1 primes; polls 2..N-1 accumulate below threshold + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS): + self.assertEqual(_check_aio_hung_io(node, rpc), []) + # poll that reaches the threshold trips + self.assertEqual(_check_aio_hung_io(node, rpc), [dev]) + + def test_progress_resets_window(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS): + _check_aio_hung_io(node, rpc) + # completions advanced -> reset + rpc2 = _rpc_with_stats({dev.nvme_bdev: _stat(101, 5)}) + self.assertEqual(_check_aio_hung_io(node, rpc2), []) + # stalling again needs the full window again + rpc3 = _rpc_with_stats({dev.nvme_bdev: _stat(101, 5)}) + self.assertEqual(_check_aio_hung_io(node, rpc3), []) + + def test_zero_queue_depth_is_idle_not_stall(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 0)}) + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS + 2): + self.assertEqual(_check_aio_hung_io(node, rpc), []) + + def test_rpc_exception_freezes_counter(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS): + _check_aio_hung_io(node, rpc) + # one failing poll must neither trip nor reset + bad = _rpc_with_stats({dev.nvme_bdev: RuntimeError("rpc down")}) + self.assertEqual(_check_aio_hung_io(node, bad), []) + # next good stalled poll trips (counter was frozen, not reset) + self.assertEqual(_check_aio_hung_io(node, rpc), [dev]) + + def test_empty_bdevs_freezes_counter(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({}) # no entry -> {"bdevs": []} + self.assertEqual(_check_aio_hung_io(node, rpc), []) + self.assertNotIn(dev.get_id(), device_monitor._aio_progress) + + def test_missing_queue_depth_rearms_sampling_and_freezes(self): + dev = _aio_dev() + node = _node([dev]) + stat = {"num_read_ops": 1, "num_write_ops": 0, "num_unmap_ops": 0} + rpc = _rpc_with_stats({dev.nvme_bdev: stat}) + self.assertEqual(_check_aio_hung_io(node, rpc), []) + rpc.bdev_set_qd_sampling_period.assert_called_once_with( + dev.nvme_bdev, constants.AIO_QD_SAMPLING_PERIOD_US) + + def test_non_online_device_ignored_and_state_cleared(self): + dev = _aio_dev() + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + _check_aio_hung_io(node, rpc) + self.assertIn(dev.get_id(), device_monitor._aio_progress) + dev.status = NVMeDevice.STATUS_UNAVAILABLE + self.assertEqual(_check_aio_hung_io(node, rpc), []) + self.assertNotIn(dev.get_id(), device_monitor._aio_progress) + + def test_nvme_devices_never_touched(self): + dev = _aio_dev(bdev_type="nvme") + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS + 2): + self.assertEqual(_check_aio_hung_io(node, rpc), []) + rpc.get_lvol_stats.assert_not_called() + + +class TestDevicePresence(WatchdogBase): + + def _node_with_inventory(self, devs, inventory): + node = _node(devs) + client = MagicMock() + client.get_blockdevices.return_value = (inventory, None) + node.client.return_value = client + return node + + def test_present_by_serial(self): + dev = _aio_dev(serial="S1", name="sdb") + # renamed on host: serial still matches + node = self._node_with_inventory([dev], [{"name": "sdx", "serial": "S1"}]) + self.assertEqual(_check_aio_device_presence(node), []) + self.assertNotIn(dev.get_id(), device_monitor._aio_absent) + + def test_present_by_name_fallback(self): + dev = _aio_dev(serial="S1", name="sdb") + node = self._node_with_inventory([dev], [{"name": "sdb", "serial": "OTHER"}]) + self.assertEqual(_check_aio_device_presence(node), []) + + def test_absent_debounced_then_reported(self): + dev = _aio_dev() + node = self._node_with_inventory([dev], [{"name": "sdz", "serial": "ZZ"}]) + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS - 1): + self.assertEqual(_check_aio_device_presence(node), []) + self.assertEqual(_check_aio_device_presence(node), [dev]) + + def test_reappearance_clears_counter(self): + dev = _aio_dev(serial="S1") + gone = self._node_with_inventory([dev], []) + # inventory [] is falsy -> unknown, so use a non-matching entry + gone = self._node_with_inventory([dev], [{"name": "x", "serial": "y"}]) + _check_aio_device_presence(gone) + back = self._node_with_inventory([dev], [{"name": "sdb", "serial": "S1"}]) + self.assertEqual(_check_aio_device_presence(back), []) + self.assertNotIn(dev.get_id(), device_monitor._aio_absent) + + def test_inventory_failure_freezes(self): + dev = _aio_dev() + node = _node([dev]) + node.client.side_effect = RuntimeError("agent down") + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS + 2): + self.assertEqual(_check_aio_device_presence(node), []) + self.assertNotIn(dev.get_id(), device_monitor._aio_absent) + + def test_no_aio_devices_no_inventory_call(self): + dev = _aio_dev(bdev_type="nvme") + node = _node([dev]) + self.assertEqual(_check_aio_device_presence(node), []) + node.client.assert_not_called() + + +class TestSweepActions(WatchdogBase): + + def _sweep(self, node, stalled=None, gone=None): + with patch.object(device_monitor, "_check_aio_device_presence", + return_value=gone or []), \ + patch.object(device_monitor, "_check_aio_hung_io", + return_value=stalled or []), \ + patch.object(device_monitor, "device_controller") as dc, \ + patch.object(device_monitor, "tasks_controller") as tc: + _sweep_aio_devices(node) + return dc, tc + + def test_single_stalled_marks_unavailable_with_countable_cause(self): + dev = _aio_dev() + node = _node([dev]) + dc, tc = self._sweep(node, stalled=[dev]) + dc.device_set_io_error.assert_called_once_with(dev.get_id(), True) + dc.device_set_unavailable.assert_called_once_with( + dev.get_id(), cause=device_monitor.CAUSE_LOCAL_FAILURE) + tc.add_node_to_auto_restart.assert_not_called() + + def test_two_stalled_escalates_to_node_restart(self): + d1, d2 = _aio_dev("dev-1", serial="S1"), _aio_dev("dev-2", serial="S2") + node = _node([d1, d2]) + dc, tc = self._sweep(node, stalled=[d1, d2]) + tc.add_node_to_auto_restart.assert_called_once_with(node) + dc.device_set_unavailable.assert_not_called() + + def test_gone_device_removed_with_countable_cause(self): + dev = _aio_dev() + node = _node([dev]) + dc, _ = self._sweep(node, gone=[dev]) + dc.device_remove.assert_called_once_with( + dev.get_id(), cause=device_monitor.CAUSE_LOCAL_FAILURE) + + def test_stall_tracking_cleared_after_action(self): + dev = _aio_dev() + device_monitor._aio_progress[dev.get_id()] = (100, 3) + node = _node([dev]) + self._sweep(node, stalled=[dev]) + self.assertNotIn(dev.get_id(), device_monitor._aio_progress) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/web/api/v2/test_storage_node_endpoints.py b/tests/unit/web/api/v2/test_storage_node_endpoints.py index 859ca945f3..f383f33374 100644 --- a/tests/unit/web/api/v2/test_storage_node_endpoints.py +++ b/tests/unit/web/api/v2/test_storage_node_endpoints.py @@ -56,6 +56,7 @@ def test_creates_add_node_task(self, client, db, cluster, tasks_controller): 'spdk_sys_mem': None, 'failure_domain': None, 'expansion': False, + 'force_format': False, }) # Default response format is 'identifier': body is the task id assert response.json() == TASK_ID