From 7ab5e627f00c2a3076bb9d60c7206a18cec3c2e3 Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Thu, 13 Aug 2026 22:03:38 -0700 Subject: [PATCH] Add service for Slurm topology collection Summary: Package and configure scontrol_topology as a single-node systemd service that collects and publishes Slurm topology every 60 seconds. Add Debian packaging and operator documentation while leaving exporter-specific routing in deployment configuration. Test Plan: Static checks: git diff --check; black --check and flake8 on gcm/tests/test_scontrol_topology.py; TOML, Python syntax, systemd-analyze, and Debian packaging verification. End to end: deployed the GCM build to a development Slurm cluster, ran a live one-shot topology collection through the configured telemetry pipeline, and confirmed fresh rows in the target dataset with the expected topology fields and node counts. --- debian/rules | 1 + gcm/README.md | 1 + gcm/monitoring/config/config.toml | 8 ++ systemd/scontrol_topology.service | 16 +++ .../docs/GCM_Monitoring/collectors/README.md | 3 +- .../collectors/scontrol_topology.md | 116 ++++++++++++++++++ 6 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 systemd/scontrol_topology.service create mode 100644 website/docs/GCM_Monitoring/collectors/scontrol_topology.md diff --git a/debian/rules b/debian/rules index adb0271b..d8b081fc 100755 --- a/debian/rules +++ b/debian/rules @@ -24,6 +24,7 @@ override_dh_auto_install: cp systemd/slurm_job_monitor.service ${SYSTEMD_OUT} cp systemd/slurm_monitor.service ${SYSTEMD_OUT} cp systemd/scontrol.service ${SYSTEMD_OUT} + cp systemd/scontrol_topology.service ${SYSTEMD_OUT} cp systemd/storage.service ${SYSTEMD_OUT} mkdir -p ${GCM_HC_LIB} diff --git a/gcm/README.md b/gcm/README.md index 4038e33d..fc8a16ce 100644 --- a/gcm/README.md +++ b/gcm/README.md @@ -22,6 +22,7 @@ Collects cluster statistics from the [Slurm](https://slurm.schedmd.com/documenta - `gcm slurm_job_monitor --sink=stdout --once` - Job queue and node state information according to `squeue` and `sinfo` - `gcm scontrol --sink=stdout --once` - Slurm control plane monitoring - `gcm scontrol_config --sink=stdout --once` - Slurm configuration file monitoring +- `gcm scontrol_topology --sink=stdout --once` - Slurm block and switch topology monitoring - `gcm storage --help` - Storage system monitoring (not used in Meta production) - `gcm nvml_monitor --sink=stdout --once` - GPU telemetry collection (not used in Meta production) diff --git a/gcm/monitoring/config/config.toml b/gcm/monitoring/config/config.toml index cc589657..60f9b9b4 100644 --- a/gcm/monitoring/config/config.toml +++ b/gcm/monitoring/config/config.toml @@ -61,3 +61,11 @@ sink="otel" sink_opts = [ "metric_resource_attributes={'key1': 'val1'}", ] + +[gcm.scontrol_topology] +log_folder = "/var/log" +interval = 60 +sink = "otel" +sink_opts = [ + "log_resource_attributes={'key1': 'val1'}", +] diff --git a/systemd/scontrol_topology.service b/systemd/scontrol_topology.service new file mode 100644 index 00000000..6fdaf05a --- /dev/null +++ b/systemd/scontrol_topology.service @@ -0,0 +1,16 @@ +[Unit] +Description=Collect Slurm topology +After=network.target + +[Service] +Type=simple +User=cluster_monitor +Environment="PATH=/usr/local/bin:/public/slurm/20.11.3/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/slurm/bin" +LogsDirectory=scontrol_topology_logs +ExecStart=/usr/bin/gcm scontrol_topology +Restart=on-failure +RestartSec=300 +Slice=hc_resources.slice + +[Install] +WantedBy=multi-user.target diff --git a/website/docs/GCM_Monitoring/collectors/README.md b/website/docs/GCM_Monitoring/collectors/README.md index 8551384f..d2823240 100644 --- a/website/docs/GCM_Monitoring/collectors/README.md +++ b/website/docs/GCM_Monitoring/collectors/README.md @@ -14,6 +14,7 @@ This directory contains documentation for all GCM monitoring collectors. Collect - **[sacctmgr_user](sacctmgr_user.md)** - Collects user account information and associations - **[scontrol](scontrol.md)** - Collects partition configuration - **[scontrol_config](scontrol_config.md)** - Collects cluster-wide configuration +- **[scontrol_topology](scontrol_topology.md)** - Collects block and switch topology - **[slurm_job_monitor](slurm_job_monitor.md)** - Real-time node and job monitoring - **[slurm_monitor](slurm_monitor.md)** - Comprehensive cluster-wide metrics aggregation - **[sprio](sprio.md)** - Collects job priority factors for pending jobs @@ -53,7 +54,7 @@ Most collectors use `run_data_collection_loop()` which provides: Data payloads use typed dataclasses for validation: - `DevicePlusJobMetrics`, `HostMetrics` (nvml_monitor) - `Sacct`, `SacctmgrQosPayload`, `SacctmgrUserPayload` (SLURM accounting) -- `Scontrol`, `ScontrolConfig` (SLURM control) +- `Scontrol`, `ScontrolConfig`, `ScontrolTopology` (SLURM control) - `NodeData`, `SLURMLog` (SLURM monitoring) ## Adding a New Collector diff --git a/website/docs/GCM_Monitoring/collectors/scontrol_topology.md b/website/docs/GCM_Monitoring/collectors/scontrol_topology.md new file mode 100644 index 00000000..b8fabfba --- /dev/null +++ b/website/docs/GCM_Monitoring/collectors/scontrol_topology.md @@ -0,0 +1,116 @@ +# scontrol_topology + +## Overview + +Collects Slurm topology from `scontrol show topo` every 60 seconds. Run one +instance per cluster, on the same controller-side host as `scontrol.service`. + +The collector supports block topology (`BlockName`, `BlockIndex`, and +`BlockSize`) and switch topology (`SwitchName`, `Level`, `LinkSpeed`, and +`Switches`). Slurm hostlists are expanded into the `Nodes` tag set, and +`node_count` records the full number of nodes before the 10,000-entry safety +limit is applied. + +**Data Type**: `DataType.LOG`, **Schema**: `ScontrolTopology` + +## Execution Scope + +Single node in the cluster. The host must have permission to run +`scontrol show topo`. + +## Output Schema + +The collector publishes one record per block or switch returned by Slurm. +Fields that do not apply to the cluster's topology plugin are omitted. + +```python +{ + "cluster": str, # Cluster identifier + "derived_cluster": str | None, # Derived identifier for heterogeneous clusters + + # Block topology + "BlockName": str | None, + "BlockIndex": int | None, + "BlockSize": int | None, + + # Switch topology + "SwitchName": str | None, + "Level": int | None, + "LinkSpeed": int | None, + "Switches": str | None, # Child switch expression + + "Nodes": list[str] | None, # Expanded Slurm hostlist + "node_count": int, # Number of nodes before truncation +} +``` + +Node lists longer than 10,000 entries are truncated to protect the collector; +`node_count` still contains the untruncated count. + +## Packaged Service + +The packaged `scontrol_topology.service` runs the collector with the settings +from the `[gcm.scontrol_topology]` section of `/etc/fb-gcm/config.toml`. The +packaged configuration uses the OpenTelemetry exporter; configure its endpoint +and resource attributes for your observability backend before enabling the +service. + +```bash +systemctl enable --now scontrol_topology.service +systemctl status scontrol_topology.service +``` + +Only enable the service on one monitoring host per cluster to avoid duplicate +rows. + +## Command-Line Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `--cluster` | String | Auto-detected | Cluster identifier | +| `--sink` | String | `stdout` | Sink destination; see [Exporters](../exporters/README.md) | +| `--sink-opt` | Multiple | - | Sink-specific option in OmegaConf dot-list syntax | +| `--log-level` | Choice | `INFO` | Logging verbosity | +| `--log-folder` | Path | `sacct_running_logs` | Parent directory for collector logs | +| `--stdout` | Flag | False | Write collector logs to standard output | +| `--heterogeneous-cluster-v1` | Flag | False | Compute a derived cluster identifier | +| `--interval` | Integer | 60 | Seconds between collection cycles | +| `--once` | Flag | False | Collect once and exit | +| `--retries` | Integer | 2 | Maximum sink write retries | +| `--dry-run` | Flag | False | Publish records to standard output | +| `--chunk-size` | Integer | `1M` | Maximum sink write chunk size in bytes | + +Values in `/etc/fb-gcm/config.toml` override these command-line defaults for +the packaged service. + +## Usage Examples + +### One-Time Collection + +Inspect a snapshot without sending it to a remote backend: + +```bash +gcm scontrol_topology --once --sink stdout +``` + +### OpenTelemetry Export + +Publish through an OTLP-compatible backend: + +```bash +gcm scontrol_topology --once \ + --sink otel \ + --sink-opt otel_endpoint=http://localhost:4318 \ + --sink-opt "log_resource_attributes={'service.name': 'gcm'}" +``` + +### Custom Collection Interval + +Collect every five minutes and publish to a file: + +```bash +gcm scontrol_topology \ + --interval 300 \ + --sink file \ + --sink-opt filepath=/tmp/slurm-topology.jsonl +```