Mimir integration

packages/mwan3/files/usr/sbin/mwan3track

@@ -119,7 +119,8 @@ disconnected() {
 		LOG notice "Interface $INTERFACE ($DEVICE) is offline"
 		env -i ACTION="disconnected" INTERFACE="$INTERFACE" DEVICE="$DEVICE" /sbin/hotplug-call iface
 	else
-		LOG notice "Skip disconnected event for $INTERFACE ($DEVICE)"
+		LOG notice "Skip disconnected event for $INTERFACE ($DEVICE), but sending alert"
+		env -i ACTION="disconnected" INTERFACE="$INTERFACE" DEVICE="$DEVICE" /usr/libexec/mwan-hooks/send-mwan-alert
 	fi
 }
 

packages/ns-plug/Makefile

@@ -100,11 +100,12 @@ define Package/ns-plug/install
 	$(INSTALL_BIN) ./files/disable_automatic_updates $(1)/usr/share/ns-plug/hooks/unregister/60disable_automatic_updates
 	$(INSTALL_CONF) ./files/config $(1)/etc/config/ns-plug
 	$(INSTALL_CONF) files/ns-plug.keep $(1)/lib/upgrade/keep.d/ns-plug
-	$(INSTALL_CONF) files/health_alarm_notify.conf $(1)/etc/netdata
 	$(INSTALL_BIN) ./files/send-mwan-alert $(1)/usr/libexec/mwan-hooks
 	$(INSTALL_BIN) ./files/backup-encryption-alert $(1)/usr/libexec
+	$(INSTALL_BIN) ./files/ns-plug-alert $(1)/usr/sbin
 	$(INSTALL_BIN) ./files/mwan-hooks $(1)/usr/libexec/ns-plug
 	$(INSTALL_BIN) ./files/ns-plug-rsyslog-fixup.uci-default $(1)/etc/uci-defaults/rsyslog-fixup
+	$(INSTALL_DATA) files/health_alarm_notify.conf $(1)/usr/share/ns-plug/
 endef
 
 $(eval $(call BuildPackage,ns-plug))

packages/ns-plug/README.md

@@ -141,9 +141,38 @@ Alerts are also logged to `/var/log/messages` and are visible within the netdata
 
 Only the following alerts are sent to the remote system:
 
+- all of them repeat every 30 minutes while active
 - disk space occupation
 - WAN down events
 
+To emulate these alerts manually with `ns-plug-alert`, use:
+
+```
+# Disk usage alert
+ns-plug-alert fire --alertname DiskSpaceCritical --severity critical \
+  --labels service=storage mountpoint=/mnt/data \
+  --annotations \
+    "summary_en=Disk space critical" \
+    "summary_it=Spazio disco critico" \
+    "description_en=Disk usage above 90% on /mnt/data" \
+    "description_it=Utilizzo disco sopra 90% su /mnt/data"
+
+ns-plug-alert resolve --alertname DiskSpaceCritical --severity critical \
+  --labels service=storage mountpoint=/mnt/data
+
+# WAN down alert
+ns-plug-alert fire --alertname WanDown --severity critical \
+  --labels service=network interface=wan0 \
+  --annotations \
+    "summary_en=WAN interface is down" \
+    "summary_it=Interfaccia non disponibile" \
+    "description_en=WAN interface wan0 is down. Internet connectivity could be affected." \
+    "description_it=Interfaccia WAN wan0 non disponibile. Connettivita Internet potrebbe essere compromessa."
+
+ns-plug-alert resolve --alertname WanDown --severity critical \
+  --labels service=network interface=wan0
+```
+
 When an alert is resolved, netdata will also send a clear command to remote server.
 
 ### MultiWAN alerts

packages/ns-plug/files/30_ns-plug_alerts

@@ -2,8 +2,7 @@
 
 # Custom disk alerts
 disks_f="/etc/netdata/health.d/disks.conf"
-if [ ! -f "$disks_f" ]; then
-    cat << EOF > "$disks_f"
+cat << EOF > "$disks_f"
 template: disk_space_usage
        on: disk.space
     class: Utilization
@@ -12,16 +11,16 @@ component: Disk
        os: linux freebsd
     hosts: *
  families: !/dev !/dev/* !/run !/run/* !/overlay *
-     calc: \$used * 100 / (\$avail + \$used)
-    units: %
-    every: 1m
-     warn: \$this > ((\$status >= \$WARNING ) ? (80) : (90))
-     crit: \$this > ((\$status == \$CRITICAL) ? (90) : (98))
-    delay: up 1m down 15m multiplier 1.5 max 1h
-     info: disk $family space utilization
-       to: sysadmin
+      calc: \$used * 100 / (\$avail + \$used)
+     units: %
+     every: 1m
+      warn: \$this > ((\$status >= \$WARNING ) ? (80) : (90))
+      crit: \$this > ((\$status == \$CRITICAL) ? (90) : (98))
+     delay: up 1m down 15m multiplier 1.5 max 1h
+      info: disk \$family space utilization
+        to: sysadmin
+  repeat: critical 5m warning 5m
 EOF
-fi
 
 # Disable unwanted alerts
 files="cpu disks entropy ipc load memory net netfilter processes ram softnet tcp_conn tcp_listen tcp_mem tcp_orphans tcp_resets timex udp_errors"
@@ -33,7 +32,7 @@ do
     fi
 done
 
-# Enable mwan chart
+# Enable some python plugins
 sed -i 's/python.d = no/python.d = yes/' /etc/netdata/netdata.conf
 python_f="/etc/netdata/python.d.conf"
 if [ ! -f "$python_f" ]; then
@@ -52,13 +51,5 @@ nginx_log: no
 EOF
 fi
 
-# Create mwan alert
-cat << EOF > /etc/netdata/health.d/mwan.conf
-template: wan_status
-    on: mwan.score
-lookup: min -1m foreach *
- every: 1m
-  warn: \$this < 5
-  crit: \$this <= 1
-  info: The score of the WAN, 0 means down
-EOF
+# Update netdata notification script
+cp /usr/share/ns-plug/health_alarm_notify.conf /etc/netdata/health_alarm_notify.conf

packages/ns-plug/files/config

@@ -9,3 +9,6 @@ config main 'config'
     option channel ''
     option tun_mtu ''
     option mssfix ''
+    option my_url ''
+    option my_system_key ''
+    option my_system_secret ''

packages/ns-plug/files/health_alarm_notify.conf

@@ -45,6 +45,11 @@ custom_sender() {
     secret=$(uci -q get ns-plug.config.secret)
     url=$(uci -q get ns-plug.config.alerts_url)"alerts/store"
     alert_id=${name}
+
+    logger -t alert "Alert: name=${name} severity=${severity} value=${value} chart=${chart} info=${info} src=${src}"
+
+    # Preserve original netdata status before remapping for legacy API
+    netdata_status="${status}"
     if [ "${status}" == "CRITICAL" ]; then
         status="FAILURE"
     elif [ "${status}" == "CLEAR" ]; then
@@ -72,4 +77,14 @@ custom_sender() {
              --header "Authorization: token ${secret}"  --header "Content-Type: application/json" --header "Accept: application/json"  \
              --data-raw "${payload}" ${url}
     fi
+
+    # Also forward to MY alertmanager if my_url is configured
+    if [ -n "$(uci -q get ns-plug.config.my_url)" ]; then
+        /usr/sbin/ns-plug-alert netdata \
+            --alertname "${name}" \
+            --status "${netdata_status}" \
+            --chart "${chart}" \
+            --family "${family}" \
+            --value "${value}" 2>/dev/null &
+    fi
 }