fix(observability): align alert rule unit tests with rule output

The promtool unit tests failed for two reasons:

- HighP95Latency never fired: its highest finite histogram bucket was
  le=1000, so histogram_quantile capped at 1000 and could never exceed the
  >1000ms threshold. Add a le=2000 bucket so p95 lands above the SLO.
- Every firing exp_alerts entry omitted the job label and exp_annotations
  the rules actually emit, so promtool reported label/annotation mismatches.
  Add the job label (where present) and full exp_annotations to each.

promtool test rules now passes (16 rules, all unit tests green).

Author: CelestinaBeing

monitoring/alerting/alerting_rules_test.yml

@@ -27,6 +27,11 @@ tests:
           - exp_labels:
               severity: critical
               team: backend
+            exp_annotations:
+              summary: 'Trivela backend 5xx error rate above 5%'
+              description:
+                'Error rate is 10% over the last 5 minutes. Investigate backend logs immediately.'
+              runbook_url: 'https://github.com/FinesseStudioLab/Trivela/blob/main/docs/RUNBOOK.md#backend-restart'
 
   # ── HighBackendErrorRate does NOT fire below threshold ───────────────────
   - interval: 1m
@@ -42,15 +47,19 @@ tests:
         exp_alerts: []
 
   # ── HighP95Latency ──────────────────────────────────────────────────────────
-  # Simulate p95 > 1000ms: put most samples in the >1000 bucket.
+  # Simulate p95 > 1000ms. The highest finite bucket must be above the 1000ms
+  # threshold, otherwise histogram_quantile caps at le=1000 and the alert can
+  # never fire — so most samples land in the (1000, 2000] bucket.
   - interval: 30s
     input_series:
       - series: 'trivela_http_request_duration_ms_bucket{job="trivela-backend",le="500"}'
         values: '0+10x20'
       - series: 'trivela_http_request_duration_ms_bucket{job="trivela-backend",le="1000"}'
         values: '0+11x20'
+      - series: 'trivela_http_request_duration_ms_bucket{job="trivela-backend",le="2000"}'
+        values: '0+100x20'
       - series: 'trivela_http_request_duration_ms_bucket{job="trivela-backend",le="+Inf"}'
-        values: '0+100x20' # 89 out of 100 requests > 1000ms → p95 > 1000
+        values: '0+100x20' # p95 lands in (1000, 2000] → > 1000ms SLO breach
 
     alert_rule_test:
       - eval_time: 6m
@@ -59,6 +68,12 @@ tests:
           - exp_labels:
               severity: warning
               team: backend
+            exp_annotations:
+              summary: 'Trivela p95 request latency above 1 second'
+              description:
+                'The 95th-percentile request latency is 32m 23s — above the 1 s SLO target. Identify
+                slow routes in Grafana → Trivela API dashboard.'
+              runbook_url: 'https://github.com/FinesseStudioLab/Trivela/blob/main/docs/RUNBOOK.md#latency-investigation'
 
   # ── BackendDown ──────────────────────────────────────────────────────────────
   - interval: 30s
@@ -71,8 +86,14 @@ tests:
         alertname: BackendDown
         exp_alerts:
           - exp_labels:
+              job: trivela-backend
               severity: critical
               team: backend
+            exp_annotations:
+              summary: 'Trivela backend is unreachable'
+              description:
+                'Prometheus cannot scrape the backend /metrics endpoint. Service may be down.'
+              runbook_url: 'https://github.com/FinesseStudioLab/Trivela/blob/main/docs/RUNBOOK.md#backend-restart'
 
   # ── AllRpcEndpointsUnhealthy ────────────────────────────────────────────────
   - interval: 30s
@@ -85,8 +106,15 @@ tests:
         alertname: AllRpcEndpointsUnhealthy
         exp_alerts:
           - exp_labels:
+              job: trivela-backend
               severity: critical
               team: infrastructure
+            exp_annotations:
+              summary: 'All Soroban RPC endpoints are unhealthy'
+              description:
+                'Every endpoint in the RPC pool is marked unhealthy. Contract interactions will fail
+                or fall back to the first endpoint. Check RPC node health.'
+              runbook_url: 'https://github.com/FinesseStudioLab/Trivela/blob/main/docs/RUNBOOK.md#rpc-failover'
 
   # ── RpcPoolSaturated ────────────────────────────────────────────────────────
   - interval: 30s
@@ -99,8 +127,16 @@ tests:
         alertname: RpcPoolSaturated
         exp_alerts:
           - exp_labels:
+              job: trivela-backend
               severity: warning
               team: backend
+            exp_annotations:
+              summary: 'RPC pool is saturated — callers waiting'
+              description:
+                '3 caller(s) are queued waiting for an RPC pool slot. Requests beyond the acquire
+                timeout will receive 503 POOL_SATURATED. Consider increasing PG_POOL_MAX or scaling
+                the RPC tier.'
+              runbook_url: 'https://github.com/FinesseStudioLab/Trivela/blob/main/docs/RUNBOOK.md#rpc-pool-saturation'
 
   # ── DLQGrowth ───────────────────────────────────────────────────────────────
   - interval: 1m
@@ -113,8 +149,15 @@ tests:
         alertname: DLQGrowth
         exp_alerts:
           - exp_labels:
+              job: trivela-backend
               severity: warning
               team: backend
+            exp_annotations:
+              summary: 'Dead-letter queue is growing'
+              description:
+                '30 jobs added to the DLQ in the last 15 minutes. Background jobs are failing
+                repeatedly. Review failed job logs for root cause.'
+              runbook_url: 'https://github.com/FinesseStudioLab/Trivela/blob/main/docs/RUNBOOK.md#dlq-investigation'
 
   # ── OperatorLowBalance ──────────────────────────────────────────────────────
   - interval: 1m
@@ -127,8 +170,15 @@ tests:
         alertname: OperatorLowBalance
         exp_alerts:
           - exp_labels:
+              job: trivela-backend
               severity: warning
               team: contracts
+            exp_annotations:
+              summary: 'Operator wallet balance is low'
+              description:
+                'Operator XLM balance is 30M stroops (< 5 XLM). Transaction fees may fail. Top up
+                the operator wallet immediately.'
+              runbook_url: 'https://github.com/FinesseStudioLab/Trivela/blob/main/docs/RUNBOOK.md#operator-wallet-topup'
 
   # ── CanaryJourneyFailed ─────────────────────────────────────────────────────
   - interval: 1m
@@ -141,5 +191,12 @@ tests:
         alertname: CanaryJourneyFailed
         exp_alerts:
           - exp_labels:
+              job: trivela-canary
               severity: critical
               team: backend
+            exp_annotations:
+              summary: 'Synthetic canary journey failed'
+              description:
+                'The register→credit→claim canary on testnet has not succeeded for 5 minutes. Core
+                user journey is broken. Check canary logs and RPC/contract health.'
+              runbook_url: 'https://github.com/FinesseStudioLab/Trivela/blob/main/docs/RUNBOOK.md#canary-failure'