cert-manager-mixin/tests.yaml at main · imusmanmalik/cert-manager-mixin · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
---
rule_files:
  - manifests/prometheus_alerts.yaml
  - manifests/prometheus_rules.yaml

evaluation_interval: 1m

tests:
  # Absent metrics
  - interval: 1m
    input_series:
    alert_rule_test:
      - eval_time: 10m
        alertname: CertManagerAbsent
        exp_alerts:
          - exp_labels:
              severity: critical
              job: cert-manager
            exp_annotations:
              summary: "Cert Manager has disappeared from Prometheus service discovery."
              description: "New certificates will not be able to be minted, and existing ones can't be renewed until cert-manager is back."
              runbook_url: "https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#certmanagerabsent"

  # Cert expiry
  - interval: 1m
    input_series:
      # Short-lived cert: expires in 2h, renewal starts at 30m → renewal window 1.5h
      # Threshold = 1.5h * 0.7 = 63m. Fires when time_to_expiry < 63m, i.e. at t=57m+.
      # With for: 1h, alert fires at ~118m when time_to_expiry = 2m.
      - series: certmanager_certificate_expiration_timestamp_seconds{namespace="cert-manager", exported_namespace="test", name="expired-ingress-cert", foo="bar"}
        values: 7200+0x200
      - series: certmanager_certificate_renewal_timestamp_seconds{namespace="cert-manager", exported_namespace="test", name="expired-ingress-cert", foo="bar"}
        values: 1800+0x200
      - series: certmanager_certificate_ready_status{namespace="cert-manager", exported_namespace="test", name="expired-ingress-cert", condition="True"}
        values: 1+0x200
      # Long-lived cert: expires in 90d, renewal at 60d → renewal window 30d
      # Threshold = 30d * 0.7 = 21d. At t=118m, time_to_expiry ≈ 89.99d >> 21d → does not fire.
      - series: certmanager_certificate_expiration_timestamp_seconds{namespace="cert-manager", exported_namespace="test", name="90d-ingress-cert"}
        values: 7776000+0x200
      - series: certmanager_certificate_renewal_timestamp_seconds{namespace="cert-manager", exported_namespace="test", name="90d-ingress-cert"}
        values: 5184000+0x200
      - series: certmanager_certificate_ready_status{namespace="cert-manager", exported_namespace="test", name="90d-ingress-cert", condition="True"}
        values: 1+0x200
    alert_rule_test:
      - eval_time: 118m
        alertname: CertManagerCertExpirySoon
        exp_alerts:
          - exp_labels:
              severity: warning
              exported_namespace: test
              namespace: cert-manager
              name: expired-ingress-cert
            exp_annotations:
              summary: The cert `expired-ingress-cert` is 2m 0s from expiry, it should have been renewed by now.
              description: "The domain that this cert covers will be unavailable after 2m 0s. Clients using endpoints that this cert protects will start to fail in 2m 0s."
              dashboard_url: https://grafana.example.com/d/TvuRo2iMk/cert-manager
              runbook_url: "https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#certmanagercertexpirysoon"

  # Cert not ready
  - interval: 1m
    input_series:
      - series: certmanager_certificate_ready_status{namespace="cert-manager", exported_namespace="test", name="ready", condition="True"}
        values: 1+0x30
      - series: certmanager_certificate_ready_status{namespace="cert-manager", exported_namespace="test", name="not ready", condition="False"}
        values: 1+0x30
      - series: certmanager_certificate_ready_status{namespace="cert-manager", exported_namespace="test", name="who knows", condition="Unknown"}
        values: 1+0x30
    alert_rule_test:
      - eval_time: 10m
        alertname: CertManagerCertNotReady
        exp_alerts:
          - exp_labels:
              severity: critical
              exported_namespace: test
              namespace: cert-manager
              name: not ready
              condition: "False"
            exp_annotations:
              summary: The cert `not ready` is not ready to serve traffic.
              description:
                "This certificate has not been ready to serve traffic for at least 10m. If the cert is being renewed or there is another valid cert,
                the ingress controller _may_ be able to serve that instead."
              dashboard_url: https://grafana.example.com/d/TvuRo2iMk/cert-manager
              runbook_url: "https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#certmanagercertnotready"
          - exp_labels:
              severity: critical
              exported_namespace: test
              namespace: cert-manager
              name: who knows
              condition: "Unknown"
            exp_annotations:
              summary: The cert `who knows` is not ready to serve traffic.
              description:
                "This certificate has not been ready to serve traffic for at least 10m. If the cert is being renewed or there is another valid cert,
                the ingress controller _may_ be able to serve that instead."
              dashboard_url: https://grafana.example.com/d/TvuRo2iMk/cert-manager
              runbook_url: "https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#certmanagercertnotready"

  # cert-manager rate limits
  - interval: 1m
    input_series:
      - series: certmanager_http_acme_client_request_count{status="200", host="normal.acme-v02.api.letsencrypt.org", path="/acme/new-order"}
        values: 1+1x30
      - series: certmanager_http_acme_client_request_count{status="429", host="rate-limited.acme-v02.api.letsencrypt.org", path="/acme/new-order"}
        values: 1+1x30
      - series: certmanager_http_acme_client_request_count{status="429", host="one-limited-request.acme-v02.api.letsencrypt.org", path="/acme/new-order"}
        values: 1+0x30
    alert_rule_test:
      - eval_time: 10m
        alertname: CertManagerHittingRateLimits
        exp_alerts:
          - exp_labels:
              severity: critical
              host: rate-limited.acme-v02.api.letsencrypt.org
            exp_annotations:
              summary: "Cert manager hitting LetsEncrypt rate limits."
              description: "Depending on the rate limit, cert-manager may be unable to generate certificates for up to a week."
              runbook_url: "https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#certmanagerhittingratelimits"
              dashboard_url: https://grafana.example.com/d/TvuRo2iMk/cert-manager