dashboard: fix Prometheus TDG moving average

DifferentialOrange · DifferentialOrange · commit d4288a113b99 · 2024-12-06T16:24:00.000+03:00
Before this patch, Prometheus average panels (histogram or summary `metric_sum / metric_count`) were displaying instance runtime average info. This info is rather useless since, in case of instance lifetime being longer than several dozens of metrics collect intervals (which happens often), computed average becomes almost constant. Prometheus itself advises to use moving average for this. InfluxDB panels already implement moving averages, so there's no need to update them. 1. https://prometheus.io/docs/practices/histograms/#count-and-sum-of-observations Closes #237
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Fixed
+- Prometheus TDG average panels displaying lifetime average instead of a moving one (#237)
+
+
 ## [3.2.0] - 2024-10-04
 Grafana revisions:
 - Tarantool 3:
diff --git a/dashboard/panels/tdg/graphql.libsonnet b/dashboard/panels/tdg/graphql.libsonnet
@@ -37,7 +37,7 @@ local prometheus = grafana.prometheus;
       prometheus.target(
         expr=std.format(
           |||
-            %(metrics_prefix)s%(metric_name_sum)s{%(filters)s} / %(metrics_prefix)s%(metric_name_count)s{%(filters)s}
+            rate(%(metrics_prefix)s%(metric_name_sum)s{%(filters)s}[$__rate_interval]) / rate(%(metrics_prefix)s%(metric_name_count)s{%(filters)s})[$__rate_interval])
           |||,
           {
             metrics_prefix: cfg.metrics_prefix,
diff --git a/dashboard/panels/tdg/tasks.libsonnet b/dashboard/panels/tdg/tasks.libsonnet
@@ -83,7 +83,7 @@ local prometheus = grafana.prometheus;
       prometheus.target(
         expr=std.format(
           |||
-            %(metrics_prefix)s%(metric_name_sum)s{%(filters)s} / %(metrics_prefix)s%(metric_name_count)s{%(filters)s}
+            rate(%(metrics_prefix)s%(metric_name_sum)s{%(filters)s}[$__rate_interval]) / rate(%(metrics_prefix)s%(metric_name_count)s{%(filters)s}[$__rate_interval])
           |||,
           {
             metrics_prefix: cfg.metrics_prefix,
@@ -196,7 +196,7 @@ local prometheus = grafana.prometheus;
       prometheus.target(
         expr=std.format(
           |||
-            %(metrics_prefix)s%(metric_name_sum)s{%(filters)s} / %(metrics_prefix)s%(metric_name_count)s{%(filters)s}
+            rate(%(metrics_prefix)s%(metric_name_sum)s{%(filters)s}[$__rate_interval]) / rate(%(metrics_prefix)s%(metric_name_count)s{%(filters)s}[$__rate_interval])
           |||,
           {
             metrics_prefix: cfg.metrics_prefix,
diff --git a/dashboard/panels/tdg/tuples.libsonnet b/dashboard/panels/tdg/tuples.libsonnet
@@ -18,7 +18,7 @@ local prometheus = grafana.prometheus;
       prometheus.target(
         expr=std.format(
           |||
-            %(metrics_prefix)s%(metric_name_sum)s{%(filters)s} / %(metrics_prefix)s%(metric_name_count)s{%(filters)s}
+            rate(%(metrics_prefix)s%(metric_name_sum)s{%(filters)s}[$__rate_interval]) / rate(%(metrics_prefix)s%(metric_name_count)s{%(filters)s}[$__rate_interval])
           |||,
           {
             metrics_prefix: cfg.metrics_prefix,
diff --git a/tests/Prometheus/dashboard_tdg_compiled.json b/tests/Prometheus/dashboard_tdg_compiled.json
@@ -18917,7 +18917,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "tdg_scanned_tuples_sum{alias=~\"$alias\",job=~\"$job\"} / tdg_scanned_tuples_count{alias=~\"$alias\",job=~\"$job\"}\n",
+                     "expr": "rate(tdg_scanned_tuples_sum{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval]) / rate(tdg_scanned_tuples_count{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval])\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{type_name}} — {{alias}}",
@@ -19004,7 +19004,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "tdg_returned_tuples_sum{alias=~\"$alias\",job=~\"$job\"} / tdg_returned_tuples_count{alias=~\"$alias\",job=~\"$job\"}\n",
+                     "expr": "rate(tdg_returned_tuples_sum{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval]) / rate(tdg_returned_tuples_count{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval])\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{type_name}} — {{alias}}",
@@ -19932,7 +19932,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "tdg_graphql_query_time_sum{alias=~\"$alias\",job=~\"$job\"} / tdg_graphql_query_time_count{alias=~\"$alias\",job=~\"$job\"}\n",
+                     "expr": "rate(tdg_graphql_query_time_sum{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval]) / rate(tdg_graphql_query_time_count{alias=~\"$alias\",job=~\"$job\"})[$__rate_interval])\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{operation_name}} ({{schema}}, {{entity}}) — {{alias}}",
@@ -20193,7 +20193,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "tdg_graphql_mutation_time_sum{alias=~\"$alias\",job=~\"$job\"} / tdg_graphql_mutation_time_count{alias=~\"$alias\",job=~\"$job\"}\n",
+                     "expr": "rate(tdg_graphql_mutation_time_sum{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval]) / rate(tdg_graphql_mutation_time_count{alias=~\"$alias\",job=~\"$job\"})[$__rate_interval])\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{operation_name}} ({{schema}}, {{entity}}) — {{alias}}",
@@ -23385,7 +23385,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "tdg_jobs_execution_time_sum{alias=~\"$alias\",job=~\"$job\"} / tdg_jobs_execution_time_count{alias=~\"$alias\",job=~\"$job\"}\n",
+                     "expr": "rate(tdg_jobs_execution_time_sum{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval]) / rate(tdg_jobs_execution_time_count{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval])\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{name}} — {{alias}}",
@@ -23907,7 +23907,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "tdg_tasks_execution_time_sum{alias=~\"$alias\",job=~\"$job\"} / tdg_tasks_execution_time_count{alias=~\"$alias\",job=~\"$job\"}\n",
+                     "expr": "rate(tdg_tasks_execution_time_sum{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval]) / rate(tdg_tasks_execution_time_count{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval])\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{name}} ({{kind}}) — {{alias}}",
@@ -24342,7 +24342,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "tdg_system_tasks_execution_time_sum{alias=~\"$alias\",job=~\"$job\"} / tdg_system_tasks_execution_time_count{alias=~\"$alias\",job=~\"$job\"}\n",
+                     "expr": "rate(tdg_system_tasks_execution_time_sum{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval]) / rate(tdg_system_tasks_execution_time_count{alias=~\"$alias\",job=~\"$job\"}[$__rate_interval])\n",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{name}} ({{kind}}) — {{alias}}",

Original file line number	Diff line number	Diff line change
`@@ -37,7 +37,7 @@ local prometheus = grafana.prometheus;`
`37`	`37`	`prometheus.target(`
`38`	`38`	`expr=std.format(`
`39`	`39`	`\|\|\|`
`40`		`- %(metrics_prefix)s%(metric_name_sum)s{%(filters)s} / %(metrics_prefix)s%(metric_name_count)s{%(filters)s}`
	`40`	`+ rate(%(metrics_prefix)s%(metric_name_sum)s{%(filters)s}[$__rate_interval]) / rate(%(metrics_prefix)s%(metric_name_count)s{%(filters)s})[$__rate_interval])`
`41`	`41`	`\|\|\|,`
`42`	`42`	`{`
`43`	`43`	`metrics_prefix: cfg.metrics_prefix,`
Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@ local prometheus = grafana.prometheus;`
`18`	`18`	`prometheus.target(`
`19`	`19`	`expr=std.format(`
`20`	`20`	`\|\|\|`
`21`		`- %(metrics_prefix)s%(metric_name_sum)s{%(filters)s} / %(metrics_prefix)s%(metric_name_count)s{%(filters)s}`
	`21`	`+ rate(%(metrics_prefix)s%(metric_name_sum)s{%(filters)s}[$__rate_interval]) / rate(%(metrics_prefix)s%(metric_name_count)s{%(filters)s}[$__rate_interval])`
`22`	`22`	`\|\|\|,`
`23`	`23`	`{`
`24`	`24`	`metrics_prefix: cfg.metrics_prefix,`