From 11894bc21a54cd11eed6c7bda1791e0411761aac Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 15 Feb 2021 17:47:57 +0100 Subject: [PATCH] Update Grafana dashboards (#8127) --- .../substrate-networking.json | 624 +++++++++--------- .../substrate-service-tasks.json | 43 +- 2 files changed, 347 insertions(+), 320 deletions(-) diff --git a/substrate/.maintain/monitoring/grafana-dashboards/substrate-networking.json b/substrate/.maintain/monitoring/grafana-dashboards/substrate-networking.json index d2abfd1cb8..0b157e7205 100644 --- a/substrate/.maintain/monitoring/grafana-dashboards/substrate-networking.json +++ b/substrate/.maintain/monitoring/grafana-dashboards/substrate-networking.json @@ -74,7 +74,7 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1610462565248, + "iteration": 1613393276921, "links": [], "panels": [ { @@ -963,7 +963,8 @@ "steppedLine": false, "targets": [ { - "expr": "irate(${metric_namespace}_sub_libp2p_requests_out_started_total{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])", + "expr": "irate(${metric_namespace}_sub_libp2p_requests_out_success_total_sum{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m]) + on(instance) sum(irate(${metric_namespace}_sub_libp2p_requests_out_failure_total{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance)", + "hide": false, "interval": "", "legendFormat": "{{instance}}", "refId": "A" @@ -989,6 +990,7 @@ }, "yaxes": [ { + "$$hashKey": "object:209", "format": "reqps", "label": null, "logBase": 1, @@ -997,6 +999,7 @@ "show": true }, { + "$$hashKey": "object:210", "format": "short", "label": null, "logBase": 1, @@ -1032,7 +1035,7 @@ "y": 51 }, "hiddenSeries": false, - "id": 151, + "id": 448, "legend": { "avg": false, "current": false, @@ -1060,9 +1063,11 @@ "steppedLine": false, "targets": [ { - "expr": "irate(${metric_namespace}_sub_libp2p_requests_in_success_total_count{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])", + "expr": "sum(irate(${metric_namespace}_sub_libp2p_requests_out_failure_total{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[$__rate_interval])) by (instance, reason)", + "hide": false, "interval": "", - "legendFormat": "{{instance}}", + "intervalFactor": 1, + "legendFormat": "{{reason}}", "refId": "A" } ], @@ -1070,7 +1075,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Requests served per second", + "title": "Outbound requests failures", "tooltip": { "shared": true, "sort": 2, @@ -1086,6 +1091,7 @@ }, "yaxes": [ { + "$$hashKey": "object:209", "format": "reqps", "label": null, "logBase": 1, @@ -1094,6 +1100,7 @@ "show": true }, { + "$$hashKey": "object:210", "format": "short", "label": null, "logBase": 1, @@ -1227,103 +1234,6 @@ "y": 59 }, "hiddenSeries": false, - "id": 258, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": "nodename", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.5, sum(rate(${metric_namespace}_sub_libp2p_requests_in_success_total_bucket{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance, le))", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Median request serving time", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 63 - }, - "hiddenSeries": false, "id": 257, "legend": { "avg": false, @@ -1400,6 +1310,103 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 151, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(${metric_namespace}_sub_libp2p_requests_in_success_total_count{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests served per second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -1422,6 +1429,204 @@ "y": 67 }, "hiddenSeries": false, + "id": 449, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(${metric_namespace}_sub_libp2p_requests_in_failure_total{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[$__rate_interval])) by (instance, reason)", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Inbound requests failures", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:209", + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:210", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 71 + }, + "hiddenSeries": false, + "id": 258, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum(rate(${metric_namespace}_sub_libp2p_requests_in_success_total_bucket{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance, le))", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median request serving time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 75 + }, + "hiddenSeries": false, "id": 259, "legend": { "avg": false, @@ -1497,202 +1702,6 @@ "alignLevel": null } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 71 - }, - "hiddenSeries": false, - "id": 287, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": "nodename", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(irate(${metric_namespace}_sub_libp2p_requests_out_failure_total{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (reason)", - "instant": false, - "interval": "", - "legendFormat": "{{reason}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Outgoing request failures per second", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 75 - }, - "hiddenSeries": false, - "id": 286, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": "nodename", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(irate(${metric_namespace}_sub_libp2p_requests_in_failure_total{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (reason)", - "instant": false, - "interval": "", - "legendFormat": "{{reason}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Ingoing request failures per second", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, { "collapsed": false, "datasource": null, @@ -1845,7 +1854,7 @@ "lines": true, "linewidth": 1, "maxPerRow": 12, - "nullPointMode": "null as zero", + "nullPointMode": "connected", "options": { "alertThreshold": true }, @@ -1871,7 +1880,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg by (direction) (irate(${metric_namespace}_sub_libp2p_notifications_sizes_count{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[$__interval]))", + "expr": "avg by (direction) (irate(${metric_namespace}_sub_libp2p_notifications_sizes_count{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[$__rate_interval]))", "interval": "", "legendFormat": "{{direction}}", "refId": "A" @@ -1958,7 +1967,7 @@ "lines": true, "linewidth": 1, "maxPerRow": 12, - "nullPointMode": "null as zero", + "nullPointMode": "connected", "options": { "alertThreshold": true }, @@ -1984,7 +1993,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(irate(${metric_namespace}_sub_libp2p_notifications_sizes_sum{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[$__interval])) by (direction)", + "expr": "avg(irate(${metric_namespace}_sub_libp2p_notifications_sizes_sum{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[$__rate_interval])) by (direction)", "instant": false, "interval": "", "legendFormat": "{{direction}}", @@ -2674,7 +2683,7 @@ "allValue": null, "current": {}, "datasource": "$data_source", - "definition": "${metric_namespace}_sub_libp2p_requests_out_started_total{instance=~\"${nodename}\"}", + "definition": "${metric_namespace}_sub_libp2p_requests_in_success_total_count{instance=~\"${nodename}\"}", "error": null, "hide": 2, "includeAll": true, @@ -2682,7 +2691,7 @@ "multi": false, "name": "request_protocol", "options": [], - "query": "${metric_namespace}_sub_libp2p_requests_out_started_total{instance=~\"${nodename}\"}", + "query": "${metric_namespace}_sub_libp2p_requests_in_success_total_count{instance=~\"${nodename}\"}", "refresh": 1, "regex": "/protocol=\"(.*?)\"/", "skipUrlSync": false, @@ -2707,6 +2716,7 @@ "name": "data_source", "options": [], "query": "prometheus", + "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -2756,5 +2766,5 @@ "timezone": "utc", "title": "Substrate Networking", "uid": "vKVuiD9Zk", - "version": 147 -} + "version": 154 +} \ No newline at end of file diff --git a/substrate/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json b/substrate/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json index a3db46ec6d..944c9fb50c 100644 --- a/substrate/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json +++ b/substrate/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json @@ -37,6 +37,7 @@ "annotations": { "list": [ { + "$$hashKey": "object:326", "builtIn": 1, "datasource": "-- Grafana --", "enable": true, @@ -48,6 +49,7 @@ "type": "dashboard" }, { + "$$hashKey": "object:327", "datasource": "$data_source", "enable": true, "expr": "increase(${metric_namespace}_tasks_ended_total{reason=\"panic\", instance=~\"${nodename}\"}[10m])", @@ -64,6 +66,7 @@ "type": "tags" }, { + "$$hashKey": "object:621", "datasource": "$data_source", "enable": true, "expr": "changes(${metric_namespace}_process_start_time_seconds{instance=~\"${nodename}\"}[10m])", @@ -81,7 +84,7 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1610462629581, + "iteration": 1613393319015, "links": [], "panels": [ { @@ -164,7 +167,7 @@ }, "lines": false, "linewidth": 1, - "nullPointMode": "null as zero", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -180,7 +183,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(${metric_namespace}_tasks_polling_duration_sum{instance=~\"${nodename}\"}[10m])", + "expr": "irate(${metric_namespace}_tasks_polling_duration_sum{instance=~\"${nodename}\"}[$__rate_interval])", "interval": "", "legendFormat": "{{task_name}}", "refId": "A" @@ -206,6 +209,7 @@ }, "yaxes": [ { + "$$hashKey": "object:2721", "format": "percentunit", "label": null, "logBase": 1, @@ -214,6 +218,7 @@ "show": true }, { + "$$hashKey": "object:2722", "format": "short", "label": null, "logBase": 1, @@ -266,7 +271,7 @@ }, "lines": true, "linewidth": 2, - "nullPointMode": "null", + "nullPointMode": "connected", "options": { "alertThreshold": true }, @@ -282,7 +287,7 @@ "steppedLine": true, "targets": [ { - "expr": "irate(${metric_namespace}_tasks_polling_duration_count{instance=~\"${nodename}\"}[10m])", + "expr": "irate(${metric_namespace}_tasks_polling_duration_count{instance=~\"${nodename}\"}[$__rate_interval])", "interval": "", "legendFormat": "{{task_name}}", "refId": "A" @@ -308,6 +313,7 @@ }, "yaxes": [ { + "$$hashKey": "object:2571", "format": "cps", "label": null, "logBase": 1, @@ -316,6 +322,7 @@ "show": true }, { + "$$hashKey": "object:2572", "format": "short", "label": null, "logBase": 1, @@ -382,7 +389,7 @@ "steppedLine": true, "targets": [ { - "expr": "irate(${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"}[10m])", + "expr": "irate(${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"}[$__rate_interval])", "interval": "", "legendFormat": "{{task_name}}", "refId": "A" @@ -408,6 +415,7 @@ }, "yaxes": [ { + "$$hashKey": "object:771", "format": "short", "label": null, "logBase": 10, @@ -416,6 +424,7 @@ "show": true }, { + "$$hashKey": "object:772", "format": "short", "label": null, "logBase": 1, @@ -466,7 +475,7 @@ }, "lines": true, "linewidth": 1, - "nullPointMode": "null as zero", + "nullPointMode": "connected", "options": { "alertThreshold": true }, @@ -508,6 +517,7 @@ }, "yaxes": [ { + "$$hashKey": "object:919", "format": "short", "label": null, "logBase": 10, @@ -516,6 +526,7 @@ "show": true }, { + "$$hashKey": "object:920", "format": "short", "label": null, "logBase": 1, @@ -585,7 +596,7 @@ "steppedLine": true, "targets": [ { - "expr": "irate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"+Inf\"}[10m])\n - ignoring(le)\n irate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"1.024\"}[10m]) > 0", + "expr": "irate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"+Inf\"}[$__rate_interval])\n - ignoring(le)\n irate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"1.024\"}[$__rate_interval]) > 0", "interval": "", "legendFormat": "{{task_name}}", "refId": "A" @@ -611,6 +622,7 @@ }, "yaxes": [ { + "$$hashKey": "object:3040", "decimals": null, "format": "cps", "label": "Calls to `Future::poll`/second", @@ -620,6 +632,7 @@ "show": true }, { + "$$hashKey": "object:3041", "format": "short", "label": null, "logBase": 1, @@ -683,7 +696,7 @@ }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "connected", "options": { "alertThreshold": true }, @@ -725,6 +738,7 @@ }, "yaxes": [ { + "$$hashKey": "object:626", "format": "short", "label": null, "logBase": 1, @@ -733,6 +747,7 @@ "show": true }, { + "$$hashKey": "object:627", "format": "short", "label": null, "logBase": 1, @@ -782,7 +797,7 @@ }, "lines": true, "linewidth": 1, - "nullPointMode": "null", + "nullPointMode": "connected", "options": { "alertThreshold": true }, @@ -798,7 +813,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"send\"}[10m])", + "expr": "irate(${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"send\"}[$__rate_interval])", "interval": "", "legendFormat": "{{entity}}", "refId": "B" @@ -824,6 +839,7 @@ }, "yaxes": [ { + "$$hashKey": "object:626", "format": "cps", "label": null, "logBase": 1, @@ -832,6 +848,7 @@ "show": true }, { + "$$hashKey": "object:627", "format": "short", "label": null, "logBase": 1, @@ -938,5 +955,5 @@ "timezone": "utc", "title": "Substrate Service Tasks", "uid": "3LA6XNqZz", - "version": 59 -} + "version": 60 +} \ No newline at end of file