diff --git a/substrate/.maintain/monitoring/grafana-dashboards/substrate-networking.json b/substrate/.maintain/monitoring/grafana-dashboards/substrate-networking.json index dfc1430054..d2abfd1cb8 100644 --- a/substrate/.maintain/monitoring/grafana-dashboards/substrate-networking.json +++ b/substrate/.maintain/monitoring/grafana-dashboards/substrate-networking.json @@ -1,13 +1,5 @@ { "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - }, { "name": "VAR_METRIC_NAMESPACE", "type": "constant", @@ -17,11 +9,17 @@ } ], "__requires": [ + { + "type": "panel", + "id": "dashlist", + "name": "Dashboard list", + "version": "" + }, { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "6.7.3" + "version": "7.3.6" }, { "type": "panel", @@ -29,17 +27,17 @@ "name": "Graph", "version": "" }, - { - "type": "panel", - "id": "heatmap", - "name": "Heatmap", - "version": "" - }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" } ], "annotations": { @@ -76,65 +74,254 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1600780210197, + "iteration": 1610462565248, "links": [], "panels": [ { + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 308, + "options": { + "content": "", + "mode": "markdown" + }, + "pluginVersion": "7.3.6", + "repeat": "nodename", + "timeFrom": null, + "timeShift": null, + "title": "$nodename", + "type": "text" + }, + { + "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 0 + "y": 1 }, - "id": 167, - "title": "Sync", + "id": 27, + "panels": [], + "title": "Transport", "type": "row" }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 19, + "interval": "1m", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "maxPerRow": 12, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "repeatDirection": "h", + "seriesOverrides": [ + { + "alias": "established (in)", + "color": "#37872D" + }, + { + "alias": "established (out)", + "color": "#C4162A" + }, + { + "alias": "pending (out)", + "color": "#FF7383" + }, + { + "alias": "closed-recently", + "color": "#FADE2A", + "steppedLine": true + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum(${metric_namespace}_sub_libp2p_connections_opened_total{direction=\"in\", instance=~\"${nodename}\"}) by (instance) -\n sum(${metric_namespace}_sub_libp2p_connections_closed_total{direction=\"in\", instance=~\"${nodename}\"}) by (instance)\n)\n\n# Because `closed_total` can be null, this serves as fallback\nor on(instance) sum(${metric_namespace}_sub_libp2p_connections_opened_total{direction=\"in\", instance=~\"${nodename}\"}) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "established (in)", + "refId": "A" + }, + { + "expr": "(\n sum(${metric_namespace}_sub_libp2p_connections_opened_total{direction=\"out\", instance=~\"${nodename}\"}) by (instance) -\n sum(${metric_namespace}_sub_libp2p_connections_closed_total{direction=\"out\", instance=~\"${nodename}\"}) by (instance)\n)\n\n# Because `closed_total` can be null, this serves as fallback\nor on(instance) sum(${metric_namespace}_sub_libp2p_connections_opened_total{direction=\"out\", instance=~\"${nodename}\"}) by (instance)", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "established (out)", + "refId": "C" + }, + { + "expr": "sum by (instance) (${metric_namespace}_sub_libp2p_pending_connections{instance=~\"${nodename}\"})", + "hide": false, + "interval": "", + "legendFormat": "pending (out)", + "refId": "B" + }, + { + "expr": "sum(rate(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}[$__interval]))", + "hide": false, + "interval": "", + "legendFormat": "closed-per-sec", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average transport-level (TCP, QUIC, ...) connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Connections", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { - "h": 5, - "w": 24, + "h": 6, + "w": 12, "x": 0, - "y": 1 + "y": 8 }, "hiddenSeries": false, - "id": 101, + "id": 189, + "interval": "1m", "legend": { + "alignAsTable": false, "avg": false, "current": false, + "hideEmpty": false, "max": false, "min": false, - "show": false, + "rightSide": false, + "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, - "nullPointMode": "connected", + "maxPerRow": 12, + "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", - "repeat": null, - "repeatDirection": "v", + "repeat": "nodename", + "repeatDirection": "h", "seriesOverrides": [], "spaceLength": 10, "stack": false, - "steppedLine": true, + "steppedLine": false, "targets": [ { - "expr": "1 - (${metric_namespace}_sub_libp2p_peerset_num_requested{instance=~\"${nodename}\"} - ${metric_namespace}_sub_libp2p_peers_count{instance=~\"${nodename}\"}) / ${metric_namespace}_sub_libp2p_peerset_num_requested{instance=~\"${nodename}\"}", + "expr": "1 - \n\navg(\n ${metric_namespace}_sub_libp2p_distinct_peers_connections_opened_total{instance=~\"${nodename}\"} - ${metric_namespace}_sub_libp2p_distinct_peers_connections_closed_total{instance=~\"${nodename}\"}\n) by (instance)\n\n/\n\navg(\r\n sum(${metric_namespace}_sub_libp2p_connections_opened_total{instance=~\"${nodename}\"}) by (instance) - sum(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}) by (instance)\r\n) by (instance)", + "format": "time_series", + "hide": false, "interval": "", "legendFormat": "{{instance}}", "refId": "A" @@ -144,10 +331,10 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Number of peer slots filled", + "title": "Percentage of peers for which we have more than one connection open", "tooltip": { "shared": true, - "sort": 1, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -161,9 +348,420 @@ "yaxes": [ { "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", "label": null, "logBase": 1, - "max": "1.0", + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 39, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [ + { + "alias": "/.*/", + "color": "#FF780A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(${metric_namespace}_sub_libp2p_incoming_connections_handshake_errors_total{instance=~\"${nodename}\"}[$__interval])", + "hide": false, + "interval": "", + "legendFormat": "{{reason}}", + "refId": "A" + }, + { + "expr": "rate(${metric_namespace}_sub_libp2p_listeners_errors_total{instance=~\"${nodename}\"}[$__interval])", + "interval": "", + "legendFormat": "pre-handshake", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of incoming connection errors", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Errors", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "description": "Each bucket represent a certain number of nodes using a certain bandwidth range.", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(${metric_namespace}_sub_libp2p_network_bytes_total{instance=~\"${nodename}\"}[5m])", + "interval": "", + "legendFormat": "{{direction}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network bandwidth - # bytes per second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 81, + "interval": "1m", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(${metric_namespace}_sub_libp2p_pending_connections_errors_total{instance=~\"${nodename}\"}[$__interval])", + "interval": "", + "legendFormat": "{{reason}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Dialing attempt errors", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "hiddenSeries": false, + "id": 46, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "maxPerRow": 12, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}[$__interval])", + "interval": "", + "legendFormat": "{{reason}} ({{direction}})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disconnects", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "Disconnects", + "logBase": 1, + "max": null, "min": null, "show": true }, @@ -188,7 +786,126 @@ "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 40 + }, + "id": 167, + "panels": [], + "repeat": null, + "title": "Sync", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "${metric_namespace}_sub_libp2p_peerset_num_requested{instance=~\"${nodename}\"}", + "interval": "", + "legendFormat": "peers-requested", + "refId": "A" + }, + { + "expr": "polkadot_sub_libp2p_peers_count{instance=~\"${nodename}.*\"}", + "interval": "", + "legendFormat": "peers-count", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of peer slots filled", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 }, "id": 29, "panels": [], @@ -202,13 +919,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 0, - "y": 7 + "y": 47 }, "hiddenSeries": false, "id": 148, @@ -225,12 +949,14 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -290,13 +1016,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 12, - "y": 7 + "x": 0, + "y": 51 }, "hiddenSeries": false, "id": 151, @@ -313,12 +1046,14 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -378,13 +1113,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 0, - "y": 11 + "y": 55 }, "hiddenSeries": false, "id": 256, @@ -401,12 +1143,14 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -467,13 +1211,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 12, - "y": 11 + "x": 0, + "y": 59 }, "hiddenSeries": false, "id": 258, @@ -490,12 +1241,14 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -555,13 +1308,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 0, - "y": 15 + "y": 63 }, "hiddenSeries": false, "id": 257, @@ -578,12 +1338,14 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -644,13 +1406,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 12, - "y": 15 + "x": 0, + "y": 67 }, "hiddenSeries": false, "id": 259, @@ -667,12 +1436,14 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -732,13 +1503,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 0, - "y": 19 + "y": 71 }, "hiddenSeries": false, "id": 287, @@ -755,12 +1533,14 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -821,13 +1601,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, - "x": 12, - "y": 19 + "x": 0, + "y": 75 }, "hiddenSeries": false, "id": 286, @@ -844,12 +1631,14 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -911,7 +1700,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 79 }, "id": 23, "panels": [], @@ -925,13 +1714,120 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 80 + }, + "hiddenSeries": false, + "id": 447, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "${metric_namespace}_sub_libp2p_notifications_streams_opened_total{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"} - ${metric_namespace}_sub_libp2p_notifications_streams_closed_total{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of open substreams", + "tooltip": { + "shared": false, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 0, - "y": 41 + "y": 86 }, "hiddenSeries": false, "id": 31, @@ -948,17 +1844,18 @@ }, "lines": true, "linewidth": 1, - "maxPerRow": 2, + "maxPerRow": 12, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", - "repeat": null, - "repeatDirection": "v", + "repeat": "nodename", + "repeatDirection": "h", "seriesOverrides": [ { "alias": "/(in)/", @@ -1027,13 +1924,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { - "h": 7, + "h": 6, "w": 12, - "x": 12, - "y": 41 + "x": 0, + "y": 92 }, "hiddenSeries": false, "id": 37, @@ -1053,17 +1957,18 @@ }, "lines": true, "linewidth": 1, - "maxPerRow": 2, + "maxPerRow": 12, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", - "repeat": null, - "repeatDirection": "v", + "repeat": "nodename", + "repeatDirection": "h", "seriesOverrides": [ { "alias": "/(in)/", @@ -1133,13 +2038,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, - "y": 48 + "y": 98 }, "hiddenSeries": false, "id": 16, @@ -1156,12 +2068,14 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -1221,13 +2135,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 1, "gridPos": { "h": 6, "w": 12, - "x": 12, - "y": 48 + "x": 0, + "y": 104 }, "hiddenSeries": false, "id": 21, @@ -1244,13 +2165,14 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, - "pluginVersion": "6.4.5", + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, @@ -1305,6 +2227,104 @@ "alignLevel": null } }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 110 + }, + "hiddenSeries": false, + "id": 134, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "nodename", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(1.0, sum(rate(${metric_namespace}_sub_libp2p_notifications_sizes_bucket{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[5m])) by (direction, le))", + "format": "time_series", + "interval": "", + "legendFormat": "{{direction}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Maximum size of sent and received notifications in the past 5 minutes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Max. notification size", + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, { "aliasColors": {}, "bars": false, @@ -1312,13 +2332,20 @@ "dashes": false, "datasource": "$data_source", "description": "99.9% of the time, the output queue size for this protocol is below the given value", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, - "y": 54 + "y": 116 }, "hiddenSeries": false, "id": 14, @@ -1339,12 +2366,14 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [ { "alias": "max", @@ -1411,96 +2440,6 @@ "alignLevel": null } }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 1, - "fillGradient": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 54 - }, - "hiddenSeries": false, - "id": 134, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "6.4.5", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(1.0, sum(rate(${metric_namespace}_sub_libp2p_notifications_sizes_bucket{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[5m])) by (direction, le))", - "format": "time_series", - "interval": "", - "legendFormat": "{{direction}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Maximum size of sent and received notifications in the past 5 minutes", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "Max. notification size", - "logBase": 10, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, { "collapsed": false, "datasource": null, @@ -1508,608 +2447,7 @@ "h": 1, "w": 24, "x": 0, - "y": 60 - }, - "id": 27, - "panels": [], - "title": "Transport", - "type": "row" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 61 - }, - "hiddenSeries": false, - "id": 19, - "interval": "1m", - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "maxPerRow": 2, - "nullPointMode": "null as zero", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": null, - "repeatDirection": "v", - "seriesOverrides": [ - { - "alias": "established (in)", - "color": "#37872D" - }, - { - "alias": "established (out)", - "color": "#C4162A" - }, - { - "alias": "pending (out)", - "color": "#FF7383" - }, - { - "alias": "closed-recently", - "color": "#FADE2A", - "steppedLine": true - } - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "avg(sum(${metric_namespace}_sub_libp2p_connections_opened_total{direction=\"in\", instance=~\"${nodename}\"}) by (instance) - sum(${metric_namespace}_sub_libp2p_connections_closed_total{direction=\"in\", instance=~\"${nodename}\"}) by (instance))", - "format": "time_series", - "hide": false, - "interval": "", - "legendFormat": "established (in)", - "refId": "A" - }, - { - "expr": "avg(sum(${metric_namespace}_sub_libp2p_connections_opened_total{direction=\"out\", instance=~\"${nodename}\"}) by (instance) - sum(${metric_namespace}_sub_libp2p_connections_closed_total{direction=\"out\", instance=~\"${nodename}\"}) by (instance))", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "established (out)", - "refId": "C" - }, - { - "expr": "avg(sum by (instance) (${metric_namespace}_sub_libp2p_pending_connections{instance=~\"${nodename}\"}))", - "hide": false, - "interval": "", - "legendFormat": "pending (out)", - "refId": "B" - }, - { - "expr": "avg(sum by(instance) (increase(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}[$__interval])))", - "hide": false, - "interval": "", - "legendFormat": "closed-recently", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Average transport-level (TCP, QUIC, ...) connections per node", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Connections", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 67 - }, - "hiddenSeries": false, - "id": 189, - "interval": "1m", - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "maxPerRow": 2, - "nullPointMode": "null as zero", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "1 - \n\navg(\n ${metric_namespace}_sub_libp2p_distinct_peers_connections_opened_total{instance=~\"${nodename}\"} - ${metric_namespace}_sub_libp2p_distinct_peers_connections_closed_total{instance=~\"${nodename}\"}\n) by (instance)\n\n/\n\navg(\r\n sum(${metric_namespace}_sub_libp2p_connections_opened_total{instance=~\"${nodename}\"}) by (instance) - sum(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}) by (instance)\r\n) by (instance)", - "format": "time_series", - "hide": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Percentage of peers for which we have more than one connection open", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 73 - }, - "hiddenSeries": false, - "id": 39, - "interval": "1m", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*/", - "color": "#FF780A" - } - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "avg(increase(${metric_namespace}_sub_libp2p_incoming_connections_handshake_errors_total{instance=~\"${nodename}\"}[$__interval])) by (reason)", - "interval": "", - "legendFormat": "{{reason}}", - "refId": "A" - }, - { - "expr": "avg(increase(${metric_namespace}_sub_libp2p_listeners_errors_total{instance=~\"${nodename}\"}[$__interval]))", - "interval": "", - "legendFormat": "pre-handshake", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Number of incoming connection errors, averaged by node", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Errors", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "max": 100, - "min": 0, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "datasource": "$data_source", - "description": "Each bucket represent a certain number of nodes using a certain bandwidth range.", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 73 - }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 4, - "legend": { - "show": false - }, - "reverseYBuckets": false, - "targets": [ - { - "expr": "${metric_namespace}_network_per_sec_bytes{instance=~\"${nodename}\"}", - "format": "time_series", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Heatmap of network bandwidth", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": "2.5m", - "yAxis": { - "decimals": null, - "format": "Bps", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 79 - }, - "hiddenSeries": false, - "id": 81, - "interval": "1m", - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": null, - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "avg(increase(${metric_namespace}_sub_libp2p_pending_connections_errors_total{instance=~\"${nodename}\"}[$__interval])) by (reason)", - "interval": "", - "legendFormat": "{{reason}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Dialing attempt errors, averaged per node", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 79 - }, - "hiddenSeries": false, - "id": 46, - "interval": "1m", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "maxPerRow": 2, - "nullPointMode": "null as zero", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": null, - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "avg(increase(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}[$__interval])) by (reason)", - "interval": "", - "legendFormat": "{{reason}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disconnects, averaged per node", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": "Disconnects", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 86 + "y": 122 }, "id": 52, "panels": [], @@ -2122,13 +2460,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, - "w": 24, + "w": 12, "x": 0, - "y": 87 + "y": 123 }, "hiddenSeries": false, "id": 54, @@ -2150,14 +2495,15 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", - "repeat": null, - "repeatDirection": "v", + "repeat": "nodename", + "repeatDirection": "h", "seriesOverrides": [ { "alias": "/discard/", @@ -2179,7 +2525,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(increase(${metric_namespace}_finality_grandpa_communication_gossip_validator_messages{instance=~\"${nodename}\"}[$__interval])) by (action, message)", + "expr": "rate(${metric_namespace}_finality_grandpa_communication_gossip_validator_messages{instance=~\"${nodename}\"}[$__interval])", "interval": "", "legendFormat": "{{message}} => {{action}}", "refId": "A" @@ -2233,7 +2579,7 @@ "h": 1, "w": 24, "x": 0, - "y": 93 + "y": 129 }, "id": 25, "panels": [], @@ -2242,736 +2588,38 @@ "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", + "datasource": null, "description": "", - "fill": 0, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": null, "gridPos": { - "h": 5, - "w": 12, + "h": 1, + "w": 24, "x": 0, - "y": 94 + "y": 130 }, - "hiddenSeries": false, - "id": 33, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": true, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(${metric_namespace}_sub_libp2p_kbuckets_num_nodes{instance=~\"${nodename}\"}) by (instance)", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], + "headings": true, + "id": 423, + "limit": 10, + "pluginVersion": "7.2.1", + "query": "kademlia", + "recent": false, + "search": false, + "starred": false, + "tags": [], "timeFrom": null, - "timeRegions": [], "timeShift": null, - "title": "Number of entries in Kademlia k-buckets", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "max": 0, - "min": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 0, - "fillGradient": 7, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 94 - }, - "hiddenSeries": false, - "id": 35, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(${metric_namespace}_sub_libp2p_kademlia_random_queries_total{instance=~\"${nodename}\"}[5m])", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Kademlia random discovery queries started per second", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "cps", - "label": "Queries per second", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 99 - }, - "hiddenSeries": false, - "id": 111, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "${metric_namespace}_sub_libp2p_kademlia_records_count{instance=~\"${nodename}\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Number of Kademlia records", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 99 - }, - "hiddenSeries": false, - "id": 112, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "${metric_namespace}_sub_libp2p_kademlia_records_sizes_total{instance=~\"${nodename}\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Total size of Kademlia records", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "description": "", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 103 - }, - "hiddenSeries": false, - "id": 211, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": true, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "${metric_namespace}_authority_discovery_known_authorities_count{instance=~\"${nodename}\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Number of authorities discovered by authority-discovery", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "max": 0, - "min": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "description": "", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 103 - }, - "hiddenSeries": false, - "id": 233, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": true, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "${metric_namespace}_authority_discovery_amount_external_addresses_last_published{instance=~\"${nodename}\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Number of addresses published by authority-discovery", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "max": 0, - "min": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 108 - }, - "hiddenSeries": false, - "id": 68, - "interval": "1m", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": null, - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(${metric_namespace}_authority_discovery_dht_event_received{name=\"value_found\", instance=~\"${nodename}\"}[2h]) / ignoring(name) (\n rate(${metric_namespace}_authority_discovery_dht_event_received{name=\"value_found\", instance=~\"${nodename}\"}[2h]) +\n ignoring(name) rate(${metric_namespace}_authority_discovery_dht_event_received{name=\"value_not_found\", instance=~\"${nodename}\"}[2h])\n)", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Authority discovery get_value success rate in past two hours", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1.0", - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 108 - }, - "hiddenSeries": false, - "id": 234, - "interval": "1m", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeatDirection": "v", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(${metric_namespace}_authority_discovery_dht_event_received{name=\"value_put\", instance=~\"${nodename}\"}[2h]) / ignoring(name) (\n rate(${metric_namespace}_authority_discovery_dht_event_received{name=\"value_put\", instance=~\"${nodename}\"}[2h]) +\n ignoring(name) rate(${metric_namespace}_authority_discovery_dht_event_received{name=\"value_put_failed\", instance=~\"${nodename}\"}[2h])\n)", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Authority discovery put_value success rate in past two hours", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1.0", - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Kademlia and Authority Discovery metrics moved to \"kademlia-and-authority-discovery\" dashboard.", + "type": "dashlist" } ], "refresh": "1m", - "schemaVersion": 22, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { @@ -2981,9 +2629,9 @@ "current": {}, "datasource": "$data_source", "definition": "${metric_namespace}_process_start_time_seconds", + "error": null, "hide": 0, - "includeAll": true, - "index": -1, + "includeAll": false, "label": "Instance name filter", "multi": true, "name": "nodename", @@ -3003,15 +2651,15 @@ "allValue": null, "current": {}, "datasource": "$data_source", - "definition": "${metric_namespace}_sub_libp2p_notifications_sizes_count", + "definition": "${metric_namespace}_sub_libp2p_notifications_sizes_count{instance=~\"${nodename}\"}", + "error": null, "hide": 2, "includeAll": true, - "index": -1, "label": null, "multi": false, "name": "notif_protocol", "options": [], - "query": "${metric_namespace}_sub_libp2p_notifications_sizes_count", + "query": "${metric_namespace}_sub_libp2p_notifications_sizes_count{instance=~\"${nodename}\"}", "refresh": 1, "regex": "/protocol=\"(.*?)\"/", "skipUrlSync": false, @@ -3026,15 +2674,15 @@ "allValue": null, "current": {}, "datasource": "$data_source", - "definition": "${metric_namespace}_sub_libp2p_requests_out_started_total", + "definition": "${metric_namespace}_sub_libp2p_requests_out_started_total{instance=~\"${nodename}\"}", + "error": null, "hide": 2, "includeAll": true, - "index": -1, "label": null, "multi": false, "name": "request_protocol", "options": [], - "query": "${metric_namespace}_sub_libp2p_requests_out_started_total", + "query": "${metric_namespace}_sub_libp2p_requests_out_started_total{instance=~\"${nodename}\"}", "refresh": 1, "regex": "/protocol=\"(.*?)\"/", "skipUrlSync": false, @@ -3048,9 +2696,10 @@ { "current": { "selected": false, - "text": "Prometheus", - "value": "Prometheus" + "text": "prometheus.parity-mgmt", + "value": "prometheus.parity-mgmt" }, + "error": null, "hide": 0, "includeAll": false, "label": "Source of data", @@ -3066,15 +2715,18 @@ { "current": { "value": "${VAR_METRIC_NAMESPACE}", - "text": "${VAR_METRIC_NAMESPACE}" + "text": "${VAR_METRIC_NAMESPACE}", + "selected": false }, + "error": null, "hide": 2, "label": "Prefix of the metrics", "name": "metric_namespace", "options": [ { "value": "${VAR_METRIC_NAMESPACE}", - "text": "${VAR_METRIC_NAMESPACE}" + "text": "${VAR_METRIC_NAMESPACE}", + "selected": false } ], "query": "${VAR_METRIC_NAMESPACE}", @@ -3101,11 +2753,8 @@ "1d" ] }, - "timezone": "", + "timezone": "utc", "title": "Substrate Networking", "uid": "vKVuiD9Zk", - "variables": { - "list": [] - }, - "version": 121 + "version": 147 } diff --git a/substrate/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json b/substrate/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json index 539fdec086..a3db46ec6d 100644 --- a/substrate/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json +++ b/substrate/.maintain/monitoring/grafana-dashboards/substrate-service-tasks.json @@ -13,7 +13,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "6.7.3" + "version": "7.3.6" }, { "type": "panel", @@ -26,6 +26,12 @@ "id": "prometheus", "name": "Prometheus", "version": "1.0.0" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" } ], "annotations": { @@ -75,9 +81,36 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1599471940817, + "iteration": 1610462629581, "links": [], "panels": [ + { + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 42, + "options": { + "content": "", + "mode": "markdown" + }, + "pluginVersion": "7.3.6", + "repeat": "nodename", + "timeFrom": null, + "timeShift": null, + "title": "$nodename", + "type": "text" + }, { "collapsed": false, "datasource": null, @@ -85,7 +118,7 @@ "h": 1, "w": 24, "x": 0, - "y": 0 + "y": 1 }, "id": 29, "panels": [], @@ -94,17 +127,24 @@ }, { "aliasColors": {}, - "bars": false, + "bars": true, "dashLength": 10, "dashes": false, "datasource": "$data_source", - "fill": 0, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 2, "fillGradient": 0, "gridPos": { "h": 6, "w": 24, "x": 0, - "y": 1 + "y": 2 }, "hiddenSeries": false, "id": 11, @@ -122,23 +162,25 @@ "total": false, "values": true }, - "lines": true, - "linewidth": 2, - "nullPointMode": "null", + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, - "stack": false, - "steppedLine": true, + "stack": true, + "steppedLine": false, "targets": [ { - "expr": "avg(irate(${metric_namespace}_tasks_polling_duration_sum{instance=~\"${nodename}\"}[10m])) by (task_name)", + "expr": "irate(${metric_namespace}_tasks_polling_duration_sum{instance=~\"${nodename}\"}[10m])", "interval": "", "legendFormat": "{{task_name}}", "refId": "A" @@ -148,7 +190,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "CPU time spent on each task (average per node)", + "title": "CPU time spent on each task", "tooltip": { "shared": true, "sort": 2, @@ -191,13 +233,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 24, "x": 0, - "y": 7 + "y": 8 }, "hiddenSeries": false, "id": 30, @@ -219,19 +268,21 @@ "linewidth": 2, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { - "expr": "avg(irate(${metric_namespace}_tasks_polling_duration_count{instance=~\"${nodename}\"}[10m])) by (task_name)", + "expr": "irate(${metric_namespace}_tasks_polling_duration_count{instance=~\"${nodename}\"}[10m])", "interval": "", "legendFormat": "{{task_name}}", "refId": "A" @@ -241,7 +292,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Task polling rate per second (average per node)", + "title": "Task polling rate per second", "tooltip": { "shared": true, "sort": 2, @@ -284,104 +335,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 13 - }, - "hiddenSeries": false, - "id": 31, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "max(irate(${metric_namespace}_tasks_polling_duration_count{instance=~\"${nodename}\"}[10m])) by (task_name)", - "interval": "", - "legendFormat": "{{task_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Task polling rate per second (maximum per node)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "cps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 24, "x": 0, - "y": 19 + "y": 14 }, "hiddenSeries": false, "id": 15, @@ -401,19 +368,21 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { - "expr": "avg by(task_name) (irate(${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"}[10m]))", + "expr": "irate(${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"}[10m])", "interval": "", "legendFormat": "{{task_name}}", "refId": "A" @@ -423,7 +392,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Number of tasks started per second (average per node)", + "title": "Number of tasks started per second", "tooltip": { "shared": true, "sort": 2, @@ -466,104 +435,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 25 - }, - "hiddenSeries": false, - "id": 16, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "max by(task_name) (irate(${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"}[10m]))", - "interval": "", - "legendFormat": "{{task_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Number of tasks started per second (maximum over all nodes)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 10, - "max": null, - "min": "0", - "show": true + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 24, "x": 0, - "y": 31 + "y": 20 }, "hiddenSeries": false, "id": 2, @@ -583,19 +468,21 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { - "expr": "avg by(task_name) (${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"} - sum(${metric_namespace}_tasks_ended_total{instance=~\"${nodename}\"}) without(reason))", + "expr": "${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"} - sum(${metric_namespace}_tasks_ended_total{instance=~\"${nodename}\"}) without(reason)\n\n# Fallback if tasks_ended_total is null for that task\nor on(instance, task_name) ${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"}", "interval": "", "legendFormat": "{{task_name}}", "refId": "A" @@ -605,98 +492,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Number of tasks running (average per node)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 10, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$data_source", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 37 - }, - "hiddenSeries": false, - "id": 3, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "max by(task_name) (${metric_namespace}_tasks_spawned_total{instance=~\"${nodename}\"} - sum(${metric_namespace}_tasks_ended_total{instance=~\"${nodename}\"}) without(reason))", - "interval": "", - "legendFormat": "{{task_name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Number of tasks running (maximum over all nodes)", + "title": "Number of tasks running", "tooltip": { "shared": true, "sort": 2, @@ -740,13 +536,20 @@ "dashes": false, "datasource": "$data_source", "decimals": null, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 24, "x": 0, - "y": 43 + "y": 26 }, "hiddenSeries": false, "id": 7, @@ -768,19 +571,21 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { - "expr": "avg(\n irate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"+Inf\"}[10m])\n - ignoring(le)\n irate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"1.024\"}[10m])\n) by (task_name) > 0", + "expr": "irate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"+Inf\"}[10m])\n - ignoring(le)\n irate(${metric_namespace}_tasks_polling_duration_bucket{instance=~\"${nodename}\", le=\"1.024\"}[10m]) > 0", "interval": "", "legendFormat": "{{task_name}}", "refId": "A" @@ -790,7 +595,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Calls to `Future::poll` that took more than one second (average per node)", + "title": "Number of calls to `Future::poll` that took more than one second", "tooltip": { "shared": true, "sort": 2, @@ -835,11 +640,11 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 32 }, "id": 27, "panels": [], - "title": "Misc", + "title": "Unbounded Channels", "type": "row" }, { @@ -848,13 +653,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 50 + "y": 33 }, "hiddenSeries": false, "id": 32, @@ -873,19 +685,21 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"send\"} - ignoring(action) ${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"received\"}) by (entity)", + "expr": "(\n ${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"send\"} - ignoring(action) ${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"received\"}\n)\n\n# Fallback if the `received` is null\nor on(instance) ${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"send\"}", "interval": "", "legendFormat": "{{entity}}", "refId": "B" @@ -895,7 +709,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Unbounded channels size (average per node)", + "title": "Unbounded channels size", "tooltip": { "shared": true, "sort": 2, @@ -938,13 +752,20 @@ "dashLength": 10, "dashes": false, "datasource": "$data_source", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 57 + "y": 40 }, "hiddenSeries": false, "id": 33, @@ -963,19 +784,21 @@ "linewidth": 1, "nullPointMode": "null", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "7.3.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "nodename", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "avg(irate(${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"send\"}[10m])) by (entity)", + "expr": "irate(${metric_namespace}_unbounded_channel_len{instance=~\"${nodename}\", action = \"send\"}[10m])", "interval": "", "legendFormat": "{{entity}}", "refId": "B" @@ -985,7 +808,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Unbounded channels rate (average per node)", + "title": "Unbounded channels message sending rate (1s)", "tooltip": { "shared": true, "sort": 2, @@ -1024,7 +847,7 @@ } ], "refresh": false, - "schemaVersion": 22, + "schemaVersion": 26, "style": "dark", "tags": [], "templating": { @@ -1034,9 +857,9 @@ "current": {}, "datasource": "$data_source", "definition": "${metric_namespace}_process_start_time_seconds", + "error": null, "hide": 0, - "includeAll": true, - "index": -1, + "includeAll": false, "label": "Instance filter", "multi": true, "name": "nodename", @@ -1055,15 +878,18 @@ { "current": { "value": "${VAR_METRIC_NAMESPACE}", - "text": "${VAR_METRIC_NAMESPACE}" + "text": "${VAR_METRIC_NAMESPACE}", + "selected": false }, + "error": null, "hide": 2, "label": "Prefix of the metrics", "name": "metric_namespace", "options": [ { "value": "${VAR_METRIC_NAMESPACE}", - "text": "${VAR_METRIC_NAMESPACE}" + "text": "${VAR_METRIC_NAMESPACE}", + "selected": false } ], "query": "${VAR_METRIC_NAMESPACE}", @@ -1076,6 +902,7 @@ "text": "prometheus.parity-mgmt", "value": "prometheus.parity-mgmt" }, + "error": null, "hide": 0, "includeAll": false, "label": "Source of all the data", @@ -1108,11 +935,8 @@ "1d" ] }, - "timezone": "", + "timezone": "utc", "title": "Substrate Service Tasks", "uid": "3LA6XNqZz", - "variables": { - "list": [] - }, - "version": 52 + "version": 59 }