{ "__inputs": [ { "name": "VAR_METRIC_NAMESPACE", "type": "constant", "label": "Prefix of the metrics", "value": "polkadot", "description": "" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "6.7.3" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "panel", "id": "heatmap", "name": "Heatmap", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } ], "annotations": { "list": [ { "$$hashKey": "object:821", "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "limit": 100, "name": "Annotations & Alerts", "showIn": 0, "type": "dashboard" }, { "$$hashKey": "object:822", "datasource": "$data_source", "enable": true, "expr": "count(count(${metric_namespace}_sub_libp2p_connections / max_over_time(${metric_namespace}_sub_libp2p_connections[1h]) < 0.1) >= count(${metric_namespace}_sub_libp2p_connections) / 10)", "hide": false, "iconColor": "#C4162A", "limit": 100, "name": "Connection losses events", "showIn": 0, "step": "5m", "tags": [], "titleFormat": "Network-wide connectivity loss", "type": "tags" } ] }, "description": "Information related to the networking layer of Substrate", "editable": true, "gnetId": null, "graphTooltip": 0, "id": null, "iteration": 1590742405831, "links": [], "panels": [ { "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 167, "title": "Sync", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 0, "fillGradient": 0, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 1 }, "hiddenSeries": false, "id": 101, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "v", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "expr": "1 - (${metric_namespace}_sub_libp2p_peerset_num_requested{instance=~\"${nodename}\"} - ${metric_namespace}_sub_libp2p_peers_count{instance=~\"${nodename}\"}) / ${metric_namespace}_sub_libp2p_peerset_num_requested{instance=~\"${nodename}\"}", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of peer slots filled", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1044", "format": "percentunit", "label": null, "logBase": 1, "max": "1.0", "min": null, "show": true }, { "$$hashKey": "object:1045", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 29, "panels": [], "repeat": "request_protocol", "title": "Requests (${request_protocol})", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 0, "y": 7 }, "hiddenSeries": false, "id": 148, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "irate(${metric_namespace}_sub_libp2p_requests_out_started_total{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Requests emitted per second", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 12, "y": 7 }, "hiddenSeries": false, "id": 151, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "irate(${metric_namespace}_sub_libp2p_requests_in_total_count{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Requests served per second", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 146, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.5, sum(rate(${metric_namespace}_sub_libp2p_requests_out_finished_bucket{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance, le)) > 0", "instant": false, "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Median request answer time", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1230", "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1231", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 12, "y": 11 }, "hiddenSeries": false, "id": 145, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.5, sum(rate(${metric_namespace}_sub_libp2p_requests_in_total_bucket{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance, le))", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Median request serving time", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 0, "y": 15 }, "hiddenSeries": false, "id": 150, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(${metric_namespace}_sub_libp2p_requests_out_finished_bucket{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance, le)) > 0", "instant": false, "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99th percentile request answer time", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 12, "y": 15 }, "hiddenSeries": false, "id": 149, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(${metric_namespace}_sub_libp2p_requests_in_total_bucket{instance=~\"${nodename}\", protocol=\"${request_protocol}\"}[5m])) by (instance, le))", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99th percentile request serving time", "tooltip": { "shared": false, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 185 }, "id": 23, "panels": [], "repeat": "notif_protocol", "title": "Notifications (${notif_protocol})", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 186 }, "hiddenSeries": false, "id": 31, "interval": "1m", "legend": { "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "maxPerRow": 2, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "v", "seriesOverrides": [ { "$$hashKey": "object:850", "alias": "/(in)/", "color": "#73BF69" }, { "$$hashKey": "object:851", "alias": "/(out)/", "color": "#F2495C" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg by (direction) (irate(${metric_namespace}_sub_libp2p_notifications_sizes_count{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[$__interval]))", "interval": "", "legendFormat": "{{direction}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Average network notifications per second", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:874", "format": "cps", "label": "Notifs/sec", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:875", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 186 }, "hiddenSeries": false, "id": 37, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": false, "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "maxPerRow": 2, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "v", "seriesOverrides": [ { "$$hashKey": "object:942", "alias": "/(in)/", "color": "#73BF69" }, { "$$hashKey": "object:943", "alias": "/(out)/", "color": "#F2495C" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(irate(${metric_namespace}_sub_libp2p_notifications_sizes_sum{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[$__interval])) by (direction)", "instant": false, "interval": "", "legendFormat": "{{direction}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Average bandwidth used by notifications", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:966", "format": "Bps", "label": "Bandwidth", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:967", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 193 }, "hiddenSeries": false, "id": 16, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(${metric_namespace}_sub_libp2p_out_events_notifications_sizes{instance=~\"${nodename}\", protocol=\"${notif_protocol}\", action=\"sent\"} - ignoring(action) ${metric_namespace}_sub_libp2p_out_events_notifications_sizes{instance=~\"${nodename}\", protocol=\"${notif_protocol}\", action=\"received\"}) by (instance) > 0", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total sizes of notifications waiting to be delivered to the rest of Substrate", "tooltip": { "shared": false, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 1, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 193 }, "hiddenSeries": false, "id": 21, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pluginVersion": "6.4.5", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(${metric_namespace}_sub_libp2p_notifications_sizes_sum{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[5m])) by (direction, protocol) / sum(rate(${metric_namespace}_sub_libp2p_notifications_sizes_count{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[5m])) by (direction, protocol)", "format": "time_series", "interval": "", "legendFormat": "{{direction}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Average size of sent and received notifications in the past 5 minutes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2050", "format": "bytes", "label": "Max. notification size", "logBase": 10, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2051", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "description": "99.9% of the time, the output queue size for this protocol is below the given value", "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 201 }, "hiddenSeries": false, "id": 14, "legend": { "alignAsTable": false, "avg": false, "current": true, "hideEmpty": false, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "max", "fill": 1, "linewidth": 0 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(${metric_namespace}_sub_libp2p_notifications_queues_size_bucket{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[2m])) by (le, instance))", "hide": false, "interval": "", "legendFormat": "{{protocol}}", "refId": "A" }, { "expr": "max(histogram_quantile(0.99, sum(rate(${metric_namespace}_sub_libp2p_notifications_queues_size_bucket{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[2m])) by (le, instance)))", "interval": "", "legendFormat": "max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99th percentile of queues sizes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": "300", "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 1, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 201 }, "hiddenSeries": false, "id": 134, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pluginVersion": "6.4.5", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1.0, sum(rate(${metric_namespace}_sub_libp2p_notifications_sizes_bucket{instance=~\"${nodename}\", protocol=\"${notif_protocol}\"}[5m])) by (direction, le))", "format": "time_series", "interval": "", "legendFormat": "{{direction}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Maximum size of sent and received notifications in the past 5 minutes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1524", "format": "bytes", "label": "Max. notification size", "logBase": 10, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1525", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1853 }, "id": 27, "panels": [], "title": "Transport", "type": "row" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 1854 }, "hiddenSeries": false, "id": 19, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": false, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": false }, "lines": false, "linewidth": 1, "maxPerRow": 2, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "v", "seriesOverrides": [ { "alias": "established (in)", "color": "#37872D" }, { "alias": "established (out)", "color": "#C4162A" }, { "alias": "pending (out)", "color": "#FF7383" }, { "alias": "closed-recently", "color": "#FADE2A", "steppedLine": true } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "avg(sum(${metric_namespace}_sub_libp2p_connections_opened_total{direction=\"in\", instance=~\"${nodename}\"}) by (instance) - sum(${metric_namespace}_sub_libp2p_connections_closed_total{direction=\"in\", instance=~\"${nodename}\"}) by (instance))", "format": "time_series", "hide": false, "interval": "", "legendFormat": "established (in)", "refId": "A" }, { "expr": "avg(sum(${metric_namespace}_sub_libp2p_connections_opened_total{direction=\"out\", instance=~\"${nodename}\"}) by (instance) - sum(${metric_namespace}_sub_libp2p_connections_closed_total{direction=\"out\", instance=~\"${nodename}\"}) by (instance))", "hide": false, "instant": false, "interval": "", "legendFormat": "established (out)", "refId": "C" }, { "expr": "avg(sum by (instance) (${metric_namespace}_sub_libp2p_pending_connections{instance=~\"${nodename}\"}))", "hide": false, "interval": "", "legendFormat": "pending (out)", "refId": "B" }, { "expr": "avg(sum by(instance) (increase(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}[$__interval])))", "hide": false, "interval": "", "legendFormat": "closed-recently", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Average transport-level (TCP, QUIC, ...) connections per node", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Connections", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 1860 }, "hiddenSeries": false, "id": 39, "interval": "1m", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": false, "linewidth": 1, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/.*/", "color": "#FF780A" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "avg(increase(${metric_namespace}_sub_libp2p_incoming_connections_handshake_errors_total{instance=~\"${nodename}\"}[$__interval])) by (reason)", "interval": "", "legendFormat": "{{reason}}", "refId": "A" }, { "expr": "avg(increase(${metric_namespace}_sub_libp2p_listeners_errors_total{instance=~\"${nodename}\"}[$__interval]))", "interval": "", "legendFormat": "pre-handshake", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of incoming connection errors, averaged by node", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Errors", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateOranges", "exponent": 0.5, "max": 100, "min": 0, "mode": "spectrum" }, "dataFormat": "timeseries", "datasource": "$data_source", "description": "Each bucket represent a certain number of nodes using a certain bandwidth range.", "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1860 }, "heatmap": {}, "hideZeroBuckets": false, "highlightCards": true, "id": 4, "legend": { "show": false }, "reverseYBuckets": false, "targets": [ { "expr": "${metric_namespace}_network_per_sec_bytes{instance=~\"${nodename}\"}", "format": "time_series", "interval": "", "legendFormat": "", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Heatmap of network bandwidth", "tooltip": { "show": true, "showHistogram": false }, "type": "heatmap", "xAxis": { "show": true }, "xBucketNumber": null, "xBucketSize": "2.5m", "yAxis": { "decimals": null, "format": "Bps", "logBase": 1, "max": null, "min": null, "show": true, "splitFactor": null }, "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 1866 }, "hiddenSeries": false, "id": 81, "interval": "1m", "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": false }, "lines": false, "linewidth": 1, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "v", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "avg(increase(${metric_namespace}_sub_libp2p_pending_connections_errors_total{instance=~\"${nodename}\"}[$__interval])) by (reason)", "interval": "", "legendFormat": "{{reason}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Dialing attempt errors, averaged per node", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 1866 }, "hiddenSeries": false, "id": 46, "interval": "1m", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": false, "linewidth": 1, "maxPerRow": 2, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "v", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "avg(increase(${metric_namespace}_sub_libp2p_connections_closed_total{instance=~\"${nodename}\"}[$__interval])) by (reason)", "interval": "", "legendFormat": "{{reason}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disconnects, averaged per node", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Disconnects", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1873 }, "id": 52, "panels": [], "title": "GrandPa", "type": "row" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 1874 }, "hiddenSeries": false, "id": 54, "interval": "1m", "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": true, "values": true }, "lines": false, "linewidth": 1, "nullPointMode": "null as zero", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "v", "seriesOverrides": [ { "alias": "/discard/", "color": "#FA6400", "zindex": -2 }, { "alias": "/keep/", "color": "#73BF69", "zindex": 2 }, { "alias": "/process_and_discard/", "color": "#5794F2" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "expr": "avg(increase(${metric_namespace}_finality_grandpa_communication_gossip_validator_messages{instance=~\"${nodename}\"}[$__interval])) by (action, message)", "interval": "", "legendFormat": "{{message}} => {{action}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "GrandPa messages received from the network, and action", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1880 }, "id": 25, "panels": [], "repeat": null, "title": "Kademlia", "type": "row" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$data_source", "description": "", "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 1881 }, "hiddenSeries": false, "id": 33, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": false, "linewidth": 1, "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/.*/", "color": "#B877D9" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "${metric_namespace}_sub_libp2p_kbuckets_num_nodes{instance=~\"${nodename}\"}", "format": "time_series", "instant": true, "interval": "", "legendFormat": "Number nodes in all kbuckets", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Distribution over number of entries in k-buckets", "tooltip": { "shared": false, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "max": 0, "min": null, "mode": "histogram", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 7, "fillGradient": 7, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1881 }, "hiddenSeries": false, "id": 35, "interval": "1m", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "60 * sum(irate(${metric_namespace}_sub_libp2p_random_kademalia_queries_total{instance=~\"${nodename}\"}[$__interval]))", "interval": "", "legendFormat": "Number of Kademlia random queries started per minute on all nodes", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of Kademlia discovery queries per minute on all nodes combined", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "cpm", "label": "Queries per minute", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 0, "y": 1887 }, "hiddenSeries": false, "id": 111, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "max", "fillBelowTo": "min", "lines": false }, { "alias": "min", "lines": false } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(${metric_namespace}_sub_libp2p_kademlia_records_count{instance=~\"${nodename}\"})", "interval": "", "legendFormat": "avg", "refId": "A" }, { "expr": "min(${metric_namespace}_sub_libp2p_kademlia_records_count{instance=~\"${nodename}\"})", "interval": "", "legendFormat": "min", "refId": "B" }, { "expr": "max(${metric_namespace}_sub_libp2p_kademlia_records_count{instance=~\"${nodename}\"})", "interval": "", "legendFormat": "max", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of Kademlia records (average/min/max per node)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 0, "fillGradient": 0, "gridPos": { "h": 4, "w": 12, "x": 12, "y": 1887 }, "hiddenSeries": false, "id": 112, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "max", "fillBelowTo": "min", "lines": false }, { "alias": "min", "lines": false } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(${metric_namespace}_sub_libp2p_kademlia_records_sizes_total{instance=~\"${nodename}\"})", "interval": "", "legendFormat": "avg", "refId": "A" }, { "expr": "min(${metric_namespace}_sub_libp2p_kademlia_records_sizes_total{instance=~\"${nodename}\"})", "interval": "", "legendFormat": "min", "refId": "B" }, { "expr": "max(${metric_namespace}_sub_libp2p_kademlia_records_sizes_total{instance=~\"${nodename}\"})", "interval": "", "legendFormat": "max", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total size of Kademlia records (average/min/max per node)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 1891 }, "hiddenSeries": false, "id": 68, "interval": "1m", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "v", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(\n # The amount of inflight Kademlia queries per node.\r\n sum by (instance) (\r\n # The total amount of Kademlia GET_VALUE queries started.\r\n ${metric_namespace}_authority_discovery_authority_addresses_requested_total{instance=~\"${nodename}\"}\r\n \r\n # The total amount of Kademlia PUT_VALUE queries started.\r\n + ${metric_namespace}_authority_discovery_times_published_total{instance=~\"${nodename}\"}\r\n )\r\n - sum by (instance) (\r\n # The total amount of Kademlia queries (both GET_VALUE and PUT_VALUE) finished.\r\n ${metric_namespace}_authority_discovery_dht_event_received{instance=~\"${nodename}\"}\r\n )\n)", "interval": "", "legendFormat": "in-progress", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Authority discovery Kademlia queries in progress, averaged per node", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$data_source", "fill": 1, "fillGradient": 0, "gridPos": { "h": 4, "w": 24, "x": 0, "y": 1896 }, "hiddenSeries": false, "id": 72, "interval": "1m", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": false, "linewidth": 1, "nullPointMode": "null", "options": { "dataLinks": [] }, "percentage": false, "pointradius": 2, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": "v", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(sum without(instance) (delta(${metric_namespace}_authority_discovery_times_published_total{instance=~\"${nodename}\"}[$__interval])))", "interval": "", "legendFormat": "publications", "refId": "A" }, { "expr": "avg(sum without(instance) (delta(${metric_namespace}_authority_discovery_dht_event_received{instance=~\"${nodename}\", name=\"value_put\"}[$__interval])))", "interval": "", "legendFormat": "successes", "refId": "B" }, { "expr": "avg(sum without(instance) (delta(${metric_namespace}_authority_discovery_dht_event_received{instance=~\"${nodename}\", name=\"value_put_failed\"}[$__interval])))", "interval": "", "legendFormat": "failures", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Authority discovery publications, averaged per node", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "1m", "schemaVersion": 22, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": {}, "datasource": "$data_source", "definition": "${metric_namespace}_cpu_usage_percentage", "hide": 0, "includeAll": true, "index": -1, "label": "Instance name filter", "multi": true, "name": "nodename", "options": [], "query": "${metric_namespace}_cpu_usage_percentage", "refresh": 1, "regex": "/instance=\"(.*?)\"/", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "$data_source", "definition": "${metric_namespace}_sub_libp2p_notifications_sizes_count", "hide": 2, "includeAll": true, "index": -1, "label": null, "multi": false, "name": "notif_protocol", "options": [], "query": "${metric_namespace}_sub_libp2p_notifications_sizes_count", "refresh": 1, "regex": "/protocol=\"(.*?)\"/", "skipUrlSync": false, "sort": 5, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "$data_source", "definition": "${metric_namespace}_sub_libp2p_requests_out_started_total", "hide": 2, "includeAll": true, "index": -1, "label": null, "multi": false, "name": "request_protocol", "options": [], "query": "${metric_namespace}_sub_libp2p_requests_out_started_total", "refresh": 1, "regex": "/protocol=\"(.*?)\"/", "skipUrlSync": false, "sort": 5, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "current": { "selected": false, "tags": [], "text": "prometheus.parity-mgmt", "value": "prometheus.parity-mgmt" }, "hide": 0, "includeAll": false, "label": "Source of data", "multi": false, "name": "data_source", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "current": { "value": "${VAR_METRIC_NAMESPACE}", "text": "${VAR_METRIC_NAMESPACE}" }, "hide": 2, "label": "Prefix of the metrics", "name": "metric_namespace", "options": [ { "value": "${VAR_METRIC_NAMESPACE}", "text": "${VAR_METRIC_NAMESPACE}" } ], "query": "${VAR_METRIC_NAMESPACE}", "skipUrlSync": false, "type": "constant" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "", "title": "Substrate Networking", "uid": "vKVuiD9Zk", "variables": { "list": [] }, "version": 103 }