.maintain/sentry-node: Add monitoring to docker-compose stack (#5321)

* Substrate Dashboard example

* Improve README

* Update README_dashboard.md

* Add screenshots

* Minor fix

* Minor fix, image link

* .maintain/sentry-node: Add monitoring to docker-compose stack

With this patch a user can run the following fully configured and
monitored setup with a single command:

`docker-compose -f .maintain/sentry-node/docker-compose.yml up`

- 2 validators in two different network namespaces, connected via one
sentry node.

- Polkadot-js/apps to connect to one of the nodes above.

- Prometheus scraping the 3 Substrate nodes.

- Grafana displaying data from Prometheus with community dashboards

* .maintain/monitoring/grafana: Change default datasource name

* .maintain/monitoring/grafana: Add metric namespace option

* .maintain/monitoring/grafana: Remove `host` metric from most metrics

* .maintain/monitoring/grafana: Remove underscore from metric_namespace

* .maintain/monitoring: Use `instance` label instead of `hostname`

To identify a scrape target, one should use `instance` and not
`hostname` as multiple targets might run on the same node.

See https://prometheus.io/docs/concepts/jobs_instances/ for details.

* .maintain/monitoring: Introduce instance variable

* .maintain/monitoring/grafana: Rename substrate_block_height_number

* .maitain/monitoring/grafana: Use instance instead of host in legend

* .maintain/monitoring: Remove node exporter dependency

* .maintain/sentry-node/prometheus: Simplify configuration

* .maintain/monitoring/grafana: Update README and remove images

* .maintain/sentry-node: Improve docs

* .maintain/monitoring/grafana: Use metric_namespace template variable

* Use --sentry from v0.7.29 instead of a reserved-node

* .maintain/sentry-node: Revert sentry-a using validator-b as bootnode

Co-authored-by: DerFredy - @derfredy:matrix.org <derfredy@gmail.com>
Co-authored-by: david <davidd@custom.home>

substrate/.maintain/monitoring/grafana-dashboards/README_dashboard.md

@@ -0,0 +1,14 @@
+## Substrate Dashboard
+
+Shared templated Grafana dashboards.
+
+To import the dashboards follow the [Grafana
+documentation](https://grafana.com/docs/grafana/latest/reference/export_import/).
+You can see an example setup [here](../../../.maintain/sentry-node).
+
+#### Required labels on Prometheus metrics
+
+- `instance` referring to a single scrape target (see [Prometheus docs for
+  details](https://prometheus.io/docs/concepts/jobs_instances/)).
+
+- `network` referring to the Blockchain network e.g. Kusama.

substrate/.maintain/sentry-node/docker-compose.yml

@@ -19,6 +19,9 @@
 #   - validator-a: localhost:9944
 #   - validator-b: localhost:9945
 #   - sentry-a: localhost:9946
+#   - grafana: localhost:3001
+#   - prometheus: localhost:9090
+
 
 version: "3.7"
 services:
@@ -80,7 +83,7 @@ services:
       - "--port"
       - "30333"
       - "--charlie"
-      - "--bootnodes"
+      - "--sentry"
       - "/dns4/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR"
       - "--bootnodes"
       - "/dns4/validator-b/tcp/30333/p2p/QmSVnNf9HwVMT1Y4cK1P6aoJcEZjmoTXpjKBmAABLMnZEk"
@@ -95,6 +98,7 @@ services:
       - "--log"
       - "sub-authority-discovery=trace"
       - "--sentry"
+      - "--prometheus-external"
 
   validator-b:
     image: parity/substrate
@@ -127,12 +131,43 @@ services:
       - "--unsafe-rpc-external"
       - "--log"
       - "sub-authority-discovery=trace"
+      - "--prometheus-external"
 
   ui:
     image: polkadot-js/apps
     ports:
       - "3000:80"
 
+  prometheus:
+    image: prom/prometheus
+    networks:
+      - network-a
+      - internet
+    ports:
+      - "9090:9090"
+    links:
+      - validator-a:validator-a
+      - sentry-a:sentry-a
+      - validator-b:validator-b
+    volumes:
+      - ./prometheus/:/etc/prometheus/
+    restart: always
+
+  grafana:
+    image: grafana/grafana
+    user: "104"
+    depends_on:
+      - prometheus
+    networks:
+      - network-a
+      - internet
+    ports:
+      - 3001:3000
+    volumes:
+      - ./grafana/provisioning/:/etc/grafana/provisioning
+      - ../monitoring/grafana-dashboards/:/etc/grafana/provisioning/dashboard-definitions
+    restart: always
+
 networks:
   network-a:
   internet:

substrate/.maintain/sentry-node/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+- name: 'Prometheus'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  editable: false
+  options:
+    path: /etc/grafana/provisioning/dashboard-definitions

substrate/.maintain/sentry-node/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,50 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+- name: Prometheus
+  # <string, required> datasource type. Required
+  type: prometheus
+  # <string, required> access mode. direct or proxy. Required
+  access: proxy
+  # <int> org id. will default to orgId 1 if not specified
+  orgId: 1
+  # <string> url
+  url: http://prometheus:9090
+  # <string> database password, if used
+  password:
+  # <string> database user, if used
+  user:
+  # <string> database name, if used
+  database:
+  # <bool> enable/disable basic auth
+  basicAuth: false
+  # <string> basic auth username, if used
+  basicAuthUser:
+  # <string> basic auth password, if used
+  basicAuthPassword:
+  # <bool> enable/disable with credentials headers
+  withCredentials:
+  # <bool> mark as default datasource. Max one per org
+  isDefault: true
+  # <map> fields that will be converted to json and stored in json_data
+  jsonData:
+     graphiteVersion: "1.1"
+     tlsAuth: false
+     tlsAuthWithCACert: false
+  # <string> json object of data that will be encrypted.
+  secureJsonData:
+    tlsCACert: "..."
+    tlsClientCert: "..."
+    tlsClientKey: "..."
+  version: 1
+  # <bool> allow users to edit datasources from the UI.
+  editable: true

substrate/.maintain/sentry-node/prometheus/prometheus.yml

@@ -0,0 +1,15 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'substrate_validator-a'
+    static_configs:
+      - targets: ['validator-a:9615']
+        labels:
+          network: dev
+      - targets: ['sentry-a:9615']
+        labels:
+          network: dev
+      - targets: ['validator-b:9615']
+        labels:
+          network: dev