{ "uid": "vm-fleet", "title": "VM fleet", "tags": [ "exe.dev", "node_exporter" ], "timezone": "browser", "schemaVersion": 41, "version": 1, "refresh": "1m", "time": { "from": "now-6h", "to": "now" }, "description": "Every VM in ~/monitoring/prometheus/vms and every website in ~/monitoring/prometheus/sites. https://exe.dev/deploy/grafana", "panels": [ { "id": 1, "type": "stat", "title": "VMs up", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "count(up{job=\"node\"} == 1) or vector(0)", "instant": true, "range": false } ], "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "value" }, "fieldConfig": { "defaults": { "unit": "none", "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [] } }, { "id": 2, "type": "stat", "title": "VMs down", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "count(up{job=\"node\"} == 0) or vector(0)", "instant": true, "range": false } ], "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "value" }, "fieldConfig": { "defaults": { "unit": "none", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] } }, { "id": 3, "type": "stat", "title": "Failed systemd units", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "count(node_systemd_unit_state{job=\"node\",state=\"failed\"} == 1) or vector(0)", "instant": true, "range": false } ], "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "value" }, "fieldConfig": { "defaults": { "unit": "none", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] } }, { "id": 4, "type": "stat", "title": "Websites down", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "count(probe_success{job=\"sites\"} == 0) or vector(0)", "instant": true, "range": false } ], "options": { "colorMode": "background", "graphMode": "none", "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "value" }, "fieldConfig": { "defaults": { "unit": "none", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] } }, { "id": 5, "type": "table", "title": "VMs", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 0, "y": 4, "w": 24, "h": 8 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (vm) (up{job=\"node\"})", "instant": true, "range": false, "format": "table" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (vm) (100 * (1 - avg by (vm) (rate(node_cpu_seconds_total{job=\"node\",mode=\"idle\"}[5m]))))", "instant": true, "range": false, "format": "table" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (vm) (100 * (1 - node_memory_MemAvailable_bytes{job=\"node\"} / node_memory_MemTotal_bytes{job=\"node\"}))", "instant": true, "range": false, "format": "table" }, { "refId": "D", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (vm) (100 * (1 - node_filesystem_avail_bytes{job=\"node\",mountpoint=\"/\"} / node_filesystem_size_bytes{job=\"node\",mountpoint=\"/\"}))", "instant": true, "range": false, "format": "table" }, { "refId": "E", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (vm) (node_load5{job=\"node\"})", "instant": true, "range": false, "format": "table" }, { "refId": "F", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (vm) (time() - node_boot_time_seconds{job=\"node\"})", "instant": true, "range": false, "format": "table" } ], "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "Value #A": "Up", "Value #B": "CPU", "Value #C": "Memory", "Value #D": "Disk /", "Value #E": "Load 5m", "Value #F": "Uptime", "vm": "VM" } } } ], "fieldConfig": { "defaults": { "custom": { "minWidth": 80 } }, "overrides": [ { "matcher": { "id": "byName", "options": "Up" }, "properties": [ { "id": "unit", "value": "none" }, { "id": "custom.width", "value": 70 }, { "id": "mappings", "value": [ { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "up", "color": "green" } } } ] }, { "id": "custom.cellOptions", "value": { "type": "color-text" } } ] }, { "matcher": { "id": "byName", "options": "CPU" }, "properties": [ { "id": "unit", "value": "percent" }, { "id": "decimals", "value": 0 }, { "id": "custom.cellOptions", "value": { "type": "gauge", "mode": "basic" } }, { "id": "max", "value": 100 }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 90 } ] } } ] }, { "matcher": { "id": "byName", "options": "Memory" }, "properties": [ { "id": "unit", "value": "percent" }, { "id": "decimals", "value": 0 }, { "id": "custom.cellOptions", "value": { "type": "gauge", "mode": "basic" } }, { "id": "max", "value": 100 }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 90 } ] } } ] }, { "matcher": { "id": "byName", "options": "Disk /" }, "properties": [ { "id": "unit", "value": "percent" }, { "id": "decimals", "value": 0 }, { "id": "custom.cellOptions", "value": { "type": "gauge", "mode": "basic" } }, { "id": "max", "value": 100 }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 90 } ] } } ] }, { "matcher": { "id": "byName", "options": "Load 5m" }, "properties": [ { "id": "unit", "value": "none" }, { "id": "decimals", "value": 2 } ] }, { "matcher": { "id": "byName", "options": "Uptime" }, "properties": [ { "id": "unit", "value": "s" } ] } ] }, "options": { "showHeader": true, "sortBy": [ { "displayName": "VM", "desc": false } ] } }, { "id": 6, "type": "timeseries", "title": "CPU used", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 0, "y": 12, "w": 8, "h": 8 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "100 * (1 - avg by (vm) (rate(node_cpu_seconds_total{job=\"node\",mode=\"idle\"}[$__rate_interval])))", "legendFormat": "{{vm}}" } ], "fieldConfig": { "defaults": { "unit": "percent", "custom": { "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 7, "type": "timeseries", "title": "Memory used", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 8, "y": 12, "w": 8, "h": 8 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "100 * (1 - node_memory_MemAvailable_bytes{job=\"node\"} / node_memory_MemTotal_bytes{job=\"node\"})", "legendFormat": "{{vm}}" } ], "fieldConfig": { "defaults": { "unit": "percent", "custom": { "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 8, "type": "timeseries", "title": "Disk / used", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 16, "y": 12, "w": 8, "h": 8 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "100 * (1 - node_filesystem_avail_bytes{job=\"node\",mountpoint=\"/\"} / node_filesystem_size_bytes{job=\"node\",mountpoint=\"/\"})", "legendFormat": "{{vm}}" } ], "fieldConfig": { "defaults": { "unit": "percent", "custom": { "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 9, "type": "timeseries", "title": "Network in", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 0, "y": 20, "w": 8, "h": 8 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (vm) (rate(node_network_receive_bytes_total{job=\"node\",device!~\"lo|docker.*|br-.*|veth.*\"}[$__rate_interval]))", "legendFormat": "{{vm}}" } ], "fieldConfig": { "defaults": { "unit": "Bps", "custom": { "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 10, "type": "timeseries", "title": "Network out", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 8, "y": 20, "w": 8, "h": 8 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (vm) (rate(node_network_transmit_bytes_total{job=\"node\",device!~\"lo|docker.*|br-.*|veth.*\"}[$__rate_interval]))", "legendFormat": "{{vm}}" } ], "fieldConfig": { "defaults": { "unit": "Bps", "custom": { "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 11, "type": "timeseries", "title": "Failed systemd units", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 16, "y": 20, "w": 8, "h": 8 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "node_systemd_unit_state{job=\"node\",state=\"failed\"} == 1", "legendFormat": "{{vm}} {{name}}" } ], "fieldConfig": { "defaults": { "unit": "none", "custom": { "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 12, "type": "table", "title": "Websites", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "x": 0, "y": 28, "w": 24, "h": 7 }, "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "max by (instance) (probe_success{job=\"sites\"})", "instant": true, "range": false, "format": "table" }, { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "max by (instance) (probe_http_status_code{job=\"sites\"})", "instant": true, "range": false, "format": "table" }, { "refId": "C", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "max by (instance) (probe_duration_seconds{job=\"sites\"})", "instant": true, "range": false, "format": "table" }, { "refId": "D", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "max by (instance) ((probe_ssl_earliest_cert_expiry{job=\"sites\"} - time()) / 86400)", "instant": true, "range": false, "format": "table" } ], "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "Value #A": "Up", "Value #B": "HTTP status", "Value #C": "Response time", "Value #D": "Certificate expires in", "instance": "URL" } } } ], "fieldConfig": { "defaults": { "custom": { "minWidth": 80 } }, "overrides": [ { "matcher": { "id": "byName", "options": "Up" }, "properties": [ { "id": "unit", "value": "none" }, { "id": "custom.width", "value": 70 }, { "id": "mappings", "value": [ { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "up", "color": "green" } } } ] }, { "id": "custom.cellOptions", "value": { "type": "color-text" } } ] }, { "matcher": { "id": "byName", "options": "HTTP status" }, "properties": [ { "id": "unit", "value": "none" } ] }, { "matcher": { "id": "byName", "options": "Response time" }, "properties": [ { "id": "unit", "value": "s" } ] }, { "matcher": { "id": "byName", "options": "Certificate expires in" }, "properties": [ { "id": "unit", "value": "d" }, { "id": "decimals", "value": 0 } ] } ] }, "description": "URLs listed in ~/monitoring/prometheus/sites/*.yml, checked every minute by the Blackbox exporter." } ], "templating": { "list": [] }, "annotations": { "list": [] }, "editable": true }