From 612cd9143033f72fe310859049b7b819d32c926d Mon Sep 17 00:00:00 2001 From: Anton Zadvorny Date: Mon, 10 Feb 2020 08:50:05 +0300 Subject: [PATCH] Initial commit --- .env.example | 10 + .gitignore | 3 + README.md | 13 ++ config/alertrules.yml | 10 + .../provisioning/dashboards/dashboard.yml | 12 ++ .../provisioning/dashboards/display.json | 175 ++++++++++++++++++ .../provisioning/datasources/datasource.yml | 11 ++ docker-compose.config.yml | 19 ++ docker-compose.yml | 65 +++++++ exporter/Dockerfile | 11 ++ exporter/exporter.py | 93 ++++++++++ exporter/requirements.txt | 1 + templates/alertmanager.yml | 17 ++ templates/prometheus.yml | 38 ++++ 14 files changed, 478 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 config/alertrules.yml create mode 100644 config/grafana/provisioning/dashboards/dashboard.yml create mode 100644 config/grafana/provisioning/dashboards/display.json create mode 100644 config/grafana/provisioning/datasources/datasource.yml create mode 100644 docker-compose.config.yml create mode 100644 docker-compose.yml create mode 100644 exporter/Dockerfile create mode 100644 exporter/exporter.py create mode 100644 exporter/requirements.txt create mode 100644 templates/alertmanager.yml create mode 100644 templates/prometheus.yml diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..bd87e50 --- /dev/null +++ b/.env.example @@ -0,0 +1,10 @@ +TARGETS=https://smt-develop2.ru/displays.php?r=GetDisplaysStates&p=15047 +SCRAPE_INTERVAL=15s +GRAFANA_USER=admin +GRAFANA_PASSWORD=admin +ALERT_EMAIL_TO=example@example.com +ALERT_EMAIL_FROM=alert@example.com +ALERT_SMTP_HOST=smtp.example.com:25 +ALERT_SMTP_USER=smtp_user +ALERT_SMTP_PASSWORD=smtp_password +ALERT_SLACK_WEBHOOK=https://slack.com/webhook diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0f27fc4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.env +config/alertmanager.yml +config/prometheus.yml diff --git a/README.md b/README.md new file mode 100644 index 0000000..3b40aa0 --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +``` +# После этого задать переменные в файле .env +cp .env.example .env + +# Сгенерировать конфиги на основе установленных в файле .env переменных +docker-compose -f docker-compose.config.yml run gomplate + +# Запустить весь стек +docker-compose up + +# Открыть Grafana +open http://localhost:3000 +``` diff --git a/config/alertrules.yml b/config/alertrules.yml new file mode 100644 index 0000000..9dd8a7a --- /dev/null +++ b/config/alertrules.yml @@ -0,0 +1,10 @@ +groups: + - name: display + rules: + - alert: DisplayDown + expr: state_connected == 0 + for: 1h + labels: + severity: critical + annotations: + summary: "Display {{ $labels.display_id }} down" diff --git a/config/grafana/provisioning/dashboards/dashboard.yml b/config/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..ff7db10 --- /dev/null +++ b/config/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: "Prometheus" + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/config/grafana/provisioning/dashboards/display.json b/config/grafana/provisioning/dashboards/display.json new file mode 100644 index 0000000..1cde220 --- /dev/null +++ b/config/grafana/provisioning/dashboards/display.json @@ -0,0 +1,175 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ + { + "title": "", + "url": "" + } + ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "repeat": "targets", + "repeatDirection": "v", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "state_last_forecasts_count{target=~\"$targets\"}", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Display {{display_id}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Last Forecasts Count for $targets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": true, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 21, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "datasource": "Prometheus", + "definition": "state_up", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "targets", + "options": [], + "query": "state_up", + "refresh": 1, + "regex": "/target=\\\"(.*)\\\"/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Display State", + "uid": "WYXKDQUWz", + "version": 6 +} diff --git a/config/grafana/provisioning/datasources/datasource.yml b/config/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..bb37f13 --- /dev/null +++ b/config/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: true + editable: true \ No newline at end of file diff --git a/docker-compose.config.yml b/docker-compose.config.yml new file mode 100644 index 0000000..8af3464 --- /dev/null +++ b/docker-compose.config.yml @@ -0,0 +1,19 @@ +version: "3" + +services: + gomplate: + image: hairyhenderson/gomplate + container_name: gomplate + command: "--input-dir /templates --output-dir /config" + volumes: + - ./templates:/templates + - ./config:/config + environment: + - TARGETS=${TARGETS} + - SCRAPE_INTERVAL=${SCRAPE_INTERVAL} + - ALERT_EMAIL_TO=${ALERT_EMAIL_TO} + - ALERT_EMAIL_FROM=${ALERT_EMAIL_FROM} + - ALERT_SMTP_HOST=${ALERT_SMTP_HOST} + - ALERT_SMTP_USER=${ALERT_SMTP_USER} + - ALERT_SMTP_PASSWORD=${ALERT_SMTP_PASSWORD} + - ALERT_SLACK_WEBHOOK=${ALERT_SLACK_WEBHOOK} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5b09050 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,65 @@ +version: "3" + +services: + prometheus: + image: prom/prometheus:v2.15.2 + container_name: prometheus + restart: unless-stopped + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--web.console.libraries=/etc/prometheus/console_libraries" + - "--web.console.templates=/etc/prometheus/consoles" + - "--web.enable-lifecycle" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention=200h" + networks: + - monitoring + volumes: + - prometheus_data:/prometheus + - ./config/prometheus.yml:/etc/prometheus/prometheus.yml + - ./config/alertrules.yml:/etc/prometheus/alertrules.yml + ports: + - 9090:9090 + + display: + build: + context: ./exporter + container_name: display + restart: unless-stopped + networks: + - monitoring + + alertmanager: + image: prom/alertmanager:v0.20.0 + container_name: alertmanager + restart: unless-stopped + command: + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--storage.path=/alertmanager" + networks: + - monitoring + volumes: + - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml + + grafana: + image: grafana/grafana:6.5.3 + container_name: grafana + restart: unless-stopped + networks: + - monitoring + volumes: + - grafana_data:/var/lib/grafana + - ./config/grafana/provisioning:/etc/grafana/provisioning + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_USER} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} + - GF_USERS_ALLOW_SIGN_UP=false + ports: + - 3000:3000 + +networks: + monitoring: {} + +volumes: + prometheus_data: {} + grafana_data: {} diff --git a/exporter/Dockerfile b/exporter/Dockerfile new file mode 100644 index 0000000..d9ba915 --- /dev/null +++ b/exporter/Dockerfile @@ -0,0 +1,11 @@ +FROM library/python:3.7-slim + +WORKDIR /app + +COPY requirements.txt /app/ +RUN pip3.7 install -r requirements.txt + +COPY exporter.py /app/ +EXPOSE 9115 + +CMD ["python3", "exporter.py"] diff --git a/exporter/exporter.py b/exporter/exporter.py new file mode 100644 index 0000000..d871938 --- /dev/null +++ b/exporter/exporter.py @@ -0,0 +1,93 @@ +import os +import sys +import json +import aiohttp +import logging + +from collections import namedtuple +from functools import reduce +from aiohttp import web + +PORT = os.getenv("PORT", 9115) + +UP_HELP = "# HELP state_up State Up/Down status." +UP_TYPE = "# TYPE state_up gauge" +UP_FORMAT = "state_up %d" + +CONNECTED_HELP = "# HELP state_connected Connected status." +CONNECTED_TYPE = "# TYPE state_connected gauge" +CONNECTED_FORMAT = "state_connected{display_id=\"%d\"} %d" + +LAST_DURATION_HELP = "# HELP state_last_duration Last duration." +LAST_DURATION_TYPE = "# TYPE state_last_duration gauge" +LAST_DURATION_FORMAT = "state_last_duration{display_id=\"%d\"} %d" + +LAST_FORECASTS_COUNT_HELP = "# HELP state_last_forecasts_count Last forecasts count." +LAST_FORECASTS_COUNT_TYPE = "# TYPE state_last_forecasts_count gauge" +LAST_FORECASTS_COUNT_FORMAT = "state_last_forecasts_count{display_id=\"%d\"} %d" + +Metrics = namedtuple("Metrics", "connected duration forecasts") + +def reducer(metrics, state): + display_id = int(state["DisplayId"]) + + forecasts = -1 + if state["Connected"]: + forecasts = int(state["LastForecastsCount"]) + + metrics.connected.append(CONNECTED_FORMAT % (display_id, int(state["Connected"]))) + metrics.duration.append(LAST_DURATION_FORMAT % (display_id, int(state["LastDuration"]))) + metrics.forecasts.append(LAST_FORECASTS_COUNT_FORMAT % (display_id, forecasts)) + + return metrics + +def format_metrics(up, metrics): + if up: + return "\n".join([ + UP_HELP, + UP_TYPE, + UP_FORMAT % int(up), + CONNECTED_HELP, + CONNECTED_TYPE, + "\n".join(metrics.connected), + LAST_DURATION_HELP, + LAST_DURATION_TYPE, + "\n".join(metrics.duration), + LAST_FORECASTS_COUNT_HELP, + LAST_FORECASTS_COUNT_TYPE, + "\n".join(metrics.forecasts), + ]) + + return "\n".join([UP_HELP, UP_TYPE, UP_FORMAT % int(up)]) + + +async def persistent_session(app): + app["PERSISTENT_SESSION"] = aiohttp.ClientSession() + yield + await app["PERSISTENT_SESSION"].close() + +async def fetch(session, url): + async with session.get(url) as response: + data = await response.read() + return json.loads(data) + +async def handle(request): + try: + target = request.query["target"] + session = request.app["PERSISTENT_SESSION"] + data = await fetch(session, target) + states = data["GetDisplaysStatesResult"]["DisplayState"] + metrics = reduce(reducer, states, Metrics([], [], [])) + + return web.Response(text=format_metrics(True, metrics)) + except: + return web.Response(text=format_metrics(False, None)) + +logging.basicConfig(stream=sys.stdout) + +app = web.Application() +app.router.add_get('/probe', handle) +app.cleanup_ctx.append(persistent_session) + +if __name__ == '__main__': + web.run_app(app, port=PORT) diff --git a/exporter/requirements.txt b/exporter/requirements.txt new file mode 100644 index 0000000..675ab08 --- /dev/null +++ b/exporter/requirements.txt @@ -0,0 +1 @@ +aiohttp==3.6.2 diff --git a/templates/alertmanager.yml b/templates/alertmanager.yml new file mode 100644 index 0000000..842b96f --- /dev/null +++ b/templates/alertmanager.yml @@ -0,0 +1,17 @@ +route: + receiver: "default" + +receivers: + - name: "default" + email_configs: + - to: "{{ .Env.ALERT_EMAIL_TO }}" + from: "{{ .Env.ALERT_EMAIL_FROM }}" + smarthost: "{{ .Env.ALERT_SMTP_HOST }}" + auth_username: "{{ .Env.ALERT_SMTP_USER }}" + auth_password: "{{ .Env.ALERT_SMTP_PASSWORD }}" + send_resolved: true + + slack_configs: + - api_url: "{{ .Env.ALERT_SLACK_WEBHOOK }}" + send_resolved: true + diff --git a/templates/prometheus.yml b/templates/prometheus.yml new file mode 100644 index 0000000..c62a118 --- /dev/null +++ b/templates/prometheus.yml @@ -0,0 +1,38 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: "docker-host" + +rule_files: + - alertrules.yml + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + + - job_name: "grafana" + static_configs: + - targets: ["grafana:3000"] + + - job_name: "display" + metrics_path: /probe + scrape_interval: {{ .Env.SCRAPE_INTERVAL }} + static_configs: + - targets: + {{ range (split .Env.TARGETS ",") }} - "{{ . }}" + {{ end }} + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: target + - target_label: __address__ + replacement: display:9115 + +alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: ["alertmanager:9093"]