Initial commit

.env.example

@@ -0,0 +1,10 @@
+TARGETS=https://smt-develop2.ru/displays.php?r=GetDisplaysStates&p=15047
+SCRAPE_INTERVAL=15s
+GRAFANA_USER=admin
+GRAFANA_PASSWORD=admin
+ALERT_EMAIL_TO=example@example.com
+ALERT_EMAIL_FROM=alert@example.com
+ALERT_SMTP_HOST=smtp.example.com:25
+ALERT_SMTP_USER=smtp_user
+ALERT_SMTP_PASSWORD=smtp_password
+ALERT_SLACK_WEBHOOK=https://slack.com/webhook

.gitignore

@@ -0,0 +1,3 @@
+.env
+config/alertmanager.yml
+config/prometheus.yml

README.md

@@ -0,0 +1,13 @@
+```
+# После этого задать переменные в файле .env
+cp .env.example .env
+
+# Сгенерировать конфиги на основе установленных в файле .env переменных
+docker-compose -f docker-compose.config.yml run gomplate
+
+# Запустить весь стек
+docker-compose up
+
+# Открыть Grafana
+open http://localhost:3000
+```

config/alertrules.yml

@@ -0,0 +1,10 @@
+groups:
+  - name: display
+    rules:
+    - alert: DisplayDown
+      expr: state_connected == 0
+      for: 1h
+      labels:
+        severity: critical
+      annotations:
+        summary: "Display {{ $labels.display_id }} down"

config/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: "Prometheus"
+    orgId: 1
+    folder: ""
+    type: file
+    disableDeletion: false
+    editable: true
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/provisioning/dashboards

config/grafana/provisioning/dashboards/display.json

@@ -0,0 +1,175 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fill": 0,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": [
+          {
+            "title": "",
+            "url": ""
+          }
+        ]
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "repeat": "targets",
+      "repeatDirection": "v",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "state_last_forecasts_count{target=~\"$targets\"}",
+          "format": "time_series",
+          "hide": false,
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "Display {{display_id}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Last Forecasts Count for $targets",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "transparent": true,
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "schemaVersion": 21,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "datasource": "Prometheus",
+        "definition": "state_up",
+        "hide": 0,
+        "includeAll": false,
+        "label": null,
+        "multi": true,
+        "name": "targets",
+        "options": [],
+        "query": "state_up",
+        "refresh": 1,
+        "regex": "/target=\\\"(.*)\\\"/",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ]
+  },
+  "timezone": "",
+  "title": "Display State",
+  "uid": "WYXKDQUWz",
+  "version": 6
+}

config/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    orgId: 1
+    url: http://prometheus:9090
+    basicAuth: false
+    isDefault: true
+    editable: true

docker-compose.config.yml

@@ -0,0 +1,19 @@
+version: "3"
+
+services:
+  gomplate:
+    image: hairyhenderson/gomplate
+    container_name: gomplate
+    command: "--input-dir /templates --output-dir /config"
+    volumes:
+      - ./templates:/templates
+      - ./config:/config
+    environment:
+      - TARGETS=${TARGETS}
+      - SCRAPE_INTERVAL=${SCRAPE_INTERVAL}
+      - ALERT_EMAIL_TO=${ALERT_EMAIL_TO}
+      - ALERT_EMAIL_FROM=${ALERT_EMAIL_FROM}
+      - ALERT_SMTP_HOST=${ALERT_SMTP_HOST}
+      - ALERT_SMTP_USER=${ALERT_SMTP_USER}
+      - ALERT_SMTP_PASSWORD=${ALERT_SMTP_PASSWORD}
+      - ALERT_SLACK_WEBHOOK=${ALERT_SLACK_WEBHOOK}

docker-compose.yml

@@ -0,0 +1,65 @@
+version: "3"
+
+services:
+  prometheus:
+    image: prom/prometheus:v2.15.2
+    container_name: prometheus
+    restart: unless-stopped
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--web.console.libraries=/etc/prometheus/console_libraries"
+      - "--web.console.templates=/etc/prometheus/consoles"
+      - "--web.enable-lifecycle"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention=200h"
+    networks:
+      - monitoring
+    volumes:
+      - prometheus_data:/prometheus
+      - ./config/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./config/alertrules.yml:/etc/prometheus/alertrules.yml
+    ports:
+      - 9090:9090
+
+  display:
+    build:
+      context: ./exporter
+    container_name: display
+    restart: unless-stopped
+    networks:
+      - monitoring
+
+  alertmanager:
+    image: prom/alertmanager:v0.20.0
+    container_name: alertmanager
+    restart: unless-stopped
+    command:
+      - "--config.file=/etc/alertmanager/alertmanager.yml"
+      - "--storage.path=/alertmanager"
+    networks:
+      - monitoring
+    volumes:
+      - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+
+  grafana:
+    image: grafana/grafana:6.5.3
+    container_name: grafana
+    restart: unless-stopped
+    networks:
+      - monitoring
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./config/grafana/provisioning:/etc/grafana/provisioning
+    environment:
+      - GF_SECURITY_ADMIN_USER=${GRAFANA_USER}
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
+      - GF_USERS_ALLOW_SIGN_UP=false
+    ports:
+      - 3000:3000
+
+networks:
+  monitoring: {}
+
+volumes:
+  prometheus_data: {}
+  grafana_data: {}

exporter/Dockerfile

@@ -0,0 +1,11 @@
+FROM library/python:3.7-slim
+
+WORKDIR /app
+
+COPY requirements.txt /app/
+RUN pip3.7 install -r requirements.txt
+
+COPY exporter.py /app/
+EXPOSE 9115
+
+CMD ["python3", "exporter.py"]

exporter/exporter.py

@@ -0,0 +1,93 @@
+import os
+import sys
+import json
+import aiohttp
+import logging
+
+from collections import namedtuple
+from functools import reduce
+from aiohttp import web
+
+PORT = os.getenv("PORT", 9115)
+
+UP_HELP = "# HELP state_up State Up/Down status."
+UP_TYPE = "# TYPE state_up gauge"
+UP_FORMAT = "state_up %d"
+
+CONNECTED_HELP = "# HELP state_connected Connected status."
+CONNECTED_TYPE = "# TYPE state_connected gauge"
+CONNECTED_FORMAT = "state_connected{display_id=\"%d\"} %d"
+
+LAST_DURATION_HELP = "# HELP state_last_duration Last duration."
+LAST_DURATION_TYPE = "# TYPE state_last_duration gauge"
+LAST_DURATION_FORMAT = "state_last_duration{display_id=\"%d\"} %d"
+
+LAST_FORECASTS_COUNT_HELP = "# HELP state_last_forecasts_count Last forecasts count."
+LAST_FORECASTS_COUNT_TYPE = "# TYPE state_last_forecasts_count gauge"
+LAST_FORECASTS_COUNT_FORMAT = "state_last_forecasts_count{display_id=\"%d\"} %d"
+
+Metrics = namedtuple("Metrics", "connected duration forecasts")
+
+def reducer(metrics, state):
+  display_id = int(state["DisplayId"])
+
+  forecasts = -1
+  if state["Connected"]:
+    forecasts = int(state["LastForecastsCount"])
+
+  metrics.connected.append(CONNECTED_FORMAT % (display_id, int(state["Connected"])))
+  metrics.duration.append(LAST_DURATION_FORMAT % (display_id, int(state["LastDuration"])))
+  metrics.forecasts.append(LAST_FORECASTS_COUNT_FORMAT % (display_id, forecasts))
+
+  return metrics
+
+def format_metrics(up, metrics):
+  if up:
+    return "\n".join([
+      UP_HELP,
+      UP_TYPE,
+      UP_FORMAT % int(up),
+      CONNECTED_HELP,
+      CONNECTED_TYPE,
+      "\n".join(metrics.connected),
+      LAST_DURATION_HELP,
+      LAST_DURATION_TYPE,
+      "\n".join(metrics.duration),
+      LAST_FORECASTS_COUNT_HELP,
+      LAST_FORECASTS_COUNT_TYPE,
+      "\n".join(metrics.forecasts),
+    ])
+
+  return "\n".join([UP_HELP, UP_TYPE, UP_FORMAT % int(up)])
+
+
+async def persistent_session(app):
+  app["PERSISTENT_SESSION"] = aiohttp.ClientSession()
+  yield
+  await app["PERSISTENT_SESSION"].close()
+
+async def fetch(session, url):
+  async with session.get(url) as response:
+    data = await response.read()
+    return json.loads(data)
+
+async def handle(request):
+  try:
+    target = request.query["target"]
+    session = request.app["PERSISTENT_SESSION"]
+    data = await fetch(session, target)
+    states = data["GetDisplaysStatesResult"]["DisplayState"]
+    metrics = reduce(reducer, states, Metrics([], [], []))
+
+    return web.Response(text=format_metrics(True, metrics))
+  except:
+    return web.Response(text=format_metrics(False, None))
+
+logging.basicConfig(stream=sys.stdout)
+
+app = web.Application()
+app.router.add_get('/probe', handle)
+app.cleanup_ctx.append(persistent_session)
+
+if __name__ == '__main__':
+  web.run_app(app, port=PORT)

exporter/requirements.txt

@@ -0,0 +1 @@
+aiohttp==3.6.2

templates/alertmanager.yml

@@ -0,0 +1,17 @@
+route:
+  receiver: "default"
+
+receivers:
+  - name: "default"
+    email_configs:
+      - to: "{{ .Env.ALERT_EMAIL_TO }}"
+        from: "{{ .Env.ALERT_EMAIL_FROM }}"
+        smarthost: "{{ .Env.ALERT_SMTP_HOST }}"
+        auth_username: "{{ .Env.ALERT_SMTP_USER }}"
+        auth_password: "{{ .Env.ALERT_SMTP_PASSWORD }}"
+        send_resolved: true
+
+    slack_configs:
+      - api_url: "{{ .Env.ALERT_SLACK_WEBHOOK }}"
+        send_resolved: true
+

templates/prometheus.yml

@@ -0,0 +1,38 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+  external_labels:
+    monitor: "docker-host"
+
+rule_files:
+  - alertrules.yml
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  - job_name: "grafana"
+    static_configs:
+      - targets: ["grafana:3000"]
+
+  - job_name: "display"
+    metrics_path: /probe
+    scrape_interval: {{ .Env.SCRAPE_INTERVAL }}
+    static_configs:
+      - targets:
+      {{ range (split .Env.TARGETS ",") }}  - "{{ . }}"
+      {{ end }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: target
+      - target_label: __address__
+        replacement: display:9115
+
+alerting:
+  alertmanagers:
+    - scheme: http
+      static_configs:
+        - targets: ["alertmanager:9093"]