docker部署prometheus告警钉钉通知

环境
centos7 <br>
docker 20.10.13 <br>
prometheus <br>
node_exporter
alertmanager
prometheus-webhook-dingtalk

prometheus安装

# 拉镜像
docker pull prom/prometheus
# 创建目录
mkdir -p /srv/prometheus/rules /srv/prometheus/data
# docker 安装
docker run --name prometheus  -p 9090:9090 \
-v /srv/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \
-v /srv/prometheus/rules:/etc/prometheus/rules \
-v /srv/prometheus/data:/data/prometheus \
-d prom/prometheus \
--config.file="/etc/prometheus/prometheus.yml" \
--web.listen-address="0.0.0.0:9090"

prometheus.yml

# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["localhost:9090"]

node_exporter安装

docker pull prometheus/node-exporter
# 建目录
mkdir -p  /srv/node_exporter/proc /srv/node_exporter/sys
docker run -d --name node_exporter -p 9100:9100 -v "/srv/node_exporter/proc:/host/proc:ro" -v "/srv/node_exporter/sys:/host/sys:ro" -v "/:/rootfs:ro" quay.io/prometheus/node-exporter --path.procfs=/host/proc --path.rootfs=/rootfs --path.sysfs=/host/sys --collector.filesystem.ignored-mount-points='^/(sys|proc|dev|host|etc)($|/)'

钉钉通知安装

# 拉取镜像
docker pull prometheus-webhook-dingtalk

dingtalk配置
config.yml

targets:
  webhook1:
    url: https://oapi.dingtalk.com/robot/send?access_token=344c8cb85b3ecd25746664d97445f3fb898f8e8bbb8fa0a9d24db5cb8f5445tr

docker安装

docker run -d -p 8060:8060 --name webhook timonwong/prometheus-webhook-dingtalk  \
-v /srv/webhook/config.yml:/etc/prometheus-webhook-dingtalk/config.yml \
--config.file=/etc/prometheus-webhook-dingtalk/config.yml

docker run -d -p 8060:8060 --name webhook timonwong/prometheus-webhook-dingtalk  \
-v /srv/webhook/config.yml:/etc/prometheus-webhook-dingtalk/config.yml \
-ding.profile="webhook1=https://oapi.dingtalk.com/robot/send?access_token=344c8cb85b3ecd25746664d97445f3fb898f8e8bbb8fa0a9d24db5cb8f5337ba"

==上面的两种指定挂在目录都会报错==

prometheus-webhook-dingtalk: error: path 'config.yml' does not exist, try --help

可以正常启动容器的方式

docker run -d -p 8060:8060 --name webhook timonwong/prometheus-webhook-dingtalk 

进入容器内修改配置文件

# 找到容器在宿主机目录
docker inspect webhook|grep Dir

目录

"LowerDir": "/var/lib/docker/overlay2/970538084a39222b2033dd7a4c4d301afb35ad868998de4f864b2d193855cd96-init/diff:/var/lib/docker/overlay2/65c4d6384949573adc6dad8c8219d4c4f3a119c987ae3419067432f4890ce547/diff:/var/lib/docker/overlay2/625c033578a975f9af4a7b00abb74b02211cc84e98603af2c08a8f7fe60e4798/diff:/var/lib/docker/overlay2/689f8f6d5639e0617f238d0eec2e4694dd2bea758c453f8f43a04f3e87058a37/diff:/var/lib/docker/overlay2/da966eb3c1f6be0a3a86c76ad0fe608b746062c8c18bcee380331f2f234dc0c2/diff",
                "MergedDir": "/var/lib/docker/overlay2/970538084a39222b2033dd7a4c4d301afb35ad868998de4f864b2d193855cd96/merged",
                "UpperDir": "/var/lib/docker/overlay2/970538084a39222b2033dd7a4c4d301afb35ad868998de4f864b2d193855cd96/diff",
                "WorkDir": "/var/lib/docker/overlay2/970538084a39222b2033dd7a4c4d301afb35ad868998de4f864b2d193855cd96/work"
            "WorkingDir": "/prometheus-webhook-dingtalk",

进入MergedDir

cd /var/lib/docker/overlay2/970538084a39222b2033dd7a4c4d301afb35ad868998de4f864b2d193855cd96/merged

cd etc/prometheus-webhook-dingtalk
# 修改配置文件
vim config.yml
替换自己的钉钉机器人token
重启docker
docker restart webhook

alertmanager安装

# 拉取镜像
docker pull prom/alertmanager
# 创建目录

# 部署
docker run -d -p 9093:9093 --name alertmanager \
    -v /srv/alertmanager/config/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
    -v /srv/alertmanager/data:/alertmanager/data \
    prom/alertmanager:latest \
    --config.file="/etc/alertmanager/alertmanager.yml" \
    --web.listen-address="0.0.0.0:9093"

alertmanager.yml

global:
  resolve_timeout: 5m
route:
  receiver: webhook
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 5m
  group_by: [alertname]
  routes:
  - receiver: webhook
    group_wait: 10s
receivers:
- name: webhook
  webhook_configs:
  - url: http://localhost:8060/dingtalk/webhook1/send  
    send_resolved: true

配置模版

配置rule

groups:
- name: nodeExporter
  rules:
  - alert: nodeExporterDown
    expr: up{job='node_exporter'} == 0
    for: 1m
    labels:
      severity: page
    # Prometheus templates apply here in the annotation and label fields of the alert.
      annotations:
        #description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes."
        #summary: "Instance {{ $labels.instance }} down"

上面description和summary放开会报错

 cannot unmarshal !!map into string

之前其他机器没问题，这次报错，不知道为什么

修改prometheus配置文件

rule_files:
  - "rules/*.yml"
  # - "first_rules.yml"
  # - "second_rules.yml"

重启prometheus

docker restart prometheus

验证钉钉通知

停止和重启node_exporter验证

docker stop node_exporter 

docker start node_exporter

钉钉通知

钉钉机器人设置