prometheus+alertmanager告警

部署docker-compose.yaml

mkdir -p /data/docker-compose/prometheus
cat > /data/docker-compose/prometheus/docker-compose.yml <<EOF
networks:
  monitor:
    driver: bridge

services:
  prometheus:
    image: prom/prometheus
    container_name: prometheus
    hostname: prometheus
    restart: always
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--web.enable-admin-api'
      - '--storage.tsdb.retention.time=7d'
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/rules.yaml:/etc/prometheus/rules/rules.yaml
      - /etc/localtime:/etc/localtime
      - ./prometheus/data:/prometheus/
    ports:
      - "9090:9090"
    networks:
      - monitor

  alertmanager:
    image: prom/alertmanager
    container_name: alertmanager
    hostname: alertmanager
    restart: always
    volumes:
      - /etc/localtime:/etc/localtime
      - ./alertmanager/emal.tmpl:/etc/alertmanager/email.tmpl
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
    ports:
      - "9093:9093"
    networks:
      - monitor

  grafana:
    image: grafana/grafana
    container_name: grafana
    hostname: grafana
    restart: always
    volumes:
      - /etc/localtime:/etc/localtime
      - ./grafana/data:/var/lib/grafana
      #- ./grafana/conf/sample.ini:/usr/share/grafana/conf/sample.ini
      - ./grafana/conf/grafana.ini:/etc/grafana/grafana.ini
    ports:
      - "3000:3000"
    networks:
      - monitor
EOF

创建相关配置目录

cd /data/docker-compose/prometheus
mkdir alertmanager grafana prometheus

创建alertmanager配置文件

cat > alertmanager/emal.tmpl <<EOF
<h2>@告警通知</h2>
告警程序: prometheus_alert <br>
告警级别: {{ .Labels.severity }} 级 <br>
告警类型: {{ .Labels.alertname }} <br>
故障主机: {{ .Labels.instance }} <br>
告警主题: {{ .Annotations.summary }} <br>
告警详情: {{ .Annotations.description }} <br>
触发时间: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }} <br>
{{ end }}{{ end -}}
{{- if gt (len .Alerts.Resolved) 0 -}}{{ range .Alerts }}
<h2>@告警恢复</h2>
告警程序: prometheus_alert <br>
故障主机: {{ .Labels.instance }}<br>
故障主题: {{ .Annotations.summary }}<br>
告警详情: {{ .Annotations.description }}<br>
告警时间: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }}<br>
恢复时间: {{ .EndsAt.Local.Format "2006-01-02 15:04:05" }}<br>
{{ end }}{{ end -}}
{{- end }}
EOF

cat > alertmanager/alertmanager.yml <<EOF
global:
  resolve_timeout: 30s
  smtp_smarthost: 'smtp.163.com:465' 
  smtp_from: 'demo@163.com'              # 你的邮箱
  smtp_auth_username: 'demo@163.com'     # 你的邮箱
  smtp_auth_password: 'DCFVGYHBNJMKLOIU' # QQ邮箱授权码，需要在设置，开启SMTP服务，然后复制授权码到这里
  smtp_hello: '163.com'                  # QQ:qq.com 163:163.com   162:162.com
  smtp_require_tls: false

templates:
  - '/etc/alertmanager/email.tmpl'

route:
  group_by: ['env','instance','type','group','job','alertname']
  group_wait: 30s
  group_interval: 1m
  repeat_interval: 1h
  receiver: 'email'

receivers:
  - name: 'email'
    email_configs:
      - to: 'demo@163.com'         # 你的邮箱
        html: '{{ template "email.to.html" . }}'
        send_resolved: true

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

创建prometheus配置文件

cat > prometheus/prometheus.yml <<EOF
global:
  scrape_interval:     20s  # 调整为15秒，减轻负载
  evaluation_interval: 15s
  #scrape_timeout: 240s

# 配置Alertmanager地址
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - 192.168.4.20:9093  # 保持你的IP:端口配置

rule_files:
  - "/etc/prometheus/rules/rules.yaml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['192.168.4.20:9090']
        labels:
          instance: 'prometheus-server'

  - job_name: 'cloud'
    static_configs:
      - targets: ['192.168.4.20:9100']
        labels:
          instance: '192.168.4.20'
EOF

cat > prometheus/rules.yaml <<EOF
groups:
- name: server-alarm
  rules:
  - alert: "内存告警"
    expr: (1 - (node_memory_MemAvailable_bytes / (node_memory_MemTotal_bytes))) * 100 > 90
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高内存 使用率！"
      description: "{{$labels.instance}}: 内存使用率在 90% 以上"
      value: "{{ printf \"%.0f\" $value }}%"

  - alert: "CPU告警"
    expr: (1 - avg(irate(node_cpu_seconds_total{mode="idle"}[2m])) by(instance)) * 100 > 80
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高CPU 使用率！"
      description: "{{$labels.instance}}: CPU使用率在 80% 以上"
      value: "{{ printf \"%.0f\" $value }}%"

  - alert: "系统盘告警"
    expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} * 100) > 80
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！"
      description: "{{$labels.instance}}: / 磁盘使用率在 80% 以上"
      value: "{{ printf \"%.0f\" $value }}%"

  - alert: "数据盘告警"
    expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/data"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/data"} * 100) > 80
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！"
      description: "{{$labels.instance}}: /data 磁盘使用率在 80% 以上"
      value: "{{ printf \"%.0f\" $value }}%"

  - alert: "磁盘使用率过高"
    expr: (1 - node_filesystem_free_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"}) * 100 > 80
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 磁盘使用率过高"
      description: "{{$labels.instance}} 挂载点 {{$labels.mountpoint}} 使用率超过80%"
      value: "{{ printf \"%.0f\" $value }}%"
EOF

部署node_exporter

wget https://packages.emal.cc/package/softinstall/produce/prometheus/prometheus.tar.gz
tar -zxvf prometheus.tar.gz -C /data

# 创建service服务
cat > /usr/lib/systemd/system/node_exporter.service <<EOF
[Service]
ExecStart=/data/prometheus/node_exporter-1.9.1.linux-386/node_exporter --web.listen-address=0.0.0.0:25070 --collector.textfile.directory=/opt/node_exporter/text/
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure


[Install]
WantedBy=multi-user.target

[Unit]
Description=node_exporter
After=network.target
EOF

# /opt/node_exporter/text/ 这是自定义的采集，可以写脚本来获取，如下
cat > /opt/node_exporter/text/text_runner <<EOF
#!/usr/bin/env bash

METRIC_FILE="/opt/node_exporter/text/text.prom"

# 容器总数
dockers_all=$(docker ps -aq|wc -l)

# 容器运行个数
dockers_running=$(docker ps -a|awk 'NR>1{print $NF}'|wc -l)

# 异常容器
dockers_Exit=$(docker ps -a|grep "Ex"|wc -l)

# 可用内存
frees_status=$(free -m|awk 'NR==2{print $4}')

# buff
frees_buff=$(free -m|awk 'NR==2{print $6}')

# frees_used
frees_used=$(free -m|awk 'NR==2{print $3}')

# 写入临时文件
cat > "$METRIC_FILE" << EOF
# HELP dockers_all docker容器总个数
# TYPE dockers_all gauge
dockers_all ${dockers_all}

# HELP dockers_Exit docker容器停止个数
# TYPE dockers_Exit gauge
dockers_Exit ${dockers_Exit}

# HELP dockers_running docker容器运行个数
# TYPE dockers_running gauge
dockers_running ${dockers_running}

# HELP frees_status 可用内存
# TYPE frees_status gauge
frees_status ${frees_status}

# HELP frees_buff 缓存
# TYPE frees_buff gauge
frees_buff ${frees_buff}

# HELP frees_used 系统使用
# TYPE frees_used gauge
frees_used ${frees_used}

# HELP docker_exit_container 异常的容器名称
# TYPE docker_exit_container gauge
EOF

docker ps -a | awk '/Exited \(0\)/ {
    print "docker_exit_container{name=\"" $NF "\"} 1"
}' >> "$METRIC_FILE"
EOF

# text.prom 文件就是text_runner执行后的结果

部署docker-compose.yaml

docker-compose up -d

查询是否能够获取到数据

curl http://localhost:9100/metrics | grep "dockers_running"