prometheus-alert告警部署

部署prometheus

# docker-compose.yaml
version: '2.4'

networks:
  monitor:
    driver: bridge

services:
  prometheus:
    image: prom/prometheus
    container_name: prometheus
    hostname: prometheus
    restart: always
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--web.enable-admin-api'
      - '--storage.tsdb.retention.time=7d'
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/rules.yaml:/etc/prometheus/rules/rules.yaml
      - /etc/localtime:/etc/localtime
    ports:
      - "9090:9090"
    networks:
      - monitor

  alertmanager:
    image: prom/alertmanager
    container_name: alertmanager
    hostname: alertmanager
    restart: always
    volumes:
      - /etc/localtime:/etc/localtime
      - ./alertmanager/emal.tmpl:/etc/alertmanager/email.tmpl
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
    ports:
      - "9093:9093"
    networks:
      - monitor

  grafana:
    image: grafana/grafana
    container_name: grafana
    hostname: grafana
    restart: always
    volumes:
      - /etc/localtime:/etc/localtime
      - ./grafana/data:/var/lib/grafana
    ports:
      - "3000:3000"
    networks:
      - monitor

创建目录

mkdir -p alertmanager grafana prometheus
chmod -R 755 alertmanager grafana prometheus

创建文件prometheus

cat > prometheus/prometheus.yml << EOF
global:
  scrape_interval:     20s  # 调整为15秒，减轻负载
  evaluation_interval: 15s
  #scrape_timeout: 240s

# 配置Alertmanager地址
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - 192.168.4.20:9093  # 保持你的IP:端口配置

rule_files:
  - "/etc/prometheus/rules/rules.yaml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['192.168.4.20:9090']
        labels:
          instance: 'prometheus-server'

#  - job_name: 'cadvisor'
#    static_configs:
#      - targets: ['192.168.4.20:8899']  # 这里改回映射的宿主机端口8899（之前的8080是容器内端口）
#        labels:
#          instance: 'cadvisor-monitor'

  - job_name: 'cloud'
    static_configs:
      - targets: ['192.168.4.20:9100']
        labels:
          instance: '192.168.4.20'

      - targets: ['192.168.4.21:9100']
        labels:
          instance: '192.168.4.21'

      - targets: ['192.168.4.22:9100']
        labels:
          instance: '192.168.4.22'
EOF

cat > rules.yaml << EOF
groups:
- name: server-alarm
  rules:
  - alert: "内存告警"
    expr: (1 - (node_memory_MemAvailable_bytes / (node_memory_MemTotal_bytes))) * 100 > 90
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高内存 使用率！"
      description: "{{$labels.instance}}: 内存使用率在 90% 以上"
      value: "{{ printf \"%.0f\" $value }}%"

  - alert: "CPU告警"
    expr: (1 - avg(irate(node_cpu_seconds_total{mode="idle"}[2m])) by(instance)) * 100 > 80
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高CPU 使用率！"
      description: "{{$labels.instance}}: CPU使用率在 80% 以上"
      value: "{{ printf \"%.0f\" $value }}%"

  - alert: "系统盘告警"
    expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} * 100) > 80
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！"
      description: "{{$labels.instance}}: / 磁盘使用率在 80% 以上"
      value: "{{ printf \"%.0f\" $value }}%"

  - alert: "数据盘告警"
    expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/data"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/data"} * 100) > 80
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！"
      description: "{{$labels.instance}}: /data 磁盘使用率在 80% 以上"
      value: "{{ printf \"%.0f\" $value }}%"

  - alert: "磁盘使用率过高"
    expr: (1 - node_filesystem_free_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"}) * 100 > 80
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 磁盘使用率过高"
      description: "{{$labels.instance}} 挂载点 {{$labels.mountpoint}} 使用率超过80%"
      value: "{{ printf \"%.0f\" $value }}%"

  - alert: "typecho备份数据异常"
    expr: typecho_backup_total != 4
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "typecho备份数据异常"
      description: "{{ $labels.instance }} typecho当前备份总数不等于预期值4"
      value: "{{ $value }}"

#  - alert: "测试"
#    expr: up == 1
#    for: 30s
#    labels:
#      severity: warning
#    annotations:
#      summary: "{{ $labels.instance }}: 实例在线"
#      description: "{{ $labels.instance }} of job {{ $labels.job }} 已在线"
#      value: "up"
EOF

创建文件alertmanager

# 邮件告警
cat > alertmanager.yml << EOF
global:
  resolve_timeout: 30s

route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1h
  # 将所有告警发送到 webhook 接收器
  receiver: 'prometheus-alert'

receivers:
- name: 'prometheus-alert'
  webhook_configs:
  - url: 'http://192.168.4.20:8085/prometheusalert?type=email&tpl=prometheus-email&email=kexinzz@163.com'  # Prometheus-Alert 的接收地址
    send_resolved: true  # 发送恢复通知
EOF

# 飞书告警
cat > alertmanager.yml << EOf
global:
  resolve_timeout: 1m

route:
  group_by: ['env','instance','type','group','job','alertname']  # 根据告警名和实例分组
  group_wait: 30s                      # 组告警等待时间
  group_interval: 2m                    # 组告警间隔
  repeat_interval: 4h                   # 重复告警间隔
  receiver: 'prometheusalert-feishu'    # 默认接收器

receivers:
- name: 'prometheusalert-feishu'
  webhook_configs:
  - url: 'http://192.168.4.20:8085/prometheusalert?type=fs&tpl=prometheus-fs&fsurl=https://kexinzz.cn.com/open-apis/bot/v2/hook/4b874766-75f0-4d45-9d7a-dced826ead54'
    send_resolved: true
    # 添加 HTTP 配置以确保正确发送
    http_config:
      follow_redirects: true
EOF

部署prometheus-alert

# docker-compose.yaml
cat > docker-compose.yaml << EOF
services:
  prometheus-alert:
    image: feiyu563/prometheus-alert:v4.9.2
    container_name: prometheus-alert
    restart: unless-stopped
    ports:
      - "8085:8080"  # 宿主机的8085映射到容器的8080
    environment:
      PA_LOGIN_USER: admin
      PA_LOGIN_PASSWORD: Xzm7_GTD.com
      PA_TITLE: PrometheusAlert
      PA_OPEN_FEISHU: "1"
      PA_FSURL: "https://kexinzz.cn.com/open-apis/bot/v2/hook/4b874766-75f0-4d45-9d7a-dced826ead54"
      PA_OPEN_DINGDING: "0"
      PA_OPEN_WEIXIN: "0"

    volumes:
      # 【关键修改】使用宿主机目录挂载
      - ./data/:/app/data/          # 挂载数据目录（告警记录、日志等）
      - ./conf/:/app/conf/           # 挂载配置文件目录
      - ./logs/:/app/logs/           # 挂载日志目录
      - ./db:/app/db
EOF

创建文件

mkdir -p data logs conf db
chmod -R 755 data logs conf db

##### 添加配置文件 conf/app.conf
wget https://package.prd.emal.cc/package/softinstall/doc/doc/app.conf

更换相对应的配置

运行docker-compose

# 运行prometheus、prometheus-alert yaml
docker-compose up -d

登录地址

# prometheus-alert
http://127.0.0.1:8085

# alertmanager
http://127.0.0.1:9093

# prometheus
http://127.0.0.1:9090

# grafana
http://127.0.0.1:3000

prometheus-alert登录

prometheus-alert 邮件配置

# 邮件
{{ range .alerts }}
{{ if eq .status "resolved" }}
<h2>@告警恢复</h2>
<b>主题:</b> {{ .labels.alertname }} <br>
<b>级别:</b> {{ .labels.severity }} <br>
<b>主机:</b> {{ .labels.instance }} <br>
<b>开始</b>: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }} <br>
<b>结束:</b> {{ slice .endsAt 0 10 }} {{ slice .endsAt 11 19 }} <br>
<b>详情</b>: {{ .annotations.description }} <br>
{{ else }}
<h2>@告警通知</h2>
<b>主题:</b> {{ .labels.alertname }} <br>
<b>级别:</b> {{ .labels.severity }} <br>
<b>主机:</b> {{ .labels.instance }} <br>
<b>时间:</b> {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }} <br>
<b>当前值:</b> {{ .annotations.value }} <br>
<b>详情:</b> {{ .annotations.description }} <br>
{{ end }}
{{ end }}

prometheus-alert 飞书配置

# 飞书
{{ range .alerts }}
{{ if eq .status "resolved" }}
{
    "msg_type": "interactive",
    "card": {
        "config": {
            "wide_screen_mode": true
        },
        "header": {
            "title": {
                "tag": "plain_text",
                "content": "测试告警"
            },
            "template": "green"
        },
        "elements": [
            {
                "tag": "markdown",
                "content": "**主题** {{ .labels.alertname }}\n**级别**: {{ .labels.severity }}\n**主机**: {{ .labels.instance }}\n**开始**: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}\n**结束**: {{ slice .endsAt 0 10 }} {{ slice .endsAt 11 19 }}\n**详情**: {{ .annotations.description }}"
            }
        ]
    }
}
{{ else }}
{
    "msg_type": "interactive",
    "card": {
        "config": {
            "wide_screen_mode": true
        },
        "header": {
            "title": {
                "tag": "plain_text",
                "content": "测试告警"
            },
            "template": "red"
        },
        "elements": [
            {
                "tag": "markdown",
                "content": "**主题** {{ .labels.alertname }}\n**级别**: {{ .labels.severity }}\n**主机**: {{ .labels.instance }}\n**时间**: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}\n**当前值**: {{ .annotations.value }}\n**详情**: {{ .annotations.description }}"
            }
        ]
    }
}
{{ end }}
{{ end }}

每日提醒值班

#!/usr/bin/env bash

# 飞书Webhook地址
WEBHOOK_URL="https://test.com.cc/open-apis/bot/v2/hook/a6eb1f09-cb69-45cf-80bd-dc880e99c2a8"

# 值班人员列表（邮箱地址）
USERS=(
    "zhangshan@cnpc.com.cn"
    "lisi@cnpc.com.cn"
)

# 获取今天是今年的第几天（1-365/366）
DAY_OF_YEAR=$(date +%j | sed 's/^0*//')

# 计算索引（从0开始）
INDEX=$(( (DAY_OF_YEAR - 1) % ${#USERS[@]} ))

# 获取今天值班人员的邮箱
TODAY_USER=${USERS[$INDEX]}

# 获取值班人员姓名（从邮箱提取，可根据需要修改）
TODAY_NAME=$(echo $TODAY_USER | cut -d'@' -f1)

# 构建JSON数据
JSON_DATA=$(cat <<EOF
{
    "msg_type": "interactive",
    "card": {
        "config": {
            "wide_screen_mode": true
        },
        "header": {
            "title": {
                "tag": "plain_text",
                "content": "demo云告警信息"
            },
            "template": "green"
        },
        "elements": [
            {
                "tag": "markdown",
                "content": "**resolved每日提醒: 今天该你值班了**\n**运维工具：**\n    运维平台：https://aa.com.cn\n    监控平台：https://aa.com.cn\n    火狐浏览器：https://aa.com.cn\n    镜像仓库：https://aa.com.cn\n    文件服务器：https://aa.com.cn\n**今日值班:** <at email=${TODAY_USER}></at>\n\n -------------------------------------\n   demo云告警信息"
            }
        ]
    }
}
EOF
)

# 发送到飞书
curl -X POST -H "Content-Type: application/json" \
     -d "$JSON_DATA" \
     $WEBHOOK_URL

# 输出日志
echo "$(date): 已发送值班提醒给 $TODAY_NAME ($TODAY_USER)"

prometheus-alert告警部署

部署prometheus

创建目录

创建文件prometheus

创建文件alertmanager

部署prometheus-alert

创建文件

更换相对应的配置

运行docker-compose

登录地址

prometheus-alert 邮件配置

prometheus-alert 飞书配置

每日提醒值班

评论 (0)