prometheus+alertmanager告警 部署docker-compose.yamlmkdir -p /data/docker-compose/prometheus cat > /data/docker-compose/prometheus/docker-compose.yml <<EOF networks: monitor: driver: bridge services: prometheus: image: prom/prometheus container_name: prometheus hostname: prometheus restart: always command: - '--config.file=/etc/prometheus/prometheus.yml' - '--web.enable-admin-api' - '--storage.tsdb.retention.time=7d' volumes: - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - ./prometheus/rules.yaml:/etc/prometheus/rules/rules.yaml - /etc/localtime:/etc/localtime - ./prometheus/data:/prometheus/ ports: - "9090:9090" networks: - monitor alertmanager: image: prom/alertmanager container_name: alertmanager hostname: alertmanager restart: always volumes: - /etc/localtime:/etc/localtime - ./alertmanager/emal.tmpl:/etc/alertmanager/email.tmpl - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml ports: - "9093:9093" networks: - monitor grafana: image: grafana/grafana container_name: grafana hostname: grafana restart: always volumes: - /etc/localtime:/etc/localtime - ./grafana/data:/var/lib/grafana #- ./grafana/conf/sample.ini:/usr/share/grafana/conf/sample.ini - ./grafana/conf/grafana.ini:/etc/grafana/grafana.ini ports: - "3000:3000" networks: - monitor EOF创建相关配置目录cd /data/docker-compose/prometheus mkdir alertmanager grafana prometheus创建alertmanager配置文件cat > alertmanager/emal.tmpl <<EOF <h2>@告警通知</h2> 告警程序: prometheus_alert <br> 告警级别: {{ .Labels.severity }} 级 <br> 告警类型: {{ .Labels.alertname }} <br> 故障主机: {{ .Labels.instance }} <br> 告警主题: {{ .Annotations.summary }} <br> 告警详情: {{ .Annotations.description }} <br> 触发时间: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }} <br> {{ end }}{{ end -}} {{- if gt (len .Alerts.Resolved) 0 -}}{{ range .Alerts }} <h2>@告警恢复</h2> 告警程序: prometheus_alert <br> 故障主机: {{ .Labels.instance }}<br> 故障主题: {{ .Annotations.summary }}<br> 告警详情: {{ .Annotations.description }}<br> 告警时间: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }}<br> 恢复时间: {{ .EndsAt.Local.Format "2006-01-02 15:04:05" }}<br> {{ end }}{{ end -}} {{- end }} EOF cat > alertmanager/alertmanager.yml <<EOF global: resolve_timeout: 30s smtp_smarthost: 'smtp.163.com:465' smtp_from: 'demo@163.com' # 你的邮箱 smtp_auth_username: 'demo@163.com' # 你的邮箱 smtp_auth_password: 'DCFVGYHBNJMKLOIU' # QQ邮箱授权码，需要在设置，开启SMTP服务，然后复制授权码到这里 smtp_hello: '163.com' # QQ:qq.com 163:163.com 162:162.com smtp_require_tls: false templates: - '/etc/alertmanager/email.tmpl' route: group_by: ['env','instance','type','group','job','alertname'] group_wait: 30s group_interval: 1m repeat_interval: 1h receiver: 'email' receivers: - name: 'email' email_configs: - to: 'demo@163.com' # 你的邮箱 html: '{{ template "email.to.html" . }}' send_resolved: true inhibit_rules: - source_match: severity: 'critical' target_match: severity: 'warning' equal: ['alertname', 'dev', 'instance']创建prometheus配置文件cat > prometheus/prometheus.yml <<EOF global: scrape_interval: 20s # 调整为15秒，减轻负载 evaluation_interval: 15s #scrape_timeout: 240s # 配置Alertmanager地址 alerting: alertmanagers: - static_configs: - targets: - 192.168.4.20:9093 # 保持你的IP:端口配置 rule_files: - "/etc/prometheus/rules/rules.yaml" scrape_configs: - job_name: 'prometheus' static_configs: - targets: ['192.168.4.20:9090'] labels: instance: 'prometheus-server' - job_name: 'cloud' static_configs: - targets: ['192.168.4.20:9100'] labels: instance: '192.168.4.20' EOF cat > prometheus/rules.yaml <<EOF groups: - name: server-alarm rules: - alert: "内存告警" expr: (1 - (node_memory_MemAvailable_bytes / (node_memory_MemTotal_bytes))) * 100 > 90 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高内存 使用率！" description: "{{$labels.instance}}: 内存使用率在 90% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "CPU告警" expr: (1 - avg(irate(node_cpu_seconds_total{mode="idle"}[2m])) by(instance)) * 100 > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高CPU 使用率！" description: "{{$labels.instance}}: CPU使用率在 80% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "系统盘告警" expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} * 100) > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！" description: "{{$labels.instance}}: / 磁盘使用率在 80% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "数据盘告警" expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/data"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/data"} * 100) > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！" description: "{{$labels.instance}}: /data 磁盘使用率在 80% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "磁盘使用率过高" expr: (1 - node_filesystem_free_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"}) * 100 > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 磁盘使用率过高" description: "{{$labels.instance}} 挂载点 {{$labels.mountpoint}} 使用率超过80%" value: "{{ printf \"%.0f\" $value }}%" EOF部署node_exporterwget https://packages.emal.cc/package/softinstall/produce/prometheus/prometheus.tar.gz tar -zxvf prometheus.tar.gz -C /data # 创建service服务 cat > /usr/lib/systemd/system/node_exporter.service <<EOF [Service] ExecStart=/data/prometheus/node_exporter-1.9.1.linux-386/node_exporter --web.listen-address=0.0.0.0:25070 --collector.textfile.directory=/opt/node_exporter/text/ ExecReload=/bin/kill -HUP $MAINPID KillMode=process Restart=on-failure [Install] WantedBy=multi-user.target [Unit] Description=node_exporter After=network.target EOF # /opt/node_exporter/text/ 这是自定义的采集，可以写脚本来获取，如下 cat > /opt/node_exporter/text/text_runner <<EOF #!/usr/bin/env bash METRIC_FILE="/opt/node_exporter/text/text.prom" # 容器总数 dockers_all=$(docker ps -aq|wc -l) # 容器运行个数 dockers_running=$(docker ps -a|awk 'NR>1{print $NF}'|wc -l) # 异常容器 dockers_Exit=$(docker ps -a|grep "Ex"|wc -l) # 可用内存 frees_status=$(free -m|awk 'NR==2{print $4}') # buff frees_buff=$(free -m|awk 'NR==2{print $6}') # frees_used frees_used=$(free -m|awk 'NR==2{print $3}') # 写入临时文件 cat > "$METRIC_FILE" << EOF # HELP dockers_all docker容器总个数 # TYPE dockers_all gauge dockers_all ${dockers_all} # HELP dockers_Exit docker容器停止个数 # TYPE dockers_Exit gauge dockers_Exit ${dockers_Exit} # HELP dockers_running docker容器运行个数 # TYPE dockers_running gauge dockers_running ${dockers_running} # HELP frees_status 可用内存 # TYPE frees_status gauge frees_status ${frees_status} # HELP frees_buff 缓存 # TYPE frees_buff gauge frees_buff ${frees_buff} # HELP frees_used 系统使用 # TYPE frees_used gauge frees_used ${frees_used} # HELP docker_exit_container 异常的容器名称 # TYPE docker_exit_container gauge EOF docker ps -a | awk '/Exited \(0\)/ { print "docker_exit_container{name=\"" $NF "\"} 1" }' >> "$METRIC_FILE" EOF # text.prom 文件就是text_runner执行后的结果部署docker-compose.yamldocker-compose up -d查询是否能够获取到数据curl http://localhost:9100/metrics | grep "dockers_running"

prometheus+grafana实现监控告警 更新grafana 邮件配置# 部署见连接: https://cloud.prd.emal.cc/index.php/archives/92/ # 在docker-compose.yaml 里面添加映射将这个配置文件里面的[smtp]启用 - ./grafana/conf/grafana.ini:/etc/grafana/grafana.ini #在docker-compose.yaml添加这样 修改文件grafana.ini [smtp] enabled = true host = smtp.163.com:465 user = demo@163.com #改成你的邮箱 password = SDCFWRDXCGGHNJKB #改成你的邮箱16位授权码 skip_verify = true from_address = demo@163.com #改为你的邮箱 from_name = Grafana # 完成后启动docker-compose docker-compose down && docker-compose up -d编辑grafana页面创建告警{{ define "alert" }} {{- if gt (len .Alerts.Firing) 0 -}}{{ range .Alerts }} =============================== @告警通知 告警程序: Grafana 告警级别: {{ .Labels.severity }} 告警类型: {{ .Labels.alertname }} 故障主机: {{ .Labels.instance }} 告警主题: {{ .Annotations.summary }} 告警详情: {{ .Annotations.description }} 触发时间: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }} 静默地址: https://grafana.prd.emal.cc/alerting/list =============================== {{ end }}{{ end -}} {{- if gt (len .Alerts.Resolved) 0 -}}{{ range .Alerts }} =============================== @告警恢复 告警程序: prometheus_alert 故障主机: {{ .Labels.instance }} 故障主题: {{ .Annotations.summary }} 告警详情: {{ .Annotations.description }} 告警时间: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }} 恢复时间: {{ .EndsAt.Local.Format "2006-01-02 15:04:05" }} =============================== {{ end }}{{ end -}} {{ end }}更多里面你可以开启静默这些操作等

企业微信告警配置 创建企业微信机器人1、企业微信创建群聊2、创建机器人，记住webhook地址创建prometheus-alert# 企业微信卡片 { "msgtype": "news", "news": { "articles": [ {{ range .alerts }} { "title": "{{ if eq .status "resolved" }}✅【已恢复】{{ .labels.alertname }}{{ else }}🚨【告警通知】{{ .labels.alertname }}{{ end }}", "description": "级别: {{ .labels.severity }}\n主机: {{ .labels.instance }}\n{{ if eq .status "resolved" }}开始: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}\n结束: {{ slice .endsAt 0 10 }} {{ slice .endsAt 11 19 }}{{ else }}时间: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}\n当前值: {{ .annotations.value }}{{ end }}\n详情: {{ .annotations.description }}", "url": "https://alert.prd.emal.cc/", "picurl": "https://package.prd.emal.cc/package/img/prometheus-alter.png" } {{ end }} ] } }# 企业微信文本格式 { "msgtype": "markdown", "markdown": { "content": "{{ range .alerts }}{{ if eq .status "resolved" }}**✅【已恢复】** **{{ .labels.alertname }}**\n\n**级别**: **{{ .labels.severity }}**\n**主机**: **{{ .labels.instance }}**\n**开始**: **{{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}**\n**结束**: **{{ slice .endsAt 0 10 }} {{ slice .endsAt 11 19 }}**\n**详情**: **{{ .annotations.description }}**\n{{ else }}**🚨【告警通知】** **{{ .labels.alertname }}**\n\n**级别**: **{{ .labels.severity }}**\n**主机**: **{{ .labels.instance }}**\n**时间**: **{{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}**\n**当前值**: **{{ .annotations.value }}**\n**详情**: **{{ .annotations.description }}**\n{{ end }}{{ end }}" } }alertmanager.yml配置docker-compose restart alertmanager

飞书告警值班轮流@人员配置 创建值班轮询脚本cat > /usr/local/bin/alert_duty_rotator.sh << EOF #!/usr/bin/env bash # ===================================================== # 告警值班轮询脚本 - 每天自动@当日值班人员 # 功能：从Alertmanager获取告警，发送到飞书并@值班人 # ===================================================== # 飞书Webhook地址 FEISHU_WEBHOOK="https://aa.com.cm/open-apis/bot/v2/hook/a6eb1f09-cb69-45cf-80bd-dc880e99c2a8" # 值班人员列表（邮箱地址，按顺序轮询） DUTY_USERS=( "zhangsan@cnpc.com.cn" ) # 值班人员对应的飞书用户ID（如果有的话） # 如果飞书机器人支持通过邮箱@，则不需要这个映射 # DUTY_USER_IDS=( # "zhangshan_id" # "lisi_id" # "wanger_id" # ) # Alertmanager API地址 ALERTMANAGER_API="http://192.168.4.20:9093/api/v2/alerts" # 日志文件 LOG_FILE="/var/log/alert_duty_rotator.log" # 已发送告警缓存文件（用于跟踪告警状态变化） ALERT_CACHE_FILE="/tmp/alert_sent_cache.txt" # ===================================================== # 函数：初始化缓存文件 # ===================================================== init_cache() { if [ ! -f "$ALERT_CACHE_FILE" ]; then touch "$ALERT_CACHE_FILE" fi } # ===================================================== # 函数：获取今日值班人员 # ===================================================== get_today_duty() { # 获取今天是今年的第几天（1-365/366） DAY_OF_YEAR=$(date +%j | sed 's/^0*//') # 计算索引（从0开始） INDEX=$(( (DAY_OF_YEAR - 1) % ${#DUTY_USERS[@]} )) # 返回今天值班人员的邮箱和索引 TODAY_EMAIL="${DUTY_USERS[$INDEX]}" TODAY_NAME=$(echo $TODAY_EMAIL | cut -d'@' -f1) TODAY_INDEX=$INDEX echo "$TODAY_EMAIL|$TODAY_NAME|$TODAY_INDEX" } # ===================================================== # 函数：从Alertmanager获取所有告警 # ===================================================== get_all_alerts() { # 调用Alertmanager API获取所有告警 curl -s "$ALERTMANAGER_API" | jq -c '.[]' } # ===================================================== # 函数：获取当前所有活跃告警的指纹列表 # ===================================================== get_current_fingerprints() { local alerts=$(get_all_alerts) local fingerprints="" if [ -n "$alerts" ]; then fingerprints=$(echo "$alerts" | jq -r '.fingerprint // ""' | grep -v "^$" | tr '\n' ' ') fi echo "$fingerprints" } # ===================================================== # 函数：检查告警是否已发送过 # ===================================================== is_alert_sent() { local alert_fingerprint="$1" if grep -q "^${alert_fingerprint}|" "$ALERT_CACHE_FILE"; then return 0 # 已发送过 fi return 1 # 未发送过 } # ===================================================== # 函数：标记告警为已发送 # ===================================================== mark_alert_sent() { local alert_fingerprint="$1" local alert_info="$2" # 先删除已有的记录 sed -i "/^${alert_fingerprint}|/d" "$ALERT_CACHE_FILE" # 添加新记录 echo "${alert_fingerprint}|${alert_info}|$(date +%s)" >> "$ALERT_CACHE_FILE" } # ===================================================== # 函数：从缓存中删除告警 # ===================================================== remove_alert_from_cache() { local alert_fingerprint="$1" sed -i "/^${alert_fingerprint}|/d" "$ALERT_CACHE_FILE" } # ===================================================== # 函数：构建飞书告警卡片（支持活跃和恢复） # ===================================================== build_feishu_card() { local alert_json="$1" local duty_email="$2" local duty_name="$3" local alert_type="$4" # "active" 或 "resolved" # 解析告警信息 local alertname=$(echo "$alert_json" | jq -r '.labels.alertname // "未知"') local severity=$(echo "$alert_json" | jq -r '.labels.severity // "未知"') local instance=$(echo "$alert_json" | jq -r '.labels.instance // "未知"') local starts_at=$(echo "$alert_json" | jq -r '.startsAt // "未知"') local ends_at=$(echo "$alert_json" | jq -r '.endsAt // "未知"') local description=$(echo "$alert_json" | jq -r '.annotations.description // "无"') local value=$(echo "$alert_json" | jq -r '.annotations.value // "无"') # 格式化时间（去掉T和Z，更易读） if [ "$starts_at" != "0001-01-01T00:00:00Z" ] && [ "$starts_at" != "未知" ]; then start_time=$(echo "$starts_at" | sed 's/T/ /' | sed 's/Z//' | cut -d'.' -f1) else start_time=$(date '+%Y-%m-%d %H:%M:%S') fi # 判断告警类型并构建相应的卡片 if [ "$alert_type" = "resolved" ]; then # 恢复告警 - 使用绿色模板 if [ "$ends_at" != "0001-01-01T00:00:00Z" ] && [ "$ends_at" != "未知" ]; then end_time=$(echo "$ends_at" | sed 's/T/ /' | sed 's/Z//' | cut -d'.' -f1) else end_time=$(date '+%Y-%m-%d %H:%M:%S') fi # 构建恢复卡片 cat <<EOF { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "✅ 【已恢复】 ${alertname}" }, "template": "green" }, "elements": [ { "tag": "markdown", "content": "**主题**: ${alertname}\n**级别**: ${severity}\n**主机**: ${instance}\n**开始**: ${start_time}\n**结束**: ${end_time}\n**详情**: ${description}\n<at email=${duty_email}>${duty_name}</at>" } ] } } EOF else # 活跃告警 - 根据级别设置颜色 case "$severity" in "critical"|"紧急") template="red" emoji="🚨" ;; "warning"|"警告") template="red" emoji="⚠️" ;; *) template="red" emoji="🔔" ;; esac # 构建活跃告警卡片 cat <<EOF { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "${emoji} 【告警中】 ${alertname}" }, "template": "${template}" }, "elements": [ { "tag": "markdown", "content": "**主题**: ${alertname}\n**级别**: ${severity}\n**主机**: ${instance}\n**时间**: ${start_time}\n**当前值**: ${value}\n**详情**: ${description}\n<at email=${duty_email}>${duty_name}</at>" } ] } } EOF fi } # ===================================================== # 函数：构建值班提醒卡片（每天定时发送） # ===================================================== build_duty_reminder_card() { local duty_email="$1" local duty_name="$2" cat <<EOF { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "demo云告警信息" }, "template": "green" }, "elements": [ { "tag": "markdown", "content": "**resolved每日提醒: 今天该你值班了**\n**运维工具：**\n 运维平台：https://aa.com.cn\n 监控平台：https://aa.com.cn\n 火狐浏览器：https://aa.com.cn\n 镜像仓库：https://aa.com.cn\n 文件服务器：https://aa.com.cn\n**今日值班:** <at email=${duty_email}>${duty_name}></at>\n\n -------------------------------------\n demo云告警信息" } ] } } EOF } # ===================================================== # 函数：发送告警到飞书 # ===================================================== send_to_feishu() { local json_data="$1" response=$(curl -s -X POST -H "Content-Type: application/json" \ -d "$json_data" \ $FEISHU_WEBHOOK) # 判断飞书返回的成功响应 if [[ $response == *"\"code\":0"* ]] || [[ $response == *"\"Success\""* ]] || [[ $response == *"success"* ]] || [[ $response == *"\"msg\":\"success\""* ]]; then echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 飞书消息发送成功" >> $LOG_FILE return 0 else echo "$(date '+%Y-%m-%d %H:%M:%S') [ERROR] 飞书发送失败: $response" >> $LOG_FILE return 1 fi } # ===================================================== # 函数：处理活跃告警（新告警） # ===================================================== process_active_alerts() { local duty_info="$1" local duty_email=$(echo $duty_info | cut -d'|' -f1) local duty_name=$(echo $duty_info | cut -d'|' -f2) # 获取所有告警 alerts=$(get_all_alerts) if [ -n "$alerts" ]; then echo "$alerts" | while read -r alert; do # 获取告警信息 alert_status=$(echo "$alert" | jq -r '.status.state') alert_fingerprint=$(echo "$alert" | jq -r '.fingerprint // ""') # 如果没有fingerprint，使用其他字段组合 if [ -z "$alert_fingerprint" ] || [ "$alert_fingerprint" = "null" ]; then alertname=$(echo "$alert" | jq -r '.labels.alertname') instance=$(echo "$alert" | jq -r '.labels.instance') alert_fingerprint="${alertname}_${instance}" fi # 只处理活跃告警 if [ "$alert_status" = "active" ]; then # 检查是否已经发送过 if ! is_alert_sent "$alert_fingerprint"; then # 构建飞书卡片 card_json=$(build_feishu_card "$alert" "$duty_email" "$duty_name" "active") # 发送告警 if send_to_feishu "$card_json"; then # 标记为已发送（保存完整告警信息以便后续恢复使用） alert_info=$(echo "$alert" | jq -c '{labels: .labels, annotations: .annotations, startsAt: .startsAt}') mark_alert_sent "$alert_fingerprint" "$alert_info" echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 已发送活跃告警 [${alert_fingerprint}] 给值班人员 $duty_name" >> $LOG_FILE fi # 避免发送过快 sleep 1 fi fi done fi } # ===================================================== # 函数：检查已恢复的告警（从API中消失的） # ===================================================== check_resolved_alerts() { local duty_info="$1" local duty_email=$(echo $duty_info | cut -d'|' -f1) local duty_name=$(echo $duty_info | cut -d'|' -f2) # 获取当前所有活跃告警的指纹 local current_fingerprints=$(get_current_fingerprints) # 将当前指纹转换为数组便于查找 declare -A current_fp_map for fp in $current_fingerprints; do current_fp_map["$fp"]=1 done # 检查缓存中的告警 if [ -f "$ALERT_CACHE_FILE" ]; then while IFS= read -r line; do if [ -n "$line" ]; then cached_fingerprint=$(echo "$line" | cut -d'|' -f1) cached_info=$(echo "$line" | cut -d'|' -f2) # 如果缓存中的指纹不在当前活跃告警列表中（说明已从API消失） if [[ -z "${current_fp_map[$cached_fingerprint]}" ]]; then echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 告警已从API消失，发送恢复通知: $cached_fingerprint" >> $LOG_FILE # 从缓存信息中恢复告警数据 if [ -n "$cached_info" ] && [ "$cached_info" != "null" ]; then # 构建恢复卡片 card_json=$(build_feishu_card "$cached_info" "$duty_email" "$duty_name" "resolved") # 发送恢复通知 if send_to_feishu "$card_json"; then # 从缓存中删除 remove_alert_from_cache "$cached_fingerprint" echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 已发送恢复通知 [${cached_fingerprint}]" >> $LOG_FILE fi else # 如果没有缓存信息，直接删除 remove_alert_from_cache "$cached_fingerprint" echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 清理无效缓存 [${cached_fingerprint}]" >> $LOG_FILE fi # 避免发送过快 sleep 1 fi fi done < "$ALERT_CACHE_FILE" fi } # ===================================================== # 函数：处理所有告警（分开处理活跃和恢复） # ===================================================== process_alerts() { local duty_info="$1" echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 开始处理告警..." >> $LOG_FILE # 第一步：处理活跃告警（新告警） process_active_alerts "$duty_info" # 第二步：检查已恢复的告警（从API消失的） check_resolved_alerts "$duty_info" echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 告警处理完成" >> $LOG_FILE } # ===================================================== # 函数：发送每日值班提醒 # ===================================================== send_duty_reminder() { local duty_info="$1" local duty_email=$(echo $duty_info | cut -d'|' -f1) local duty_name=$(echo $duty_info | cut -d'|' -f2) # 构建值班提醒卡片 reminder_json=$(build_duty_reminder_card "$duty_email" "$duty_name") # 发送提醒 if send_to_feishu "$reminder_json"; then echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 已发送值班提醒给 $duty_name" >> $LOG_FILE fi } # ===================================================== # 函数：发送每日总结 # ===================================================== send_duty_summary(){ local duty_info="$1" local duty_email=$(echo $duty_info | cut -d'|' -f1) local duty_name=$(echo $duty_info | cut -d'|' -f2) JSON_DATA=$(cat <<EOF { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "自主可控云告警信息" }, "template": "green" }, "elements": [ { "tag": "markdown", "content": "**resolved每日提醒: 下班也不要放松警惕呦**\n请总结今日值班情况\n**今日值班:** <at email=${duty_email}>${duty_name}</at>\n\n -------------------------------------\n 自主可控云告警信息" } ] } } EOF ) # 发送到飞书 response=$(curl -s -X POST -H "Content-Type: application/json" \ -d "$JSON_DATA" \ $FEISHU_WEBHOOK) # 判断飞书返回的成功响应 if [[ $response == *"\"code\":0"* ]] || [[ $response == *"\"Success\""* ]] || [[ $response == *"success"* ]] || [[ $response == *"\"msg\":\"success\""* ]]; then echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 飞书消息发送成功" >> $LOG_FILE return 0 else echo "$(date '+%Y-%m-%d %H:%M:%S') [ERROR] 飞书发送失败: $response" >> $LOG_FILE return 1 fi } # ===================================================== # 函数：清理过期缓存（保留24小时） # ===================================================== cleanup_cache() { local current_time=$(date +%s) local cache_expire=86400 # 24小时 if [ -f "$ALERT_CACHE_FILE" ]; then while IFS= read -r line; do if [ -n "$line" ]; then cache_time=$(echo "$line" | cut -d'|' -f3) if [ -n "$cache_time" ] && [ $((current_time - cache_time)) -gt $cache_expire ]; then fingerprint=$(echo "$line" | cut -d'|' -f1) sed -i "/^${fingerprint}|/d" "$ALERT_CACHE_FILE" echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 清理过期缓存: $fingerprint" >> $LOG_FILE fi fi done < "$ALERT_CACHE_FILE" fi } # ===================================================== # 函数：调试模式 - 查看缓存和当前告警 # ===================================================== debug_mode() { echo "=== 当前缓存内容 ===" if [ -f "$ALERT_CACHE_FILE" ]; then cat "$ALERT_CACHE_FILE" else echo "缓存文件不存在" fi echo "" echo "=== 当前API中的告警 ===" curl -s "$ALERTMANAGER_API" | jq -r '.[] | "指纹: \(.fingerprint // "无") 状态: \(.status.state)"' 2>/dev/null || echo "无法获取告警" echo "" } # ===================================================== # 主函数 # ===================================================== main() { # 初始化缓存 init_cache # 清理过期缓存 cleanup_cache # 获取今日值班信息 duty_info=$(get_today_duty) # 检查命令行参数 case "${1:-}" in "reminder") # 只发送值班提醒 send_duty_reminder "$duty_info" ;; "summary") # 每日下班提醒 send_duty_summary "$duty_info" ;; "alerts") # 只处理告警 process_alerts "$duty_info" ;; "cleanup") # 只清理缓存 cleanup_cache ;; "debug") # 调试模式 debug_mode ;; *) # 默认：处理告警 process_alerts "$duty_info" ;; esac } # 执行主函数 main "$@" EOF安装依赖和配置# 安装jq（用于解析JSON） yum install -y jq # CentOS/RHEL # 或 apt-get install -y jq # Ubuntu/Debian # 创建日志文件 touch /var/log/alert_duty_rotator.log chmod 644 /var/log/alert_duty_rotator.log # 添加执行权限 chmod +x /usr/local/bin/alert_duty_rotator.sh配置Alertmanager Webhookglobal: resolve_timeout: 30s route: group_by: ['alertname'] group_wait: 10s group_interval: 10s repeat_interval: 1h # 将所有告警发送到 webhook 接收器 receiver: 'prometheus-alert' receivers: - name: 'prometheus-alert' webhook_configs: - url: 'http://192.168.1.20:8085/prometheusalert?type=email&tpl=prometheus-email&email=zhangsan@163.com' # Prometheus-Alert 的接收地址 - url: 'http://192.168.1.20:8086/alert' # 需要创建一个简单的webhook接收服务 send_resolved: true # 发送恢复通知创建简单的Webhook接收服务由于Shell脚本不能直接作为Webhook服务运行，需要创建一个简单的HTTP服务。以下是使用Python Flask的示例 /opt/webhook/alert_webhook.py#!/usr/bin/env python3 from flask import Flask, request, jsonify import subprocess import json import os app = Flask(__name__) # 告警脚本路径 ALERT_SCRIPT = "/usr/local/bin/alert_duty_rotator.sh" @app.route('/alert', methods=['POST']) def handle_alert(): """接收Alertmanager的告警并触发告警处理""" try: # 可以选择将告警数据保存到文件，供脚本处理 alert_data = request.json # 保存告警数据到临时文件 with open('/tmp/latest_alert.json', 'w') as f: json.dump(alert_data, f) # 触发告警处理脚本（后台运行） subprocess.Popen([ALERT_SCRIPT, "alerts"]) return jsonify({"status": "success"}), 200 except Exception as e: return jsonify({"status": "error", "message": str(e)}), 500 @app.route('/health', methods=['GET']) def health(): return jsonify({"status": "healthy"}), 200 if __name__ == '__main__': app.run(host='0.0.0.0', port=8080, debug=False)安装并启动Webhook服务pip3 install flask cat > /usr/lib/systemd/system/webhook.service << EOF [Unit] Description=webhook service After=network.target syslog.target Wants=network.target [Service] Type=simple #写你的frps的需要用绝对路径 ExecStartPre=/bin/sleep 1 ExecStart=/usr/bin/python3 /opt/webhook/alert_webhook.py #logs路径 StandardOutput=null [Install] WantedBy=multi-user.target EOF systemctl damon-reload systemctl start webhook systemctl enable webhook systemctl status webhook配置定时任务# 编辑crontab crontab -e # 添加以下内容 # 每天上午9点发送值班提醒 0 9 * * * /usr/local/bin/alert_duty_rotator.sh reminder >> /var/log/alert_duty_cron.log 2>&1 # 每天下午发送总结提醒 0 17 * * * /usr/local/bin/alert_duty_rotator.sh summary >> /var/log/alert_duty_cron.log 2>&1整体流程prometheus收集数据->发送给alertmanager->py程序实时获取alertmanager告警数据->脚本实时获取发送获取的告警信息并发送飞书@值班人员

prometheus-alert告警部署 部署prometheus# docker-compose.yaml version: '2.4' networks: monitor: driver: bridge services: prometheus: image: prom/prometheus container_name: prometheus hostname: prometheus restart: always command: - '--config.file=/etc/prometheus/prometheus.yml' - '--web.enable-admin-api' - '--storage.tsdb.retention.time=7d' volumes: - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - ./prometheus/rules.yaml:/etc/prometheus/rules/rules.yaml - /etc/localtime:/etc/localtime ports: - "9090:9090" networks: - monitor alertmanager: image: prom/alertmanager container_name: alertmanager hostname: alertmanager restart: always volumes: - /etc/localtime:/etc/localtime - ./alertmanager/emal.tmpl:/etc/alertmanager/email.tmpl - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml ports: - "9093:9093" networks: - monitor grafana: image: grafana/grafana container_name: grafana hostname: grafana restart: always volumes: - /etc/localtime:/etc/localtime - ./grafana/data:/var/lib/grafana ports: - "3000:3000" networks: - monitor创建目录mkdir -p alertmanager grafana prometheus chmod -R 755 alertmanager grafana prometheus创建文件prometheuscat > prometheus/prometheus.yml << EOF global: scrape_interval: 20s # 调整为15秒，减轻负载 evaluation_interval: 15s #scrape_timeout: 240s # 配置Alertmanager地址 alerting: alertmanagers: - static_configs: - targets: - 192.168.4.20:9093 # 保持你的IP:端口配置 rule_files: - "/etc/prometheus/rules/rules.yaml" scrape_configs: - job_name: 'prometheus' static_configs: - targets: ['192.168.4.20:9090'] labels: instance: 'prometheus-server' # - job_name: 'cadvisor' # static_configs: # - targets: ['192.168.4.20:8899'] # 这里改回映射的宿主机端口8899（之前的8080是容器内端口） # labels: # instance: 'cadvisor-monitor' - job_name: 'cloud' static_configs: - targets: ['192.168.4.20:9100'] labels: instance: '192.168.4.20' - targets: ['192.168.4.21:9100'] labels: instance: '192.168.4.21' - targets: ['192.168.4.22:9100'] labels: instance: '192.168.4.22' EOFcat > rules.yaml << EOF groups: - name: server-alarm rules: - alert: "内存告警" expr: (1 - (node_memory_MemAvailable_bytes / (node_memory_MemTotal_bytes))) * 100 > 90 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高内存 使用率！" description: "{{$labels.instance}}: 内存使用率在 90% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "CPU告警" expr: (1 - avg(irate(node_cpu_seconds_total{mode="idle"}[2m])) by(instance)) * 100 > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高CPU 使用率！" description: "{{$labels.instance}}: CPU使用率在 80% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "系统盘告警" expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} * 100) > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！" description: "{{$labels.instance}}: / 磁盘使用率在 80% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "数据盘告警" expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/data"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/data"} * 100) > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！" description: "{{$labels.instance}}: /data 磁盘使用率在 80% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "磁盘使用率过高" expr: (1 - node_filesystem_free_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"}) * 100 > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 磁盘使用率过高" description: "{{$labels.instance}} 挂载点 {{$labels.mountpoint}} 使用率超过80%" value: "{{ printf \"%.0f\" $value }}%" - alert: "typecho备份数据异常" expr: typecho_backup_total != 4 for: 30s labels: severity: warning annotations: summary: "typecho备份数据异常" description: "{{ $labels.instance }} typecho当前备份总数不等于预期值4" value: "{{ $value }}" # - alert: "测试" # expr: up == 1 # for: 30s # labels: # severity: warning # annotations: # summary: "{{ $labels.instance }}: 实例在线" # description: "{{ $labels.instance }} of job {{ $labels.job }} 已在线" # value: "up" EOF创建文件alertmanager# 邮件告警 cat > alertmanager.yml << EOF global: resolve_timeout: 30s route: group_by: ['alertname'] group_wait: 10s group_interval: 10s repeat_interval: 1h # 将所有告警发送到 webhook 接收器 receiver: 'prometheus-alert' receivers: - name: 'prometheus-alert' webhook_configs: - url: 'http://192.168.4.20:8085/prometheusalert?type=email&tpl=prometheus-email&email=kexinzz@163.com' # Prometheus-Alert 的接收地址 send_resolved: true # 发送恢复通知 EOF # 飞书告警 cat > alertmanager.yml << EOf global: resolve_timeout: 1m route: group_by: ['env','instance','type','group','job','alertname'] # 根据告警名和实例分组 group_wait: 30s # 组告警等待时间 group_interval: 2m # 组告警间隔 repeat_interval: 4h # 重复告警间隔 receiver: 'prometheusalert-feishu' # 默认接收器 receivers: - name: 'prometheusalert-feishu' webhook_configs: - url: 'http://192.168.4.20:8085/prometheusalert?type=fs&tpl=prometheus-fs&fsurl=https://kexinzz.cn.com/open-apis/bot/v2/hook/4b874766-75f0-4d45-9d7a-dced826ead54' send_resolved: true # 添加 HTTP 配置以确保正确发送 http_config: follow_redirects: true EOF部署prometheus-alert# docker-compose.yaml cat > docker-compose.yaml << EOF services: prometheus-alert: image: feiyu563/prometheus-alert:v4.9.2 container_name: prometheus-alert restart: unless-stopped ports: - "8085:8080" # 宿主机的8085映射到容器的8080 environment: PA_LOGIN_USER: admin PA_LOGIN_PASSWORD: Xzm7_GTD.com PA_TITLE: PrometheusAlert PA_OPEN_FEISHU: "1" PA_FSURL: "https://kexinzz.cn.com/open-apis/bot/v2/hook/4b874766-75f0-4d45-9d7a-dced826ead54" PA_OPEN_DINGDING: "0" PA_OPEN_WEIXIN: "0" volumes: # 【关键修改】使用宿主机目录挂载 - ./data/:/app/data/ # 挂载数据目录（告警记录、日志等） - ./conf/:/app/conf/ # 挂载配置文件目录 - ./logs/:/app/logs/ # 挂载日志目录 - ./db:/app/db EOF创建文件mkdir -p data logs conf db chmod -R 755 data logs conf db ##### 添加配置文件 conf/app.conf wget https://package.prd.emal.cc/package/softinstall/doc/doc/app.conf更换相对应的配置运行docker-compose# 运行prometheus、prometheus-alert yaml docker-compose up -d登录地址# prometheus-alert http://127.0.0.1:8085 # alertmanager http://127.0.0.1:9093 # prometheus http://127.0.0.1:9090 # grafana http://127.0.0.1:3000prometheus-alert登录prometheus-alert 邮件配置# 邮件 {{ range .alerts }} {{ if eq .status "resolved" }} <h2>@告警恢复</h2> <b>主题:</b> {{ .labels.alertname }} <br> <b>级别:</b> {{ .labels.severity }} <br> <b>主机:</b> {{ .labels.instance }} <br> <b>开始</b>: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }} <br> <b>结束:</b> {{ slice .endsAt 0 10 }} {{ slice .endsAt 11 19 }} <br> <b>详情</b>: {{ .annotations.description }} <br> {{ else }} <h2>@告警通知</h2> <b>主题:</b> {{ .labels.alertname }} <br> <b>级别:</b> {{ .labels.severity }} <br> <b>主机:</b> {{ .labels.instance }} <br> <b>时间:</b> {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }} <br> <b>当前值:</b> {{ .annotations.value }} <br> <b>详情:</b> {{ .annotations.description }} <br> {{ end }} {{ end }}prometheus-alert 飞书配置# 飞书 {{ range .alerts }} {{ if eq .status "resolved" }} { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "测试告警" }, "template": "green" }, "elements": [ { "tag": "markdown", "content": "**主题** {{ .labels.alertname }}\n**级别**: {{ .labels.severity }}\n**主机**: {{ .labels.instance }}\n**开始**: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}\n**结束**: {{ slice .endsAt 0 10 }} {{ slice .endsAt 11 19 }}\n**详情**: {{ .annotations.description }}" } ] } } {{ else }} { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "测试告警" }, "template": "red" }, "elements": [ { "tag": "markdown", "content": "**主题** {{ .labels.alertname }}\n**级别**: {{ .labels.severity }}\n**主机**: {{ .labels.instance }}\n**时间**: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}\n**当前值**: {{ .annotations.value }}\n**详情**: {{ .annotations.description }}" } ] } } {{ end }} {{ end }}每日提醒值班#!/usr/bin/env bash # 飞书Webhook地址 WEBHOOK_URL="https://test.com.cc/open-apis/bot/v2/hook/a6eb1f09-cb69-45cf-80bd-dc880e99c2a8" # 值班人员列表（邮箱地址） USERS=( "zhangshan@cnpc.com.cn" "lisi@cnpc.com.cn" ) # 获取今天是今年的第几天（1-365/366） DAY_OF_YEAR=$(date +%j | sed 's/^0*//') # 计算索引（从0开始） INDEX=$(( (DAY_OF_YEAR - 1) % ${#USERS[@]} )) # 获取今天值班人员的邮箱 TODAY_USER=${USERS[$INDEX]} # 获取值班人员姓名（从邮箱提取，可根据需要修改） TODAY_NAME=$(echo $TODAY_USER | cut -d'@' -f1) # 构建JSON数据 JSON_DATA=$(cat <<EOF { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "demo云告警信息" }, "template": "green" }, "elements": [ { "tag": "markdown", "content": "**resolved每日提醒: 今天该你值班了**\n**运维工具：**\n 运维平台：https://aa.com.cn\n 监控平台：https://aa.com.cn\n 火狐浏览器：https://aa.com.cn\n 镜像仓库：https://aa.com.cn\n 文件服务器：https://aa.com.cn\n**今日值班:** <at email=${TODAY_USER}></at>\n\n -------------------------------------\n demo云告警信息" } ] } } EOF ) # 发送到飞书 curl -X POST -H "Content-Type: application/json" \ -d "$JSON_DATA" \ $WEBHOOK_URL # 输出日志 echo "$(date): 已发送值班提醒给 $TODAY_NAME ($TODAY_USER)"