飞书告警值班轮流@人员配置

创建值班轮询脚本

cat > /usr/local/bin/alert_duty_rotator.sh << EOF
#!/usr/bin/env bash

# =====================================================
# 告警值班轮询脚本 - 每天自动@当日值班人员
# 功能：从Alertmanager获取告警，发送到飞书并@值班人
# =====================================================

# 飞书Webhook地址
FEISHU_WEBHOOK="https://aa.com.cm/open-apis/bot/v2/hook/a6eb1f09-cb69-45cf-80bd-dc880e99c2a8"

# 值班人员列表（邮箱地址，按顺序轮询）
DUTY_USERS=(
    "zhangsan@cnpc.com.cn"
)

# 值班人员对应的飞书用户ID（如果有的话）
# 如果飞书机器人支持通过邮箱@，则不需要这个映射
# DUTY_USER_IDS=(
#     "zhangshan_id"
#     "lisi_id" 
#     "wanger_id"
# )

# Alertmanager API地址
ALERTMANAGER_API="http://192.168.4.20:9093/api/v2/alerts"

# 日志文件
LOG_FILE="/var/log/alert_duty_rotator.log"

# 已发送告警缓存文件（用于跟踪告警状态变化）
ALERT_CACHE_FILE="/tmp/alert_sent_cache.txt"

# =====================================================
# 函数：初始化缓存文件
# =====================================================
init_cache() {
    if [ ! -f "$ALERT_CACHE_FILE" ]; then
        touch "$ALERT_CACHE_FILE"
    fi
}

# =====================================================
# 函数：获取今日值班人员
# =====================================================
get_today_duty() {
    # 获取今天是今年的第几天（1-365/366）
    DAY_OF_YEAR=$(date +%j | sed 's/^0*//')
    
    # 计算索引（从0开始）
    INDEX=$(( (DAY_OF_YEAR - 1) % ${#DUTY_USERS[@]} ))
    
    # 返回今天值班人员的邮箱和索引
    TODAY_EMAIL="${DUTY_USERS[$INDEX]}"
    TODAY_NAME=$(echo $TODAY_EMAIL | cut -d'@' -f1)
    TODAY_INDEX=$INDEX
    
    echo "$TODAY_EMAIL|$TODAY_NAME|$TODAY_INDEX"
}

# =====================================================
# 函数：从Alertmanager获取所有告警
# =====================================================
get_all_alerts() {
    # 调用Alertmanager API获取所有告警
    curl -s "$ALERTMANAGER_API" | jq -c '.[]'
}

# =====================================================
# 函数：获取当前所有活跃告警的指纹列表
# =====================================================
get_current_fingerprints() {
    local alerts=$(get_all_alerts)
    local fingerprints=""
    
    if [ -n "$alerts" ]; then
        fingerprints=$(echo "$alerts" | jq -r '.fingerprint // ""' | grep -v "^$" | tr '\n' ' ')
    fi
    
    echo "$fingerprints"
}

# =====================================================
# 函数：检查告警是否已发送过
# =====================================================
is_alert_sent() {
    local alert_fingerprint="$1"
    
    if grep -q "^${alert_fingerprint}|" "$ALERT_CACHE_FILE"; then
        return 0  # 已发送过
    fi
    return 1  # 未发送过
}

# =====================================================
# 函数：标记告警为已发送
# =====================================================
mark_alert_sent() {
    local alert_fingerprint="$1"
    local alert_info="$2"
    
    # 先删除已有的记录
    sed -i "/^${alert_fingerprint}|/d" "$ALERT_CACHE_FILE"
    
    # 添加新记录
    echo "${alert_fingerprint}|${alert_info}|$(date +%s)" >> "$ALERT_CACHE_FILE"
}

# =====================================================
# 函数：从缓存中删除告警
# =====================================================
remove_alert_from_cache() {
    local alert_fingerprint="$1"
    sed -i "/^${alert_fingerprint}|/d" "$ALERT_CACHE_FILE"
}

# =====================================================
# 函数：构建飞书告警卡片（支持活跃和恢复）
# =====================================================
build_feishu_card() {
    local alert_json="$1"
    local duty_email="$2"
    local duty_name="$3"
    local alert_type="$4"  # "active" 或 "resolved"
    
    # 解析告警信息
    local alertname=$(echo "$alert_json" | jq -r '.labels.alertname // "未知"')
    local severity=$(echo "$alert_json" | jq -r '.labels.severity // "未知"')
    local instance=$(echo "$alert_json" | jq -r '.labels.instance // "未知"')
    local starts_at=$(echo "$alert_json" | jq -r '.startsAt // "未知"')
    local ends_at=$(echo "$alert_json" | jq -r '.endsAt // "未知"')
    local description=$(echo "$alert_json" | jq -r '.annotations.description // "无"')
    local value=$(echo "$alert_json" | jq -r '.annotations.value // "无"')
    
    # 格式化时间（去掉T和Z，更易读）
    if [ "$starts_at" != "0001-01-01T00:00:00Z" ] && [ "$starts_at" != "未知" ]; then
        start_time=$(echo "$starts_at" | sed 's/T/ /' | sed 's/Z//' | cut -d'.' -f1)
    else
        start_time=$(date '+%Y-%m-%d %H:%M:%S')
    fi
    
    # 判断告警类型并构建相应的卡片
    if [ "$alert_type" = "resolved" ]; then
        # 恢复告警 - 使用绿色模板
        if [ "$ends_at" != "0001-01-01T00:00:00Z" ] && [ "$ends_at" != "未知" ]; then
            end_time=$(echo "$ends_at" | sed 's/T/ /' | sed 's/Z//' | cut -d'.' -f1)
        else
            end_time=$(date '+%Y-%m-%d %H:%M:%S')
        fi
        
        # 构建恢复卡片
        cat <<EOF
{
    "msg_type": "interactive",
    "card": {
        "config": {
            "wide_screen_mode": true
        },
        "header": {
            "title": {
                "tag": "plain_text",
                "content": "✅ 【已恢复】 ${alertname}"
            },
            "template": "green"
        },
        "elements": [
            {
                "tag": "markdown",
                "content": "**主题**: ${alertname}\n**级别**: ${severity}\n**主机**: ${instance}\n**开始**: ${start_time}\n**结束**: ${end_time}\n**详情**: ${description}\n<at email=${duty_email}>${duty_name}</at>"
            }
        ]
    }
}
EOF
    else
        # 活跃告警 - 根据级别设置颜色
        case "$severity" in
            "critical"|"紧急")
                template="red"
                emoji="🚨"
                ;;
            "warning"|"警告")
                template="red"
                emoji="⚠️"
                ;;
            *)
                template="red"
                emoji="🔔"
                ;;
        esac
        
        # 构建活跃告警卡片
        cat <<EOF
{
    "msg_type": "interactive",
    "card": {
        "config": {
            "wide_screen_mode": true
        },
        "header": {
            "title": {
                "tag": "plain_text",
                "content": "${emoji} 【告警中】 ${alertname}"
            },
            "template": "${template}"
        },
        "elements": [
            {
                "tag": "markdown",
                "content": "**主题**: ${alertname}\n**级别**: ${severity}\n**主机**: ${instance}\n**时间**: ${start_time}\n**当前值**: ${value}\n**详情**: ${description}\n<at email=${duty_email}>${duty_name}</at>"
            }
        ]
    }
}
EOF
    fi
}

# =====================================================
# 函数：构建值班提醒卡片（每天定时发送）
# =====================================================
build_duty_reminder_card() {
    local duty_email="$1"
    local duty_name="$2"
    
    cat <<EOF
{
    "msg_type": "interactive",
    "card": {
        "config": {
            "wide_screen_mode": true
        },
        "header": {
            "title": {
                "tag": "plain_text",
                "content": "demo云告警信息"
            },
            "template": "green"
        },
        "elements": [
            {
                "tag": "markdown",
                "content": "**resolved每日提醒: 今天该你值班了**\n**运维工具：**\n    运维平台：https://aa.com.cn\n    监控平台：https://aa.com.cn\n    火狐浏览器：https://aa.com.cn\n    镜像仓库：https://aa.com.cn\n    文件服务器：https://aa.com.cn\n**今日值班:** <at email=${duty_email}>${duty_name}></at>\n\n -------------------------------------\n   demo云告警信息"
            }
        ]
    }
}
EOF
}

# =====================================================
# 函数：发送告警到飞书
# =====================================================
send_to_feishu() {
    local json_data="$1"
    
    response=$(curl -s -X POST -H "Content-Type: application/json" \
         -d "$json_data" \
         $FEISHU_WEBHOOK)
    
    # 判断飞书返回的成功响应
    if [[ $response == *"\"code\":0"* ]] || [[ $response == *"\"Success\""* ]] || [[ $response == *"success"* ]] || [[ $response == *"\"msg\":\"success\""* ]]; then
        echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 飞书消息发送成功" >> $LOG_FILE
        return 0
    else
        echo "$(date '+%Y-%m-%d %H:%M:%S') [ERROR] 飞书发送失败: $response" >> $LOG_FILE
        return 1
    fi
}

# =====================================================
# 函数：处理活跃告警（新告警）
# =====================================================
process_active_alerts() {
    local duty_info="$1"
    local duty_email=$(echo $duty_info | cut -d'|' -f1)
    local duty_name=$(echo $duty_info | cut -d'|' -f2)
    
    # 获取所有告警
    alerts=$(get_all_alerts)
    
    if [ -n "$alerts" ]; then
        echo "$alerts" | while read -r alert; do
            # 获取告警信息
            alert_status=$(echo "$alert" | jq -r '.status.state')
            alert_fingerprint=$(echo "$alert" | jq -r '.fingerprint // ""')
            
            # 如果没有fingerprint，使用其他字段组合
            if [ -z "$alert_fingerprint" ] || [ "$alert_fingerprint" = "null" ]; then
                alertname=$(echo "$alert" | jq -r '.labels.alertname')
                instance=$(echo "$alert" | jq -r '.labels.instance')
                alert_fingerprint="${alertname}_${instance}"
            fi
            
            # 只处理活跃告警
            if [ "$alert_status" = "active" ]; then
                # 检查是否已经发送过
                if ! is_alert_sent "$alert_fingerprint"; then
                    # 构建飞书卡片
                    card_json=$(build_feishu_card "$alert" "$duty_email" "$duty_name" "active")
                    
                    # 发送告警
                    if send_to_feishu "$card_json"; then
                        # 标记为已发送（保存完整告警信息以便后续恢复使用）
                        alert_info=$(echo "$alert" | jq -c '{labels: .labels, annotations: .annotations, startsAt: .startsAt}')
                        mark_alert_sent "$alert_fingerprint" "$alert_info"
                        echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 已发送活跃告警 [${alert_fingerprint}] 给值班人员 $duty_name" >> $LOG_FILE
                    fi
                    
                    # 避免发送过快
                    sleep 1
                fi
            fi
        done
    fi
}

# =====================================================
# 函数：检查已恢复的告警（从API中消失的）
# =====================================================
check_resolved_alerts() {
    local duty_info="$1"
    local duty_email=$(echo $duty_info | cut -d'|' -f1)
    local duty_name=$(echo $duty_info | cut -d'|' -f2)
    
    # 获取当前所有活跃告警的指纹
    local current_fingerprints=$(get_current_fingerprints)
    
    # 将当前指纹转换为数组便于查找
    declare -A current_fp_map
    for fp in $current_fingerprints; do
        current_fp_map["$fp"]=1
    done
    
    # 检查缓存中的告警
    if [ -f "$ALERT_CACHE_FILE" ]; then
        while IFS= read -r line; do
            if [ -n "$line" ]; then
                cached_fingerprint=$(echo "$line" | cut -d'|' -f1)
                cached_info=$(echo "$line" | cut -d'|' -f2)
                
                # 如果缓存中的指纹不在当前活跃告警列表中（说明已从API消失）
                if [[ -z "${current_fp_map[$cached_fingerprint]}" ]]; then
                    echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 告警已从API消失，发送恢复通知: $cached_fingerprint" >> $LOG_FILE
                    
                    # 从缓存信息中恢复告警数据
                    if [ -n "$cached_info" ] && [ "$cached_info" != "null" ]; then
                        # 构建恢复卡片
                        card_json=$(build_feishu_card "$cached_info" "$duty_email" "$duty_name" "resolved")
                        
                        # 发送恢复通知
                        if send_to_feishu "$card_json"; then
                            # 从缓存中删除
                            remove_alert_from_cache "$cached_fingerprint"
                            echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 已发送恢复通知 [${cached_fingerprint}]" >> $LOG_FILE
                        fi
                    else
                        # 如果没有缓存信息，直接删除
                        remove_alert_from_cache "$cached_fingerprint"
                        echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 清理无效缓存 [${cached_fingerprint}]" >> $LOG_FILE
                    fi
                    
                    # 避免发送过快
                    sleep 1
                fi
            fi
        done < "$ALERT_CACHE_FILE"
    fi
}

# =====================================================
# 函数：处理所有告警（分开处理活跃和恢复）
# =====================================================
process_alerts() {
    local duty_info="$1"
    
    echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 开始处理告警..." >> $LOG_FILE
    
    # 第一步：处理活跃告警（新告警）
    process_active_alerts "$duty_info"
    
    # 第二步：检查已恢复的告警（从API消失的）
    check_resolved_alerts "$duty_info"
    
    echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 告警处理完成" >> $LOG_FILE
}

# =====================================================
# 函数：发送每日值班提醒
# =====================================================
send_duty_reminder() {
    local duty_info="$1"
    local duty_email=$(echo $duty_info | cut -d'|' -f1)
    local duty_name=$(echo $duty_info | cut -d'|' -f2)
    
    # 构建值班提醒卡片
    reminder_json=$(build_duty_reminder_card "$duty_email" "$duty_name")
    
    # 发送提醒
    if send_to_feishu "$reminder_json"; then
        echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 已发送值班提醒给 $duty_name" >> $LOG_FILE
    fi
}

# =====================================================
# 函数：发送每日总结
# =====================================================
send_duty_summary(){
    local duty_info="$1"
    local duty_email=$(echo $duty_info | cut -d'|' -f1)
    local duty_name=$(echo $duty_info | cut -d'|' -f2)
JSON_DATA=$(cat <<EOF
{
    "msg_type": "interactive",
    "card": {
        "config": {
            "wide_screen_mode": true
        },
        "header": {
            "title": {
                "tag": "plain_text",
                "content": "自主可控云告警信息"
            },
            "template": "green"
        },
        "elements": [
            {
                "tag": "markdown",
                "content": "**resolved每日提醒: 下班也不要放松警惕呦**\n请总结今日值班情况\n**今日值班:** <at email=${duty_email}>${duty_name}</at>\n\n -------------------------------------\n   自主可控云告警信息"
            }
        ]
    }
}
EOF
)

# 发送到飞书
response=$(curl -s -X POST -H "Content-Type: application/json" \
     -d "$JSON_DATA" \
     $FEISHU_WEBHOOK)

# 判断飞书返回的成功响应
    if [[ $response == *"\"code\":0"* ]] || [[ $response == *"\"Success\""* ]] || [[ $response == *"success"* ]] || [[ $response == *"\"msg\":\"success\""* ]]; then
        echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 飞书消息发送成功" >> $LOG_FILE

        return 0
    else
        echo "$(date '+%Y-%m-%d %H:%M:%S') [ERROR] 飞书发送失败: $response" >> $LOG_FILE
        return 1
    fi
}

# =====================================================
# 函数：清理过期缓存（保留24小时）
# =====================================================
cleanup_cache() {
    local current_time=$(date +%s)
    local cache_expire=86400  # 24小时
    
    if [ -f "$ALERT_CACHE_FILE" ]; then
        while IFS= read -r line; do
            if [ -n "$line" ]; then
                cache_time=$(echo "$line" | cut -d'|' -f3)
                if [ -n "$cache_time" ] && [ $((current_time - cache_time)) -gt $cache_expire ]; then
                    fingerprint=$(echo "$line" | cut -d'|' -f1)
                    sed -i "/^${fingerprint}|/d" "$ALERT_CACHE_FILE"
                    echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 清理过期缓存: $fingerprint" >> $LOG_FILE
                fi
            fi
        done < "$ALERT_CACHE_FILE"
    fi
}

# =====================================================
# 函数：调试模式 - 查看缓存和当前告警
# =====================================================
debug_mode() {
    echo "=== 当前缓存内容 ==="
    if [ -f "$ALERT_CACHE_FILE" ]; then
        cat "$ALERT_CACHE_FILE"
    else
        echo "缓存文件不存在"
    fi
    echo ""
    
    echo "=== 当前API中的告警 ==="
    curl -s "$ALERTMANAGER_API" | jq -r '.[] | "指纹: \(.fingerprint // "无") 状态: \(.status.state)"' 2>/dev/null || echo "无法获取告警"
    echo ""
}

# =====================================================
# 主函数
# =====================================================
main() {
    # 初始化缓存
    init_cache
    
    # 清理过期缓存
    cleanup_cache
    
    # 获取今日值班信息
    duty_info=$(get_today_duty)
    
    # 检查命令行参数
    case "${1:-}" in
        "reminder")
            # 只发送值班提醒
            send_duty_reminder "$duty_info"
            ;;
        "summary")
            # 每日下班提醒
            send_duty_summary "$duty_info"
            ;;
        "alerts")
            # 只处理告警
            process_alerts "$duty_info"
            ;;
        "cleanup")
            # 只清理缓存
            cleanup_cache
            ;;
        "debug")
            # 调试模式
            debug_mode
            ;;
        *)
            # 默认：处理告警
            process_alerts "$duty_info"
            ;;
    esac
}

# 执行主函数
main "$@"
EOF

安装依赖和配置

# 安装jq（用于解析JSON）
yum install -y jq  # CentOS/RHEL
# 或
apt-get install -y jq  # Ubuntu/Debian

# 创建日志文件
touch /var/log/alert_duty_rotator.log
chmod 644 /var/log/alert_duty_rotator.log

# 添加执行权限
chmod +x /usr/local/bin/alert_duty_rotator.sh

配置Alertmanager Webhook

global:
  resolve_timeout: 30s

route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1h
  # 将所有告警发送到 webhook 接收器
  receiver: 'prometheus-alert'

receivers:
- name: 'prometheus-alert'
  webhook_configs:
  - url: 'http://192.168.1.20:8085/prometheusalert?type=email&tpl=prometheus-email&email=zhangsan@163.com'  # Prometheus-Alert 的接收地址
  - url: 'http://192.168.1.20:8086/alert'  # 需要创建一个简单的webhook接收服务
    send_resolved: true  # 发送恢复通知

创建简单的Webhook接收服务

由于Shell脚本不能直接作为Webhook服务运行，需要创建一个简单的HTTP服务。以下是使用Python Flask的示例 /opt/webhook/alert_webhook.py

#!/usr/bin/env python3
from flask import Flask, request, jsonify
import subprocess
import json
import os

app = Flask(__name__)

# 告警脚本路径
ALERT_SCRIPT = "/usr/local/bin/alert_duty_rotator.sh"

@app.route('/alert', methods=['POST'])
def handle_alert():
    """接收Alertmanager的告警并触发告警处理"""
    try:
        # 可以选择将告警数据保存到文件，供脚本处理
        alert_data = request.json
        
        # 保存告警数据到临时文件
        with open('/tmp/latest_alert.json', 'w') as f:
            json.dump(alert_data, f)
        
        # 触发告警处理脚本（后台运行）
        subprocess.Popen([ALERT_SCRIPT, "alerts"])
        
        return jsonify({"status": "success"}), 200
    except Exception as e:
        return jsonify({"status": "error", "message": str(e)}), 500

@app.route('/health', methods=['GET'])
def health():
    return jsonify({"status": "healthy"}), 200

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080, debug=False)

安装并启动Webhook服务

pip3 install flask

cat > /usr/lib/systemd/system/webhook.service << EOF
[Unit]
Description=webhook service
After=network.target syslog.target
Wants=network.target

[Service]
Type=simple
#写你的frps的需要用绝对路径
ExecStartPre=/bin/sleep 1
ExecStart=/usr/bin/python3 /opt/webhook/alert_webhook.py
#logs路径
StandardOutput=null

[Install]
WantedBy=multi-user.target
EOF

systemctl damon-reload
systemctl start webhook
systemctl enable webhook
systemctl status webhook

配置定时任务

# 编辑crontab
crontab -e

# 添加以下内容
# 每天上午9点发送值班提醒
0 9 * * * /usr/local/bin/alert_duty_rotator.sh reminder >> /var/log/alert_duty_cron.log 2>&1

# 每天下午发送总结提醒
0 17 * * * /usr/local/bin/alert_duty_rotator.sh summary >> /var/log/alert_duty_cron.log 2>&1

整体流程

prometheus收集数据->发送给alertmanager->py程序实时获取alertmanager告警数据->脚本实时获取发送获取的告警信息并发送飞书@值班人员