typecho显示大纲目录插件 {anote icon="posttoc-download" href="https://package.prd.emal.cc/package/softinstall/produce/typecho/PostToc.tar.gz" type="success" content="点击下载插件"/}下载插件后解压安装插件下载上面置顶得插件，然后解压到 typecho/usr/plugins/ 目录中tar -zcvf PostToc.tar.gz -C typecho/usr/plugins/ chmod -R 755 typecho/usr/plugins/PostToc然后去到typecho后台启用插件

企业微信告警配置 创建企业微信机器人1、企业微信创建群聊2、创建机器人，记住webhook地址创建prometheus-alert# 企业微信卡片 { "msgtype": "news", "news": { "articles": [ {{ range .alerts }} { "title": "{{ if eq .status "resolved" }}✅【已恢复】{{ .labels.alertname }}{{ else }}🚨【告警通知】{{ .labels.alertname }}{{ end }}", "description": "级别: {{ .labels.severity }}\n主机: {{ .labels.instance }}\n{{ if eq .status "resolved" }}开始: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}\n结束: {{ slice .endsAt 0 10 }} {{ slice .endsAt 11 19 }}{{ else }}时间: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}\n当前值: {{ .annotations.value }}{{ end }}\n详情: {{ .annotations.description }}", "url": "https://alert.prd.emal.cc/", "picurl": "https://package.prd.emal.cc/package/img/prometheus-alter.png" } {{ end }} ] } }# 企业微信文本格式 { "msgtype": "markdown", "markdown": { "content": "{{ range .alerts }}{{ if eq .status "resolved" }}**✅【已恢复】** **{{ .labels.alertname }}**\n\n**级别**: **{{ .labels.severity }}**\n**主机**: **{{ .labels.instance }}**\n**开始**: **{{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}**\n**结束**: **{{ slice .endsAt 0 10 }} {{ slice .endsAt 11 19 }}**\n**详情**: **{{ .annotations.description }}**\n{{ else }}**🚨【告警通知】** **{{ .labels.alertname }}**\n\n**级别**: **{{ .labels.severity }}**\n**主机**: **{{ .labels.instance }}**\n**时间**: **{{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}**\n**当前值**: **{{ .annotations.value }}**\n**详情**: **{{ .annotations.description }}**\n{{ end }}{{ end }}" } }alertmanager.yml配置docker-compose restart alertmanager

飞书告警值班轮流@人员配置 创建值班轮询脚本cat > /usr/local/bin/alert_duty_rotator.sh << EOF #!/usr/bin/env bash # ===================================================== # 告警值班轮询脚本 - 每天自动@当日值班人员 # 功能：从Alertmanager获取告警，发送到飞书并@值班人 # ===================================================== # 飞书Webhook地址 FEISHU_WEBHOOK="https://aa.com.cm/open-apis/bot/v2/hook/a6eb1f09-cb69-45cf-80bd-dc880e99c2a8" # 值班人员列表（邮箱地址，按顺序轮询） DUTY_USERS=( "zhangsan@cnpc.com.cn" ) # 值班人员对应的飞书用户ID（如果有的话） # 如果飞书机器人支持通过邮箱@，则不需要这个映射 # DUTY_USER_IDS=( # "zhangshan_id" # "lisi_id" # "wanger_id" # ) # Alertmanager API地址 ALERTMANAGER_API="http://192.168.4.20:9093/api/v2/alerts" # 日志文件 LOG_FILE="/var/log/alert_duty_rotator.log" # 已发送告警缓存文件（用于跟踪告警状态变化） ALERT_CACHE_FILE="/tmp/alert_sent_cache.txt" # ===================================================== # 函数：初始化缓存文件 # ===================================================== init_cache() { if [ ! -f "$ALERT_CACHE_FILE" ]; then touch "$ALERT_CACHE_FILE" fi } # ===================================================== # 函数：获取今日值班人员 # ===================================================== get_today_duty() { # 获取今天是今年的第几天（1-365/366） DAY_OF_YEAR=$(date +%j | sed 's/^0*//') # 计算索引（从0开始） INDEX=$(( (DAY_OF_YEAR - 1) % ${#DUTY_USERS[@]} )) # 返回今天值班人员的邮箱和索引 TODAY_EMAIL="${DUTY_USERS[$INDEX]}" TODAY_NAME=$(echo $TODAY_EMAIL | cut -d'@' -f1) TODAY_INDEX=$INDEX echo "$TODAY_EMAIL|$TODAY_NAME|$TODAY_INDEX" } # ===================================================== # 函数：从Alertmanager获取所有告警 # ===================================================== get_all_alerts() { # 调用Alertmanager API获取所有告警 curl -s "$ALERTMANAGER_API" | jq -c '.[]' } # ===================================================== # 函数：获取当前所有活跃告警的指纹列表 # ===================================================== get_current_fingerprints() { local alerts=$(get_all_alerts) local fingerprints="" if [ -n "$alerts" ]; then fingerprints=$(echo "$alerts" | jq -r '.fingerprint // ""' | grep -v "^$" | tr '\n' ' ') fi echo "$fingerprints" } # ===================================================== # 函数：检查告警是否已发送过 # ===================================================== is_alert_sent() { local alert_fingerprint="$1" if grep -q "^${alert_fingerprint}|" "$ALERT_CACHE_FILE"; then return 0 # 已发送过 fi return 1 # 未发送过 } # ===================================================== # 函数：标记告警为已发送 # ===================================================== mark_alert_sent() { local alert_fingerprint="$1" local alert_info="$2" # 先删除已有的记录 sed -i "/^${alert_fingerprint}|/d" "$ALERT_CACHE_FILE" # 添加新记录 echo "${alert_fingerprint}|${alert_info}|$(date +%s)" >> "$ALERT_CACHE_FILE" } # ===================================================== # 函数：从缓存中删除告警 # ===================================================== remove_alert_from_cache() { local alert_fingerprint="$1" sed -i "/^${alert_fingerprint}|/d" "$ALERT_CACHE_FILE" } # ===================================================== # 函数：构建飞书告警卡片（支持活跃和恢复） # ===================================================== build_feishu_card() { local alert_json="$1" local duty_email="$2" local duty_name="$3" local alert_type="$4" # "active" 或 "resolved" # 解析告警信息 local alertname=$(echo "$alert_json" | jq -r '.labels.alertname // "未知"') local severity=$(echo "$alert_json" | jq -r '.labels.severity // "未知"') local instance=$(echo "$alert_json" | jq -r '.labels.instance // "未知"') local starts_at=$(echo "$alert_json" | jq -r '.startsAt // "未知"') local ends_at=$(echo "$alert_json" | jq -r '.endsAt // "未知"') local description=$(echo "$alert_json" | jq -r '.annotations.description // "无"') local value=$(echo "$alert_json" | jq -r '.annotations.value // "无"') # 格式化时间（去掉T和Z，更易读） if [ "$starts_at" != "0001-01-01T00:00:00Z" ] && [ "$starts_at" != "未知" ]; then start_time=$(echo "$starts_at" | sed 's/T/ /' | sed 's/Z//' | cut -d'.' -f1) else start_time=$(date '+%Y-%m-%d %H:%M:%S') fi # 判断告警类型并构建相应的卡片 if [ "$alert_type" = "resolved" ]; then # 恢复告警 - 使用绿色模板 if [ "$ends_at" != "0001-01-01T00:00:00Z" ] && [ "$ends_at" != "未知" ]; then end_time=$(echo "$ends_at" | sed 's/T/ /' | sed 's/Z//' | cut -d'.' -f1) else end_time=$(date '+%Y-%m-%d %H:%M:%S') fi # 构建恢复卡片 cat <<EOF { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "✅ 【已恢复】 ${alertname}" }, "template": "green" }, "elements": [ { "tag": "markdown", "content": "**主题**: ${alertname}\n**级别**: ${severity}\n**主机**: ${instance}\n**开始**: ${start_time}\n**结束**: ${end_time}\n**详情**: ${description}\n<at email=${duty_email}>${duty_name}</at>" } ] } } EOF else # 活跃告警 - 根据级别设置颜色 case "$severity" in "critical"|"紧急") template="red" emoji="🚨" ;; "warning"|"警告") template="red" emoji="⚠️" ;; *) template="red" emoji="🔔" ;; esac # 构建活跃告警卡片 cat <<EOF { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "${emoji} 【告警中】 ${alertname}" }, "template": "${template}" }, "elements": [ { "tag": "markdown", "content": "**主题**: ${alertname}\n**级别**: ${severity}\n**主机**: ${instance}\n**时间**: ${start_time}\n**当前值**: ${value}\n**详情**: ${description}\n<at email=${duty_email}>${duty_name}</at>" } ] } } EOF fi } # ===================================================== # 函数：构建值班提醒卡片（每天定时发送） # ===================================================== build_duty_reminder_card() { local duty_email="$1" local duty_name="$2" cat <<EOF { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "demo云告警信息" }, "template": "green" }, "elements": [ { "tag": "markdown", "content": "**resolved每日提醒: 今天该你值班了**\n**运维工具：**\n 运维平台：https://aa.com.cn\n 监控平台：https://aa.com.cn\n 火狐浏览器：https://aa.com.cn\n 镜像仓库：https://aa.com.cn\n 文件服务器：https://aa.com.cn\n**今日值班:** <at email=${duty_email}>${duty_name}></at>\n\n -------------------------------------\n demo云告警信息" } ] } } EOF } # ===================================================== # 函数：发送告警到飞书 # ===================================================== send_to_feishu() { local json_data="$1" response=$(curl -s -X POST -H "Content-Type: application/json" \ -d "$json_data" \ $FEISHU_WEBHOOK) # 判断飞书返回的成功响应 if [[ $response == *"\"code\":0"* ]] || [[ $response == *"\"Success\""* ]] || [[ $response == *"success"* ]] || [[ $response == *"\"msg\":\"success\""* ]]; then echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 飞书消息发送成功" >> $LOG_FILE return 0 else echo "$(date '+%Y-%m-%d %H:%M:%S') [ERROR] 飞书发送失败: $response" >> $LOG_FILE return 1 fi } # ===================================================== # 函数：处理活跃告警（新告警） # ===================================================== process_active_alerts() { local duty_info="$1" local duty_email=$(echo $duty_info | cut -d'|' -f1) local duty_name=$(echo $duty_info | cut -d'|' -f2) # 获取所有告警 alerts=$(get_all_alerts) if [ -n "$alerts" ]; then echo "$alerts" | while read -r alert; do # 获取告警信息 alert_status=$(echo "$alert" | jq -r '.status.state') alert_fingerprint=$(echo "$alert" | jq -r '.fingerprint // ""') # 如果没有fingerprint，使用其他字段组合 if [ -z "$alert_fingerprint" ] || [ "$alert_fingerprint" = "null" ]; then alertname=$(echo "$alert" | jq -r '.labels.alertname') instance=$(echo "$alert" | jq -r '.labels.instance') alert_fingerprint="${alertname}_${instance}" fi # 只处理活跃告警 if [ "$alert_status" = "active" ]; then # 检查是否已经发送过 if ! is_alert_sent "$alert_fingerprint"; then # 构建飞书卡片 card_json=$(build_feishu_card "$alert" "$duty_email" "$duty_name" "active") # 发送告警 if send_to_feishu "$card_json"; then # 标记为已发送（保存完整告警信息以便后续恢复使用） alert_info=$(echo "$alert" | jq -c '{labels: .labels, annotations: .annotations, startsAt: .startsAt}') mark_alert_sent "$alert_fingerprint" "$alert_info" echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 已发送活跃告警 [${alert_fingerprint}] 给值班人员 $duty_name" >> $LOG_FILE fi # 避免发送过快 sleep 1 fi fi done fi } # ===================================================== # 函数：检查已恢复的告警（从API中消失的） # ===================================================== check_resolved_alerts() { local duty_info="$1" local duty_email=$(echo $duty_info | cut -d'|' -f1) local duty_name=$(echo $duty_info | cut -d'|' -f2) # 获取当前所有活跃告警的指纹 local current_fingerprints=$(get_current_fingerprints) # 将当前指纹转换为数组便于查找 declare -A current_fp_map for fp in $current_fingerprints; do current_fp_map["$fp"]=1 done # 检查缓存中的告警 if [ -f "$ALERT_CACHE_FILE" ]; then while IFS= read -r line; do if [ -n "$line" ]; then cached_fingerprint=$(echo "$line" | cut -d'|' -f1) cached_info=$(echo "$line" | cut -d'|' -f2) # 如果缓存中的指纹不在当前活跃告警列表中（说明已从API消失） if [[ -z "${current_fp_map[$cached_fingerprint]}" ]]; then echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 告警已从API消失，发送恢复通知: $cached_fingerprint" >> $LOG_FILE # 从缓存信息中恢复告警数据 if [ -n "$cached_info" ] && [ "$cached_info" != "null" ]; then # 构建恢复卡片 card_json=$(build_feishu_card "$cached_info" "$duty_email" "$duty_name" "resolved") # 发送恢复通知 if send_to_feishu "$card_json"; then # 从缓存中删除 remove_alert_from_cache "$cached_fingerprint" echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 已发送恢复通知 [${cached_fingerprint}]" >> $LOG_FILE fi else # 如果没有缓存信息，直接删除 remove_alert_from_cache "$cached_fingerprint" echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 清理无效缓存 [${cached_fingerprint}]" >> $LOG_FILE fi # 避免发送过快 sleep 1 fi fi done < "$ALERT_CACHE_FILE" fi } # ===================================================== # 函数：处理所有告警（分开处理活跃和恢复） # ===================================================== process_alerts() { local duty_info="$1" echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 开始处理告警..." >> $LOG_FILE # 第一步：处理活跃告警（新告警） process_active_alerts "$duty_info" # 第二步：检查已恢复的告警（从API消失的） check_resolved_alerts "$duty_info" echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 告警处理完成" >> $LOG_FILE } # ===================================================== # 函数：发送每日值班提醒 # ===================================================== send_duty_reminder() { local duty_info="$1" local duty_email=$(echo $duty_info | cut -d'|' -f1) local duty_name=$(echo $duty_info | cut -d'|' -f2) # 构建值班提醒卡片 reminder_json=$(build_duty_reminder_card "$duty_email" "$duty_name") # 发送提醒 if send_to_feishu "$reminder_json"; then echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 已发送值班提醒给 $duty_name" >> $LOG_FILE fi } # ===================================================== # 函数：发送每日总结 # ===================================================== send_duty_summary(){ local duty_info="$1" local duty_email=$(echo $duty_info | cut -d'|' -f1) local duty_name=$(echo $duty_info | cut -d'|' -f2) JSON_DATA=$(cat <<EOF { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "自主可控云告警信息" }, "template": "green" }, "elements": [ { "tag": "markdown", "content": "**resolved每日提醒: 下班也不要放松警惕呦**\n请总结今日值班情况\n**今日值班:** <at email=${duty_email}>${duty_name}</at>\n\n -------------------------------------\n 自主可控云告警信息" } ] } } EOF ) # 发送到飞书 response=$(curl -s -X POST -H "Content-Type: application/json" \ -d "$JSON_DATA" \ $FEISHU_WEBHOOK) # 判断飞书返回的成功响应 if [[ $response == *"\"code\":0"* ]] || [[ $response == *"\"Success\""* ]] || [[ $response == *"success"* ]] || [[ $response == *"\"msg\":\"success\""* ]]; then echo "$(date '+%Y-%m-%d %H:%M:%S') [SUCCESS] 飞书消息发送成功" >> $LOG_FILE return 0 else echo "$(date '+%Y-%m-%d %H:%M:%S') [ERROR] 飞书发送失败: $response" >> $LOG_FILE return 1 fi } # ===================================================== # 函数：清理过期缓存（保留24小时） # ===================================================== cleanup_cache() { local current_time=$(date +%s) local cache_expire=86400 # 24小时 if [ -f "$ALERT_CACHE_FILE" ]; then while IFS= read -r line; do if [ -n "$line" ]; then cache_time=$(echo "$line" | cut -d'|' -f3) if [ -n "$cache_time" ] && [ $((current_time - cache_time)) -gt $cache_expire ]; then fingerprint=$(echo "$line" | cut -d'|' -f1) sed -i "/^${fingerprint}|/d" "$ALERT_CACHE_FILE" echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] 清理过期缓存: $fingerprint" >> $LOG_FILE fi fi done < "$ALERT_CACHE_FILE" fi } # ===================================================== # 函数：调试模式 - 查看缓存和当前告警 # ===================================================== debug_mode() { echo "=== 当前缓存内容 ===" if [ -f "$ALERT_CACHE_FILE" ]; then cat "$ALERT_CACHE_FILE" else echo "缓存文件不存在" fi echo "" echo "=== 当前API中的告警 ===" curl -s "$ALERTMANAGER_API" | jq -r '.[] | "指纹: \(.fingerprint // "无") 状态: \(.status.state)"' 2>/dev/null || echo "无法获取告警" echo "" } # ===================================================== # 主函数 # ===================================================== main() { # 初始化缓存 init_cache # 清理过期缓存 cleanup_cache # 获取今日值班信息 duty_info=$(get_today_duty) # 检查命令行参数 case "${1:-}" in "reminder") # 只发送值班提醒 send_duty_reminder "$duty_info" ;; "summary") # 每日下班提醒 send_duty_summary "$duty_info" ;; "alerts") # 只处理告警 process_alerts "$duty_info" ;; "cleanup") # 只清理缓存 cleanup_cache ;; "debug") # 调试模式 debug_mode ;; *) # 默认：处理告警 process_alerts "$duty_info" ;; esac } # 执行主函数 main "$@" EOF安装依赖和配置# 安装jq（用于解析JSON） yum install -y jq # CentOS/RHEL # 或 apt-get install -y jq # Ubuntu/Debian # 创建日志文件 touch /var/log/alert_duty_rotator.log chmod 644 /var/log/alert_duty_rotator.log # 添加执行权限 chmod +x /usr/local/bin/alert_duty_rotator.sh配置Alertmanager Webhookglobal: resolve_timeout: 30s route: group_by: ['alertname'] group_wait: 10s group_interval: 10s repeat_interval: 1h # 将所有告警发送到 webhook 接收器 receiver: 'prometheus-alert' receivers: - name: 'prometheus-alert' webhook_configs: - url: 'http://192.168.1.20:8085/prometheusalert?type=email&tpl=prometheus-email&email=zhangsan@163.com' # Prometheus-Alert 的接收地址 - url: 'http://192.168.1.20:8086/alert' # 需要创建一个简单的webhook接收服务 send_resolved: true # 发送恢复通知创建简单的Webhook接收服务由于Shell脚本不能直接作为Webhook服务运行，需要创建一个简单的HTTP服务。以下是使用Python Flask的示例 /opt/webhook/alert_webhook.py#!/usr/bin/env python3 from flask import Flask, request, jsonify import subprocess import json import os app = Flask(__name__) # 告警脚本路径 ALERT_SCRIPT = "/usr/local/bin/alert_duty_rotator.sh" @app.route('/alert', methods=['POST']) def handle_alert(): """接收Alertmanager的告警并触发告警处理""" try: # 可以选择将告警数据保存到文件，供脚本处理 alert_data = request.json # 保存告警数据到临时文件 with open('/tmp/latest_alert.json', 'w') as f: json.dump(alert_data, f) # 触发告警处理脚本（后台运行） subprocess.Popen([ALERT_SCRIPT, "alerts"]) return jsonify({"status": "success"}), 200 except Exception as e: return jsonify({"status": "error", "message": str(e)}), 500 @app.route('/health', methods=['GET']) def health(): return jsonify({"status": "healthy"}), 200 if __name__ == '__main__': app.run(host='0.0.0.0', port=8080, debug=False)安装并启动Webhook服务pip3 install flask cat > /usr/lib/systemd/system/webhook.service << EOF [Unit] Description=webhook service After=network.target syslog.target Wants=network.target [Service] Type=simple #写你的frps的需要用绝对路径 ExecStartPre=/bin/sleep 1 ExecStart=/usr/bin/python3 /opt/webhook/alert_webhook.py #logs路径 StandardOutput=null [Install] WantedBy=multi-user.target EOF systemctl damon-reload systemctl start webhook systemctl enable webhook systemctl status webhook配置定时任务# 编辑crontab crontab -e # 添加以下内容 # 每天上午9点发送值班提醒 0 9 * * * /usr/local/bin/alert_duty_rotator.sh reminder >> /var/log/alert_duty_cron.log 2>&1 # 每天下午发送总结提醒 0 17 * * * /usr/local/bin/alert_duty_rotator.sh summary >> /var/log/alert_duty_cron.log 2>&1整体流程prometheus收集数据->发送给alertmanager->py程序实时获取alertmanager告警数据->脚本实时获取发送获取的告警信息并发送飞书@值班人员

prometheus-alert告警部署 部署prometheus# docker-compose.yaml version: '2.4' networks: monitor: driver: bridge services: prometheus: image: prom/prometheus container_name: prometheus hostname: prometheus restart: always command: - '--config.file=/etc/prometheus/prometheus.yml' - '--web.enable-admin-api' - '--storage.tsdb.retention.time=7d' volumes: - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - ./prometheus/rules.yaml:/etc/prometheus/rules/rules.yaml - /etc/localtime:/etc/localtime ports: - "9090:9090" networks: - monitor alertmanager: image: prom/alertmanager container_name: alertmanager hostname: alertmanager restart: always volumes: - /etc/localtime:/etc/localtime - ./alertmanager/emal.tmpl:/etc/alertmanager/email.tmpl - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml ports: - "9093:9093" networks: - monitor grafana: image: grafana/grafana container_name: grafana hostname: grafana restart: always volumes: - /etc/localtime:/etc/localtime - ./grafana/data:/var/lib/grafana ports: - "3000:3000" networks: - monitor创建目录mkdir -p alertmanager grafana prometheus chmod -R 755 alertmanager grafana prometheus创建文件prometheuscat > prometheus/prometheus.yml << EOF global: scrape_interval: 20s # 调整为15秒，减轻负载 evaluation_interval: 15s #scrape_timeout: 240s # 配置Alertmanager地址 alerting: alertmanagers: - static_configs: - targets: - 192.168.4.20:9093 # 保持你的IP:端口配置 rule_files: - "/etc/prometheus/rules/rules.yaml" scrape_configs: - job_name: 'prometheus' static_configs: - targets: ['192.168.4.20:9090'] labels: instance: 'prometheus-server' # - job_name: 'cadvisor' # static_configs: # - targets: ['192.168.4.20:8899'] # 这里改回映射的宿主机端口8899（之前的8080是容器内端口） # labels: # instance: 'cadvisor-monitor' - job_name: 'cloud' static_configs: - targets: ['192.168.4.20:9100'] labels: instance: '192.168.4.20' - targets: ['192.168.4.21:9100'] labels: instance: '192.168.4.21' - targets: ['192.168.4.22:9100'] labels: instance: '192.168.4.22' EOFcat > rules.yaml << EOF groups: - name: server-alarm rules: - alert: "内存告警" expr: (1 - (node_memory_MemAvailable_bytes / (node_memory_MemTotal_bytes))) * 100 > 90 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高内存 使用率！" description: "{{$labels.instance}}: 内存使用率在 90% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "CPU告警" expr: (1 - avg(irate(node_cpu_seconds_total{mode="idle"}[2m])) by(instance)) * 100 > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高CPU 使用率！" description: "{{$labels.instance}}: CPU使用率在 80% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "系统盘告警" expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} * 100) > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！" description: "{{$labels.instance}}: / 磁盘使用率在 80% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "数据盘告警" expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/data"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/data"} * 100) > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！" description: "{{$labels.instance}}: /data 磁盘使用率在 80% 以上" value: "{{ printf \"%.0f\" $value }}%" - alert: "磁盘使用率过高" expr: (1 - node_filesystem_free_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"}) * 100 > 80 for: 30s labels: severity: warning annotations: summary: "{{$labels.instance}}: 磁盘使用率过高" description: "{{$labels.instance}} 挂载点 {{$labels.mountpoint}} 使用率超过80%" value: "{{ printf \"%.0f\" $value }}%" - alert: "typecho备份数据异常" expr: typecho_backup_total != 4 for: 30s labels: severity: warning annotations: summary: "typecho备份数据异常" description: "{{ $labels.instance }} typecho当前备份总数不等于预期值4" value: "{{ $value }}" # - alert: "测试" # expr: up == 1 # for: 30s # labels: # severity: warning # annotations: # summary: "{{ $labels.instance }}: 实例在线" # description: "{{ $labels.instance }} of job {{ $labels.job }} 已在线" # value: "up" EOF创建文件alertmanager# 邮件告警 cat > alertmanager.yml << EOF global: resolve_timeout: 30s route: group_by: ['alertname'] group_wait: 10s group_interval: 10s repeat_interval: 1h # 将所有告警发送到 webhook 接收器 receiver: 'prometheus-alert' receivers: - name: 'prometheus-alert' webhook_configs: - url: 'http://192.168.4.20:8085/prometheusalert?type=email&tpl=prometheus-email&email=kexinzz@163.com' # Prometheus-Alert 的接收地址 send_resolved: true # 发送恢复通知 EOF # 飞书告警 cat > alertmanager.yml << EOf global: resolve_timeout: 1m route: group_by: ['env','instance','type','group','job','alertname'] # 根据告警名和实例分组 group_wait: 30s # 组告警等待时间 group_interval: 2m # 组告警间隔 repeat_interval: 4h # 重复告警间隔 receiver: 'prometheusalert-feishu' # 默认接收器 receivers: - name: 'prometheusalert-feishu' webhook_configs: - url: 'http://192.168.4.20:8085/prometheusalert?type=fs&tpl=prometheus-fs&fsurl=https://kexinzz.cn.com/open-apis/bot/v2/hook/4b874766-75f0-4d45-9d7a-dced826ead54' send_resolved: true # 添加 HTTP 配置以确保正确发送 http_config: follow_redirects: true EOF部署prometheus-alert# docker-compose.yaml cat > docker-compose.yaml << EOF services: prometheus-alert: image: feiyu563/prometheus-alert:v4.9.2 container_name: prometheus-alert restart: unless-stopped ports: - "8085:8080" # 宿主机的8085映射到容器的8080 environment: PA_LOGIN_USER: admin PA_LOGIN_PASSWORD: Xzm7_GTD.com PA_TITLE: PrometheusAlert PA_OPEN_FEISHU: "1" PA_FSURL: "https://kexinzz.cn.com/open-apis/bot/v2/hook/4b874766-75f0-4d45-9d7a-dced826ead54" PA_OPEN_DINGDING: "0" PA_OPEN_WEIXIN: "0" volumes: # 【关键修改】使用宿主机目录挂载 - ./data/:/app/data/ # 挂载数据目录（告警记录、日志等） - ./conf/:/app/conf/ # 挂载配置文件目录 - ./logs/:/app/logs/ # 挂载日志目录 - ./db:/app/db EOF创建文件mkdir -p data logs conf db chmod -R 755 data logs conf db ##### 添加配置文件 conf/app.conf wget https://package.prd.emal.cc/package/softinstall/doc/doc/app.conf更换相对应的配置运行docker-compose# 运行prometheus、prometheus-alert yaml docker-compose up -d登录地址# prometheus-alert http://127.0.0.1:8085 # alertmanager http://127.0.0.1:9093 # prometheus http://127.0.0.1:9090 # grafana http://127.0.0.1:3000prometheus-alert登录prometheus-alert 邮件配置# 邮件 {{ range .alerts }} {{ if eq .status "resolved" }} <h2>@告警恢复</h2> <b>主题:</b> {{ .labels.alertname }} <br> <b>级别:</b> {{ .labels.severity }} <br> <b>主机:</b> {{ .labels.instance }} <br> <b>开始</b>: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }} <br> <b>结束:</b> {{ slice .endsAt 0 10 }} {{ slice .endsAt 11 19 }} <br> <b>详情</b>: {{ .annotations.description }} <br> {{ else }} <h2>@告警通知</h2> <b>主题:</b> {{ .labels.alertname }} <br> <b>级别:</b> {{ .labels.severity }} <br> <b>主机:</b> {{ .labels.instance }} <br> <b>时间:</b> {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }} <br> <b>当前值:</b> {{ .annotations.value }} <br> <b>详情:</b> {{ .annotations.description }} <br> {{ end }} {{ end }}prometheus-alert 飞书配置# 飞书 {{ range .alerts }} {{ if eq .status "resolved" }} { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "测试告警" }, "template": "green" }, "elements": [ { "tag": "markdown", "content": "**主题** {{ .labels.alertname }}\n**级别**: {{ .labels.severity }}\n**主机**: {{ .labels.instance }}\n**开始**: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}\n**结束**: {{ slice .endsAt 0 10 }} {{ slice .endsAt 11 19 }}\n**详情**: {{ .annotations.description }}" } ] } } {{ else }} { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "测试告警" }, "template": "red" }, "elements": [ { "tag": "markdown", "content": "**主题** {{ .labels.alertname }}\n**级别**: {{ .labels.severity }}\n**主机**: {{ .labels.instance }}\n**时间**: {{ slice .startsAt 0 10 }} {{ slice .startsAt 11 19 }}\n**当前值**: {{ .annotations.value }}\n**详情**: {{ .annotations.description }}" } ] } } {{ end }} {{ end }}每日提醒值班#!/usr/bin/env bash # 飞书Webhook地址 WEBHOOK_URL="https://test.com.cc/open-apis/bot/v2/hook/a6eb1f09-cb69-45cf-80bd-dc880e99c2a8" # 值班人员列表（邮箱地址） USERS=( "zhangshan@cnpc.com.cn" "lisi@cnpc.com.cn" ) # 获取今天是今年的第几天（1-365/366） DAY_OF_YEAR=$(date +%j | sed 's/^0*//') # 计算索引（从0开始） INDEX=$(( (DAY_OF_YEAR - 1) % ${#USERS[@]} )) # 获取今天值班人员的邮箱 TODAY_USER=${USERS[$INDEX]} # 获取值班人员姓名（从邮箱提取，可根据需要修改） TODAY_NAME=$(echo $TODAY_USER | cut -d'@' -f1) # 构建JSON数据 JSON_DATA=$(cat <<EOF { "msg_type": "interactive", "card": { "config": { "wide_screen_mode": true }, "header": { "title": { "tag": "plain_text", "content": "demo云告警信息" }, "template": "green" }, "elements": [ { "tag": "markdown", "content": "**resolved每日提醒: 今天该你值班了**\n**运维工具：**\n 运维平台：https://aa.com.cn\n 监控平台：https://aa.com.cn\n 火狐浏览器：https://aa.com.cn\n 镜像仓库：https://aa.com.cn\n 文件服务器：https://aa.com.cn\n**今日值班:** <at email=${TODAY_USER}></at>\n\n -------------------------------------\n demo云告警信息" } ] } } EOF ) # 发送到飞书 curl -X POST -H "Content-Type: application/json" \ -d "$JSON_DATA" \ $WEBHOOK_URL # 输出日志 echo "$(date): 已发送值班提醒给 $TODAY_NAME ($TODAY_USER)"

rancher rke部署k8s 一、rke搭建(1)先安装关闭防火墙selinux]# sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config ]# systemctl stop firewalld && systemctl disable firewalld(2)关闭swap交换分区]# swapoff -a //如果swap在/etc/fstab里面挂载了则需要"#"掉他。(3)查看主机是否是打开ipv4路由转发，没有则加上]# tee /etc/sysctl.d/rke.conf <<EOF net.ipv4.ip_forward=1 EOF ]# sysctl -p /etc/sysctl.d/rke.conf net.ipv4.ip_forward=1(4)创建普通用户，必须用普通用户来创建集群]# useradd -s /usr/bin/bash ops //更改普通用户拥有权限 ]# echo "ops ALL=(ALL) ALL" >> /etc/sudoers //将用户添加到wheel组里面 ]# usermod -aG wheel ops ]# id ops uid=1001(ops) gid=1001(ops) groups=1001(ops),10(wheel) ]# echo "Xzm7_GTD.com" |passwd --stdin ops(5)生成ssh密钥]# ssh-keygen ]# ls ~/.ssh/ authorized_keys id_rsa id_rsa.pub known_hosts //免密登录copy ]# ssh-copy-id -i ~/.ssh/id_rsa.pub ops@ip(6)安装docker]# cd /etc/yum.repos.d/ ]# wget https://download.docker.com/linux/centos/docker-ce.repo ]# yum install docker-ce docker-ce-cli containerd.io ]# systemctl start docker ]# systemctl enable docker //添加docker加速地址 ]# tee /etc/docker/daemon.json <<-'EOF' { "registry-mirrors": ["https://ba276e4c.mirror.aliyuncs.com"], "data-root": "/data/docker" } EOF ]# systemctl daemon-reload ]# systemctl restart docker(7)将用户添加到docker组]# usermod -aG docker ops ]# id ops uid=1001(ops) gid=1001(ops) groups=1001(ops),10(wheel),993(docker)(8)下载rke_linux-adm64]# wget https://github.com/rancher/rke/releases/download/v1.2.9/rke_linux-amd64 ]# mv rke_linux-adm64 /usr/local/bin/rke && chmod +x /usr/local/bin/rke ]# rke config [+] Cluster Level SSH Private Key Path [~/.ssh/id_rsa]: /home/ops/.ssh/id_rsa [+] Number of Hosts [1]: 1 [+] SSH Address of host (1) [none]: 192.168.122.100 [+] SSH Port of host (1) [22]: 22 [+] SSH Private Key Path of host (192.168.122.100) [none]: [-] You have entered empty SSH key path, trying fetch from SSH key parameter [+] SSH Private Key of host (192.168.122.100) [none]: [-] You have entered empty SSH key, defaulting to cluster level SSH key: ~/.ssh/id_rsa [+] SSH User of host (192.168.122.100) [ubuntu]: ops [+] Is host (192.168.122.100) a Control Plane host (y/n)? [y]: y [+] Is host (192.168.122.100) a Worker host (y/n)? [n]: y [+] Is host (192.168.122.100) an etcd host (y/n)? [n]: y [+] Override Hostname of host (192.168.122.100) [none]: [+] Internal IP of host (192.168.122.100) [none]: [+] Docker socket path on host (192.168.122.100) [/var/run/docker.sock]: /var/run/docker.sock [+] Network Plugin Type (flannel, calico, weave, canal, aci) [canal]: calico [+] Authentication Strategy [x509]: [+] Authorization Mode (rbac, none) [rbac]: [+] Kubernetes Docker image [rancher/hyperkube:v1.20.8-rancher1]: [+] Cluster domain [cluster.local]: [+] Service Cluster IP Range [10.43.0.0/16]: [+] Enable PodSecurityPolicy [n]: [+] Cluster Network CIDR [10.42.0.0/16]: [+] Cluster DNS Service IP [10.43.0.10]: [+] Add addon manifest URLs or YAML files [no]: ]# rke up //如报错 Failed to get job complete status for job rke-network-plugin-deploy-job in namespace kube-system ]# rke up 再执行一次注意主机配额必须2c2g最低配置，否则集群无法启动rke生成空白cluster.yml文件命令rke config --empty --name cluster.yml #生成空白cluster.yml文件 rke config --print #打印填写cluster.yml文件########################yaml################################ # If you intened to deploy Kubernetes in an air-gapped environment, # please consult the documentation on how to configure custom RKE images. nodes: - address: "192.168.30.110" port: "22" internal_address: "" role: [controlplane,etcd,worker] hostname_override: "node1" user: admin docker_socket: /var/run/docker.sock ssh_key: "" ssh_key_path: /home/ops/.ssh/id_rsa ssh_cert: "" ssh_cert_path: "" labels: {} taints: [] - address: "192.168.30.129" port: "22" internal_address: "" role: [controlplane,etcd,worker] hostname_override: "node2" user: admin docker_socket: /var/run/docker.sock ssh_key: "" ssh_key_path: ~/.ssh/id_rsa ssh_cert: "" ssh_cert_path: "" labels: {} taints: [] - address: "192.168.30.133" port: "22" internal_address: "" role: [controlplane,etcd,worker] hostname_override: "node3" user: admin docker_socket: /var/run/docker.sock ssh_key: "" ssh_key_path: ~/.ssh/id_rsa ssh_cert: "" ssh_cert_path: "" labels: {} taints: [] services: etcd: image: "" extra_args: {} extra_binds: [] extra_env: [] external_urls: [] ca_cert: "" cert: "" key: "" path: "" uid: 0 gid: 0 snapshot: null retention: "" creation: "" backup_config: null kube-api: image: "" extra_args: {} extra_binds: [] extra_env: [] service_cluster_ip_range: 10.43.0.0/16 service_node_port_range: "" pod_security_policy: false always_pull_images: false secrets_encryption_config: null audit_log: null admission_configuration: null event_rate_limit: null kube-controller: image: "" extra_args: {} extra_binds: [] extra_env: [] cluster_cidr: 10.42.0.0/16 service_cluster_ip_range: 10.43.0.0/16 scheduler: image: "" extra_args: {} extra_binds: [] extra_env: [] kubelet: image: "" extra_args: {} extra_binds: [] extra_env: [] cluster_domain: cluster.local infra_container_image: "" cluster_dns_server: 10.43.0.10 fail_swap_on: false generate_serving_certificate: false kubeproxy: image: "" extra_args: {} extra_binds: [] extra_env: [] network: plugin: flannel options: {} mtu: 0 node_selector: {} update_strategy: null authentication: strategy: x509 sans: [] webhook: null addons: "" addons_include: [] system_images: etcd: rancher/coreos-etcd:v3.4.3-rancher1 alpine: rancher/rke-tools:v0.1.58 nginx_proxy: rancher/rke-tools:v0.1.58 cert_downloader: rancher/rke-tools:v0.1.58 kubernetes_services_sidecar: rancher/rke-tools:v0.1.58 kubedns: rancher/k8s-dns-kube-dns:1.15.2 dnsmasq: rancher/k8s-dns-dnsmasq-nanny:1.15.2 kubedns_sidecar: rancher/k8s-dns-sidecar:1.15.2 kubedns_autoscaler: rancher/cluster-proportional-autoscaler:1.7.1 coredns: rancher/coredns-coredns:1.6.9 coredns_autoscaler: rancher/cluster-proportional-autoscaler:1.7.1 nodelocal: rancher/k8s-dns-node-cache:1.15.7 kubernetes: rancher/hyperkube:v1.18.3-rancher2 flannel: rancher/coreos-flannel:v0.12.0 flannel_cni: rancher/flannel-cni:v0.3.0-rancher6 calico_node: rancher/calico-node:v3.13.4 calico_cni: rancher/calico-cni:v3.13.4 calico_controllers: rancher/calico-kube-controllers:v3.13.4 calico_ctl: rancher/calico-ctl:v3.13.4 calico_flexvol: rancher/calico-pod2daemon-flexvol:v3.13.4 canal_node: rancher/calico-node:v3.13.4 canal_cni: rancher/calico-cni:v3.13.4 canal_flannel: rancher/coreos-flannel:v0.12.0 canal_flexvol: rancher/calico-pod2daemon-flexvol:v3.13.4 weave_node: weaveworks/weave-kube:2.6.4 weave_cni: weaveworks/weave-npc:2.6.4 pod_infra_container: rancher/pause:3.1 ingress: rancher/nginx-ingress-controller:nginx-0.32.0-rancher1 ingress_backend: rancher/nginx-ingress-controller-defaultbackend:1.5-rancher1 metrics_server: rancher/metrics-server:v0.3.6 windows_pod_infra_container: rancher/kubelet-pause:v0.1.4 ssh_key_path: ~/.ssh/id_rsa ssh_cert_path: "" ssh_agent_auth: false authorization: mode: rbac options: {} ignore_docker_version: null kubernetes_version: "" private_registries: [] ingress: provider: "" options: {} node_selector: {} extra_args: {} dns_policy: "" extra_envs: [] extra_volumes: [] extra_volume_mounts: [] update_strategy: null cluster_name: "" cloud_provider: name: "" prefix_path: "" addon_job_timeout: 0 bastion_host: address: "" port: "" user: "" ssh_key: "" ssh_key_path: "" ssh_cert: "" ssh_cert_path: "" monitoring: provider: "" options: {} node_selector: {} update_strategy: null replicas: null restore: restore: false snapshot_name: "" dns: null cluster.yaml(9)启动rke搭建kubernetes集群1、安装kubectl命令]# rke up ]# tee /etc/yum.repos.d/kubectl.repo <<EOF [kubernetes] name=Kubernetes baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/ enabled=1 gpgcheck=0 EOF ]# yum -y install kubectl.x86_64 //cd 到执行rkeconfig目录 ]# cd ~/ ]# vim .bash_profile export KUBECONFIG=/home/ops/kube_config_cluster.yml ]# source .bash_profile ]# kubectl get nodes NAME STATUS ROLES AGE VERSION 10.60.161.223 Ready controlplane,etcd,worker 79m v1.20.82、通过RKE安装k8s集群成功,启动的时候有些节点启动的比较慢。需要稍微等待一段时间(10)可以先找一台网络好一点的，pull全部镜像，再保存到本地，cp到所有主机]# docker save -o images.tgz `docker images|awk 'NR>1 {print $1":"$2}'`二、RKE的环境清理(1)所有节点分别执行]# mkdir rancher ] cat > clear.sh << EOF for i in $(df -h|grep kubelet |awk -F % '{print $2}') do umount -l ${i} done rm /var/lib/kubelet/* -rf rm /etc/kubernetes/* -rf rm /var/lib/rancher/* -rf rm /var/lib/etcd/* -rf rm /var/lib/cni/* -rf rm -rf /var/run/calico iptables -F && iptables -t nat -F ip link del flannel.1 docker ps -a|awk '{print $1}'|xargs docker rm -f docker volume ls|awk '{print $2}'|xargs docker volume rm rm -rf /var/etcd/ rm -rf /run/kubernetes/ for i in $(docker ps -aq) do docker rm -fv $(i) done for i in $(docker volume ls|awk '{print $2}') do docker volume rm ${i} done rm -rf /etc/cni rm -rf /opt/cni systemctl restart docker EOF ]# sh rancher/clear.sh 清理脚本(2)清理残留目录结束。如果还有问题可能需要卸载所有节点上的docker首先查看Docker版本 ]# yum list installed | grep docker docker-ce.x86_64 18.05.0.ce-3.el7.centos @docker-ce-edge 执行卸载 ]# yum -y remove docker-ce.x86_64 删除存储目录 ]# rm -rf /etc/docker ]# rm -rf /run/docker ]# rm -rf /var/lib/dockershim ]# rm -rf /var/lib/docker 如果发现删除不掉，需要先 umount，如 ]# umount /var/lib/docker/devicemapper 卸载docker三、扩容节点，缩容节点(1)添加节点修改cluster.yal 将需要添加的节点配置，然后运行more cluster.yml nodes: - address: 172.20.101.103 user: ptmind role: [controlplane,worker,etcd] - address: 172.20.101.104 user: ptmind role: [controlplane,worker,etcd] - address: 172.20.101.105 user: ptmind role: [controlplane,worker,etcd] - address: 172.20.101.106 user: ptmind role: [worker] labels: {traefik: traefik-outer}(2)在新节点创建账户useradd -s /usr/bin/bash ops -G docker #创建账户添加到docker组里 将rke节点的ops账户执行ssh-copy-id命令 ssh-copy-id ops@10.10.10.10 #rke部署节点添加新节点密钥登录(3)执行添加节点操作]# rke up --update-only(3)rke 删除节点修改cluster.yal 将需要删除的节点配置删除，然后运行more cluster.yml nodes: - address: 172.20.101.103 user: ptmind role: [controlplane,worker,etcd] - address: 172.20.101.104 user: ptmind role: [controlplane,worker,etcd] - address: 172.20.101.105 user: ptmind role: [controlplane,worker,etcd] 删除# - address: 172.20.101.106 删除# user: ptmind 删除# role: [worker] 删除# labels: {traefik: traefik-outer}(4)执行删除节点操作]# rke up --update-only问题：当node节点处于NotReady状态下，对节点不可做操作，比如做了删除节点操作，会报错，删除不了节点。解决办法：1、手动删除节点上的组件。2、通过命令移除节点的角色]# kubectl label node prod-129 node-role.kubernetes.io/controlplane-问题：k8s 集群的节点处于 SchedulingDisabled解决方法：]# kubectl patch node NodeName -p "{\"spec\":{\"unschedulable\":false}}"或者：设置不可调度 ]# kubectl cordon node07-ingress 取消节点不可调度 ]# kubectl uncordon node07-ingress 驱逐节点的pod ]# kubectl drain --ignore-daemonsets --delete-local-data node07-ingress 删除节点 ]# kubectl delete node node07-ingressyum 安装kubectl[kubernetes] name=Kubernetes baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/ enabled=1 gpgcheck=0 yum clean all yum makecache yum -y install kubectl

kubeadm部署k8s 关闭防火墙systemctl stop firewalld ; systemctl disable firewalld setenforce 0 sed -i.bak 's@SELINUX=.*@SELINUX=disabled@g' /etc/selinux/config swapoff -a && sudo sed -i '/swap/ s/^/#/' /etc/fstab && mount -a 内核参数修改echo "modprobe br_netfilter" >> /etc/profile cat > /etc/sysctl.d/k8s.conf <<EOF net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 EOF sysctl -p /etc/sysctl.d/k8s.conf 配置阿里云 Kubernetes yum 源cat <<EOF > /etc/yum.repos.d/kubernetes.repo [kubernetes] name=Kubernetes baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/ enabled=1 gpgcheck=1 repo_gpgcheck=1 gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOF安装基础软件包yum -y install epel-release yum-utilsyum -y install device-mapper-persistent-data lvm2 wget net-tools nfs-utils iptables-services lrzsz gcc gcc-c++ make cmake libxml2-devel openssl-devel curl curl-devel unzip sudo libaio-devel wget vim ncurses-devel autoconf automake zlib-devel openssh-server socat ipvsadm conntrack telnet iptables-services nginx-mod-stream开启ipvs开启 ipvs 不开启 ipvs 将会使用 iptables，但是效率低，所以官网推荐需要开通 ipvs 内核 tee ipvs.modules <<-'EOF' #!/bin/bash ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack" for kernel_module in ${ipvs_modules}; do /sbin/modinfo -F filename ${kernel_module} > /dev/null 2>&1 if [ 0 -eq 0 ]; then /sbin/modprobe ${kernel_module} fi done EOF chmod 755 ipvs.modules && bash ipvs.modules && lsmod | grep ip_vs 安装dockeryum install -y yum-utils device-mapper-persistent-data lvm2 yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo yum install docker-ce-19.03.5 docker-ce-cli-19.03.5 containerd.io -y启动dockersystemctl start docker && systemctl enable docker tee /etc/docker/daemon.json <<-'EOF' { "registry-mirrors": ["https://hxn0tro1.mirror.aliyuncs.com"], "exec-opts": ["native.cgroupdriver=systemd"] } EOF systemctl daemon-reload && systemctl restart docker && docker info |egrep " Cgroup Driver:" && systemctl status docker --no-pager安装kubeadm# 查看版本 yum search kubelet --showduplicates | sort -r yum -y install kubelet-1.23.0 kubeadm-1.23.0 kubectl-1.23.0 systemctl enable kubelet下载镜像tee image.txt <<-'EOF' docker pull registry.aliyuncs.com/google_containers/kube-proxy:v1.20.6 docker pull registry.aliyuncs.com/google_containers/kube-scheduler:v1.20.6 docker pull registry.aliyuncs.com/google_containers/kube-controller-manager:v1.20.6 docker pull registry.aliyuncs.com/google_containers/kube-apiserver:v1.20.6 docker pull calico/pod2daemon-flexvol:v3.18.0 docker pull calico/node:v3.18.0 docker pull calico/cni:v3.18.0 docker pull calico/kube-controllers:v3.18.0 docker pull registry.aliyuncs.com/google_containers/etcd:3.4.13-0 docker pull registry.aliyuncs.com/google_containers/coredns:1.7.0 docker pull registry.aliyuncs.com/google_containers/pause:3.2 EOF bash image.txt使用kubeadm初始化集群kubeadm init --kubernetes-version=1.23.0 --image-repository registry.aliyuncs.com/google_containers --apiserver-advertise-address=192.168.4.6 --service-cidr=10.10.0.0/16 --pod-network-cidr=10.244.0.0/16 --ignore-preflight-errors=all --v=5注：--image-repository registry.aliyuncs.com/google_containers 为保证拉取镜像不到国外站点拉取，手动指定仓库地址为 registry.aliyuncs.com/google_containers。kubeadm 默认从k8ss.grc.io 拉取镜像。 本地有导入到的离线镜像，所以会优先使用本地的镜像节点注册命令保存下来，以备稍后使用 --discovery-token-ca-cert-hash sha256:cbc4154daaba6b6a1355a7c54ec22ecdafab566464fd7ed257097dafedeb7065 配置 kubectl 的配置文件，保存一个证书，这样 kubectl 命令可以使用这个证书对 k8s 集群进行管理 # mkdir -p $HOME/.kube # cp -i /etc/kubernetes/admin.conf $HOME/.kube/config # chown $(id -u):$(id -g) $HOME/.kube/config # kubectl get nodes安装网络插件calico# 使用 Calico 网络插件 $ kubectl apply -f https://docs.projectcalico.org/manifests/calico.yamlFlannel# 使用 Flannel 网络插件 $ kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml # 或者访问 https://gitee.com/kexinzz/helm/blob/master/kube-flannel.ymlnode污点这里我们会发现node节点污点，删除不掉于是，我们就更改pod yaml接受容忍#找到yaml文件中 tolerations: 字段添加如下内容 tolerations: - key: node.kubernetes.io/not-ready operator: Exists effect: NoSchedule #完成后我们部署pod插件，就可以看到pod成功部署了验证k8s加入工作节点使用这些信息来生成新的join命令kubeadm token create --print-join-command在每个工作节点上，运行之前保存的加入命令，例如# 在工作节点上运行加入命令 $ sudo kubeadm join <Master节点IP>:<Master节点端口> --token <Token> --discovery-token-ca-cert-hash <CA证书哈希值> $ kubeadm join 192.168.4.6:6443 --token 6n60gw.od78mw7dtoyuz8dn \ --discovery-token-ca-cert-hash sha256:e53b20f5eeb50b5f503da46af84e0c74fe736245d32321176d9a0c0f1a119c9f \ --node-name=192.168.4.7 --v=5请注意，上述命令中的 、<Master节点IP>、<Master节点端口>、 和 <CA证书哈希值> 都需要根据你的实际情况进行替换。添加master节点生成 Join Token在现有的某个 master 节点上，获取一个用于加入的 token 和 etcd 参数 ：kubeadm init phase upload-certs --upload-certs这会返回一个证书和一个启动的使用 token。使用这些信息来生成新的 join 命令kubeadm token create --print-join-command加入新的 Master 节点kubeadm join <master-ip>:<master-port> --token <token> \ --discovery-token-ca-cert-hash sha256:<hash> \ --control-plane --certificate-key <certificate-key> \ --node-name=192.168.4.7 --v=5验证集群状态一旦kubeadm join命令成功完成，在新节点上检查kubelet和kubeadmkubectl get nodes