1 前言

一个问题，一篇文章，一出故事。
笔者今天遇到Nginx代理的上游服务器报错，由于Nginx没有自动通知功能，于是笔者结合Tigase来完成警告。

2 最佳实践

2.1 测试环境

2.1.1 Nginx反向代理环境

如何部署反向代理服务器Nginx？

2.1.2 Tigase即时消息服务

安装部署Tigase Server 8.3.x?

2.2 部署监控脚本

2.2.1 创建脚本

vim ~/scripts/checkUpstreamTimeOut.sh

加入如下配置，

#!/bin/bash

tigaseAppID="10086"
tigaseAppPassword="*********"
tigaseEmployeeID="will,jeff"
tigaseApiURL="https://tigase.cmdschool.org:8092/tigase/pushText"
logFile="/var/log/nginx/error.log"
checkLog="/var/log/nginx/checkUpstreamTimeOut.log"
searchString="upstream timed out"

checkLogs() {
  # 检查日志的标识
  errorCounter=0

  # 检查该日志是否已记录
  if [ ! -f "$checkLog" ]; then
    touch "$checkLog"
    if [ $? -ne 0 ]; then
      echo "Error creating log file $checkLog"
      return 0
    fi
  fi

  # 获取匹配的日志
  grepLog=$(grep "$searchString" "$logFile")

  IFS=$'\n'
  for errLog in $(grep "$searchString" "$logFile"); do
    # 日志为空直接跳过循环
    if [ -z "$errLog" ]; then
      continue
    fi


    if grep -qF "$errLog" "$checkLog"; then
      continue
    fi
    
    # 记录日志
    echo "$errLog" >> "$checkLog"

    # 统计错误日志的总数
    errorCounter=$(($errorCounter + 1))
  done

  echo "$errorCounter"
}

getErrSer() {
  errorCounter=$1
  errMsg=$(grep "$searchString" "$checkLog" | tail -n "$errorCounter")
  serList=$(echo "$errMsg" | grep -o 'upstream: "https://[^:]*' | cut -d '/' -f3 | sort -u)
  echo "$serList"
}

urlencode() {
    python3 -c "import sys, urllib.parse as ul; print(ul.quote_plus(sys.argv[1]))" "$@"
}

getPrefix() {
    hostName="$1"
    if [[ "$hostName" == *.* ]]; then
        echo "$hostName" | cut -d '.' -f 1
    else
        echo "$hostName"
    fi
}

errorCounter="$(checkLogs)"
serList=$(getErrSer "$errorCounter")

if [ "$errorCounter" -le "0" ]; then
        exit 0
fi

hostName=$(getPrefix `hostname`)
logMessage="$hostName Upstream TimOut:"$'\n'"$serList"
encodeMessage=$(urlencode "$logMessage")
tigaseMessageBody="$encodeMessage"
curlData="appId=$tigaseAppID&password=$tigaseAppPassword&employee_id=$tigaseEmployeeID&body=$tigaseMessageBody"
curl -X POST -d "$curlData" "$tigaseApiURL" &> /dev/null
exit 0

2.2.2 测试脚本

bash ~/scripts/checkUpstreamTimeOut.sh

2.2.3 设置脚本触发

crontab -e

加入如下设置，

*/5 * * * * bash ~/scripts/checkUpstreamTimeOut.sh