堆栈内存溢出
  繁体   English   中英

从 haproxy 日志监控用户 rest api 调用使用情况的脚本

[英]Script to monitor user's rest api calls usage from haproxy log

 原文  2022-09-25 13:43:32  0   1    bash/ shell/ awk/ sed/ script

问题描述

如何识别过去两个小时内每分钟超过 100 次的用户出现次数？ 要搜索的文件是一个 haproxy 日志，下面是示例数据。

10.11.12.220:51196 [20/Sep/2022:03:41:57.247] https-in~ haproxy_http_ext/node2 0/0/0/14/14 400 833 - - --VN 364/364/8/2/0 0/0 {https://....com} {user1} "POST /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:08:48:20.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"

如果有人为此提供任何 awk shell 脚本，请分享。 提前致谢。

#!/bin/bash
WDIR=`dirname $0`
LOG_DIR="${WDIR}/logs"
LOCK="${WDIR}/check_restAPI_call.lock"
RESTUSAGE_LOG="/etc/haproxy/logs/rest_usage.log"
HAPROXY_LOG="/etc/haproxy/logs/haproxy.log"

if [[ "$#" -ge 1 ]]; then
    echo "No arguments required to run this script. Please rerun without passing arguments"
    exit
fi

echo "--------------------------------------------------------------------------------"
echo ""
echo "                     RESTAPI CALLS USAGE SCRIPT - START                         "
echo ""
echo "--------------------------------------------------------------------------------"
echo ""
echo "Enter the duration in mins (>2) from the current time for which you need the data: \c"
read RESTUSAGE_CALLS_MIN
echo ""
if [[ ! "${RESTUSAGE_CALLS_MIN}" =~ ^[0-9]+$ ]]; then
    echo "--------------------------------------------------------------------------------"
    echo ""
    echo "Time should not be empty or entered alphabet."
    echo "Enter integer value, which is considered as MINUTES to process the RESTAPI log files."
    echo ""
    echo "--------------------------------------------------------------------------------"
    exit
fi
echo "Enter the value of the maximum limit of the RESTAPI calls/muinute: \c"
read EXCEED_LIMIT
echo ""
if [[ ! "${EXCEED_LIMIT}" =~ ^[0-9]+$ ]]; then
    echo "--------------------------------------------------------------------------------"
    echo ""
    echo "Exceeding Limit value should not be empty or entered alphabet."
    echo "Enter interger value, which set as exceeding limit to process the RESTAPI log files."
    echo ""
    echo "--------------------------------------------------------------------------------"
    exit
else
    echo "Processing the log files based on requirement. Please wait..!"
    echo ""

    CSV_FILE="${LOG_DIR}/output.csv"
        echo "Date & Time(UTC+3);IP Address;UserName;Mail ID;No. of calls;Node(s);Public/Private" > ${CSV_FILE}

    #required log files
        RESTUSAGE_CALLS="${LOG_DIR}/rest_calls.log"
    RESTUSAGE_CALLS_TMP="${LOG_DIR}/rest_calls_tmp.log"
        RESTUSAGE_CALLS_DATE="${LOG_DIR}/rest_calls_date.log"
        RESTUSAGE_CALLS_IP="${LOG_DIR}/rest_calls_IP.log"
        RESTUSAGE_TIME=$(date -d "`echo ${RESTUSAGE_CALLS_MIN}` minute ago" +%d/%b/%Y:%H:%M)
        #RESTUSAGE_TIME="26/May/2020:03:45"
        SYS_TIME=`date +%d/%b/%Y:%H:%M`
        awk -v pat="${RESTUSAGE_TIME}" '$0 ~ pat,0' ${RESTUSAGE_LOG} > ${RESTUSAGE_CALLS}
    if [[ ! -s ${RESTUSAGE_CALLS} ]]; then
        echo "No data found in the log file (${RESTUSAGE_LOG}). Please rerun the script..!"
        echo ""
        exit
    fi

    awk -v th="${EXCEED_LIMIT}" '{if (NF == 1) print $0 ; else if ($1 > th) print $2}' ${RESTUSAGE_CALLS} > ${RESTUSAGE_CALLS_TMP}
    grep "\[" ${RESTUSAGE_CALLS_TMP} | uniq > ${RESTUSAGE_CALLS_DATE}
    grep -v "\[" ${RESTUSAGE_CALLS_TMP} | sort | uniq > ${RESTUSAGE_CALLS_IP}
    if [[ ! -s ${RESTUSAGE_CALLS_TMP} || ! -s ${RESTUSAGE_CALLS_DATE} || ! -s ${RESTUSAGE_CALLS_IP} ]]; then
                echo "No RESTAPI calls are exceeding limit - ${EXCEED_LIMIT}"
        echo ""
                exit
        fi

    HAPROXY_CALLS="${LOG_DIR}/haproxy_calls.log"
    IP_USERS="${LOG_DIR}/ip_users"
    NODES="${LOG_DIR}/nodes"
    NODES_TMP="${LOG_DIR}/nodes.tmp"
    while read -r IP; do
        nslookup ${IP} > /dev/null
        if [[ $? -eq 0 ]]; then
            IP_TYPE="Public"
        else
            IP_TYPE="Private"
        fi
        for DATE_TIME in `cat ${RESTUSAGE_CALLS_DATE} | tr -d "[]"`; do
            grep ${DATE_TIME} ${HAPROXY_LOG} | grep ${IP} | grep "/rest/api" > ${HAPROXY_CALLS}
            if [[ `cat ${HAPROXY_CALLS} | wc -l` -ge ${EXCEED_LIMIT} ]]; then
                if [[ -s ${HAPROXY_CALLS} ]]; then
                    grep ${DATE_TIME} ${HAPROXY_CALLS} | grep ${IP} | cut -d" " -f15 | sort | uniq | tr -d "{}" | grep -v ^$ > ${IP_USERS}
                    while read -r USER; do
                        USER_CALLS=`grep ${DATE_TIME} ${HAPROXY_CALLS} | grep ${IP} | grep -w ${USER} | wc -l`
                        grep ${DATE_TIME} ${HAPROXY_CALLS} | grep ${IP} | grep -w ${USER} | cut -d" " -f5 | cut -d"/" -f2 > ${NODES}
                        cat ${NODES} | sort | uniq > ${NODES_TMP}
                        USER_NODES=""
                        for NODE in `cat ${NODES_TMP}`; do
                            NODE_COUNT=`grep -w ${NODE} ${NODES} | wc -l`
                            USER_NODES="${USER_NODES},${NODE}-${NODE_COUNT}"
                        done
                        USER_NODES=`echo ${USER_NODES} | sed 's/^,//'`
                        GET_USER_MAIL ${USER}
                        if [[ ! -z ${USER_CALLS} || ! -z ${USER_NODES} ]]; then
                            echo "${DATE_TIME};${IP};${USER};${MAILID};${USER_CALLS};${USER_NODES};${IP_TYPE}" >> ${CSV_FILE}
                            echo "<tr><td>${DATE_TIME}</td><td>${IP}</td><td>${USER}</td><td>${MAILID}</td><td>${USER_CALLS}</td><td>${USER_NODES}</td><td>${IP_TYPE}</td></tr>" >> ${HTML_BODY}
                        fi
                    done < ${IP_USERS}
                fi
            fi
        done
    done < ${RESTUSAGE_CALLS_IP}
fi

1 个解决方案

解决方案1
 0  2022-09-25 19:09:26

有趣的问题。 这可能是一个有用的开始。

对于每个具有用户名和时间戳的访问，都会创建一个新的访问队列条目。 此外，搜索用户的访问队列以删除超过 2 小时的时间戳。 最后，检查用户的访问队列以确定用户的访问队列中是否有超过 100 个条目。

每个用户都有一个访问队列。 所有访问队列的集合都按用户名进行索引。 每个访问队列在键条目“0”处存储头索引值，在键条目“1”处存储尾索引值，因此第一个时间戳值实际上是键“2”。

方法使用gawk 日期时间函数和队列来确定是否在 2 小时内进行了超过 100 次访问。

使用gawk 匹配 function和正则表达式分组来提取用户名和时间戳的详细信息。

使用gawk 数组 delete function删除访问，但在当前时间戳超过 2 小时后，可以在此处完成更多工作以删除用户。

样品 output：

./doit.awk notes 
AT TIME 20/Sep/2022:22:48:21 USER user2 HAS 6 ACCESSES IN 599 SECONDS

当输入看起来像这样时：


https://stackoverflow.com/questions/73844964/script-to-monitor-users-rest-api-calls-usage-from-haproxy-log


10.11.12.220:51196 [20/Sep/2022:03:41:57.247] https-in~ haproxy_http_ext/node2 0/0/0/14/14 400 833 - - --VN 364/364/8/2/0 0/0 {https://....com} {user1} "POST /rest/api/2/issue HTTP/1.1"

10.150.40.50:59552 [20/Sep/2022:08:48:20.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:08:48:20.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:08:48:20.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:08:48:20.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:08:48:20.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:08:48:20.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:22:48:21.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:22:48:22.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:22:48:23.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:22:48:24.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:22:48:25.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"
10.150.40.50:59552 [20/Sep/2022:22:58:20.968] https-in~ haproxy_http_ext/node4 0/0/0/2/2 200 609 - - --VN 2729/2729/30/7/0 0/0 {https://....com} {user2} "GET /rest/api/2/issue HTTP/1.1"

#!/usr/local/bin/gawk -f

BEGIN{
 d1 = mktime( "1971 01 01 00 00 00" )
 m=split("Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec",d,"|")
 for(o=1;o<=m;o++){ months[d[o]]=sprintf("%02d",o) }
}

function add_access(user,ts) {
  if ( user in access ) {
    head = access[user][0]++
    access[user][head] = ts
   } else {
     tail = head = 2
     access[user][0] = head
     access[user][1] = tail
     access[user][2] = ts # first value
   }
   tail = access[user][1]
   old_ts = access[user][tail]
   while ( ts - old_ts > 2 * 3600 ) {
     delete access[user][tail++]  # remove TS beyond 2 hours
     old_ts = access[user][tail]
   }
   access[user][1] = tail
   # check if access exceeds 100 in 2 hours
   access_count = ( head - tail ) + 1
   if ( access_count > 100 ) {
     elapsed = access[user][head] - access[user][tail]
     print "AT TIME " strftime("%d/%b/%Y:%H:%M:%S", old_ts) " USER " user " HAS " access_count " ACCESSES IN " elapsed " SECONDS" 
   }
}

{
  m = match($14 , /{(.*)}/, u) != 0 
  if ( m != 0 ) {
    user = u[1]
    m = match($2 , /\[([^\/]+)\/([^\/]+)\/([^:]+):([^:]+):([^:]+):([^\.]+).*\]/, d)
    if ( m != 0 ) {
      DD = d[1]
      MON = months[d[2]]
      YYYY = d[3]
      HH = d[4]
      MM = d[5]
      SS = d[6]
      ts = mktime( YYYY " " MON " " DD " " HH " " MM " " SS )
      add_access(user,ts)
    }
  }
}

暂无
暂无

声明:本站的技术帖子网页，遵循CC BY-SA 4.0协议，如果您需要转载，请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.

相关问题 Shell脚本以监视磁盘使用情况 Bash脚本监视资源使用情况和电子邮件警告 用于监视系统信息的脚本：日期和时间，CPU使用率和内存 用于监控 API json 响应的 Bash 脚本 如何从 Python 脚本中注销当前用户？ 如何使用 bash 脚本从日志文件中查找未使用的 API？ Bash脚本监视日志，匹配关键字然后发送命令，恢复监视 从who命令获取用户的登录日期 从 bash 脚本监视标准输入的变化 Bash 监控磁盘使用情况
相关标签
 
粤ICP备18138465号  © 2020-2024 STACKOOM.COM