#!/bin/bash

#============================================================================================================
# Name                : oom-checkv2.sh
# Description         : Provides a server summary and top resource-consuming processes.
# Usage               : ./oom-checkv2.sh
#============================================================================================================

# Add colors
red=$(tput setaf 1)
green=$(tput setaf 2)
yellow=$(tput setaf 3)
blue=$(tput setaf 4)
reset=$(tput sgr0)
bold=$(tput bold)

# Function to get color based on percentage
# 85-89% -> yellow, 90%+ -> red
get_usage_color() {
    local pct=$1
    if [ -z "$pct" ]; then echo "$reset"; return; fi
    if [ "$pct" -ge 90 ]; then
        echo "$red"
    elif [ "$pct" -ge 85 ]; then
        echo "$yellow"
    else
        echo "$reset"
    fi
}

# Server Summary
echo "${blue}+===================================+${reset}"
echo "${blue}| ${bold}Server Summary${reset}                    ${blue}|${reset}"
echo "${blue}+===================================+${reset}"
echo "${blue}|${reset} ${bold}Hostname:${reset} $(hostname)"
echo "${blue}|${reset} ${bold}cPanel Accounts:${reset} $(ls /var/cpanel/users/ 2>/dev/null | wc -l)"

# Cores and Load Averages
CORES=$(nproc)
LOAD_LIMIT=$(( CORES * 2 ))
read L1 L2 L3 <<< $(uptime | awk -F'average: ' '{print $2}' | sed 's/,//g')

# Color Load Averages if they exceed double the cores
COLOR_L1=$reset; awk -v v="$L1" -v lim="$LOAD_LIMIT" 'BEGIN {exit !(v>lim)}' && COLOR_L1=$red
COLOR_L2=$reset; awk -v v="$L2" -v lim="$LOAD_LIMIT" 'BEGIN {exit !(v>lim)}' && COLOR_L2=$red
COLOR_L3=$reset; awk -v v="$L3" -v lim="$LOAD_LIMIT" 'BEGIN {exit !(v>lim)}' && COLOR_L3=$red



echo "${blue}|${reset} ${bold}Uptime:${reset} $(uptime -p | sed 's/^up //')"
echo "${blue}|${reset} ${bold}load averages:${reset} ${COLOR_L1}${L1}${reset}, ${COLOR_L2}${L2}${reset}, ${COLOR_L3}${L3}${reset}"
echo "${blue}|${reset} ${bold}Cores:${reset} $CORES cores"
# Disk Usage for Root Partition
DISK_PCT=$(df -h / | awk 'NR==2 {print $5}' | sed 's/%//')
DISK_COLOR=$(get_usage_color "$DISK_PCT")
DISK_INFO=$(df -h / | awk -v c="$DISK_COLOR" -v r="$reset" 'NR==2 {printf "%s/%s (%s%s%s)", $3, $2, c, $5, r}')
echo "${blue}|${reset} ${bold}Disk Usage:${reset} $DISK_INFO"

# RAM Usage
RAM_PCT=$(free -m | awk '/^Mem:/ {print int($3*100/$2)}')
RAM_COLOR=$(get_usage_color "$RAM_PCT")
RAM_INFO=$(free -m | awk -v c="$RAM_COLOR" -v r="$reset" '/^Mem:/ {printf "%.2f/%.2fGB (%s%.2f%%%s)", $3/1024, $2/1024, c, $3*100/$2, r}')
echo "${blue}|${reset} ${bold}Ram Usage:${reset} $RAM_INFO"

# Swap Usage
SWAP_TOTAL=$(free -m | awk '/^Swap:/ {print $2}')
if [ -z "$SWAP_TOTAL" ] || [ "$SWAP_TOTAL" -eq 0 ]; then
    echo "${blue}|${reset} ${bold}Swap Usage:${reset} Swap is disabled."
else
    SWAP_PCT=$(free -m | awk '/^Swap:/ {if ($2>0) print int($3*100/$2); else print 0}')
    SWAP_COLOR=$(get_usage_color "$SWAP_PCT")
    SWAP_INFO=$(free -m | awk -v c="$SWAP_COLOR" -v r="$reset" '/^Swap:/ {printf "%.2f/%.2fGB (%s%.2f%%%s)", $3/1024, $2/1024, c, $3*100/$2, r}')
    echo "${blue}|${reset} ${bold}Swap Usage:${reset} $SWAP_INFO"
fi

echo "${blue}+===================================+${reset}"
echo "${blue}| ${bold}Server Summary${reset}                    ${blue}|${reset}"
echo "${blue}+===================================+${reset}"

echo ""


# Top 10 Processes Using RAM
echo "${yellow}---------------------------------------------------------------------${reset}"
echo "${bold}Top 10 Processes Using RAM (Grouped by User & Process)${reset}"
echo "${yellow}---------------------------------------------------------------------${reset}"
printf "${bold}%-15s %-25s %-10s %s${reset}\n" "USER" "PROCESS NAME" "COUNT" "TOTAL RAM"
echo "${yellow}---------------------------------------------------------------------${reset}"

# Use ps to get UID, RSS, and the command name
ps -eo uid,rss,comm --no-headers | awk '
{
    uid = $1;
    rss = $2;
    cmd = $3;
    for(i=4; i<=NF; i++) cmd = cmd " " $i;
    
    mem[uid SUBSEP cmd] += rss;
    count[uid SUBSEP cmd]++;
}
END {
    for (idx in mem) {
        split(idx, keys, SUBSEP);
        mb = mem[idx] / 1024;
        printf "%.2f|%s|%s|%d\n", mb, keys[1], keys[2], count[idx];
    }
}' | sort -t'|' -k1,1nr | head -n 10 | while IFS='|' read -r mb uid cmd count; do
    username=$(id -nu "$uid" 2>/dev/null || echo "$uid")
    printf "%-15s %-25s %-10s %7.2f MB\n" "$username" "$cmd" "$count" "$mb"
done

echo ""

# Top 5 Processes Using Swap
if [ -n "$SWAP_TOTAL" ] && [ "$SWAP_TOTAL" -gt 0 ]; then
    echo "${yellow}---------------------------------------------------------------------${reset}"
    echo "${bold}Top 5 Processes Using Swap:${reset}"
    echo "${yellow}---------------------------------------------------------------------${reset}"

    grep VmSwap /proc/[0-9]*/status 2>/dev/null | awk '{print $2, $1}' | sort -nr | head -n 5 | while read -r swap_kb path; do
        if [ "$swap_kb" -gt 0 ]; then
            pid=$(echo "$path" | awk -F'/' '{print $3}')
            proc_name=$(cat "/proc/$pid/comm" 2>/dev/null || echo "Unknown")
            awk -v kb="$swap_kb" -v p="$pid" -v n="$proc_name" 'BEGIN { printf "PID: %-8s | Swap: %7.2f MB | Cmd: %s\n", p, kb/1024, n }'
        fi
    done

    echo "${yellow}---------------------------------------------------------------------${reset}"
    echo "${yellow}---------------------------------------------------------------------${reset}"
fi

 


# Section: I/O Statistics
echo "${blue}+===================================+${reset}"
echo "      ${bold}I/O Statistics (iostat)${reset}"
echo "${blue}+===================================+${reset}"

if command -v iostat >/dev/null 2>&1; then
    iostat | awk -v r="$red" -v g="$green" -v y="$yellow" -v b="$bold" -v rs="$reset" '
    BEGIN {
        # Print the column headers with exactly 11 spaces per column
        printf "%-11s%-11s%-11s%-11s%-11s%-11s\n", "%user", "%nice", "%system", "%iowait", "%steal", "%idle"
        print "---------------------------------------------------------------------"
    }
    /^avg-cpu:/ {
        # Move to the next line which contains the actual percentage numbers
        getline 
        
        # 1. Pad the raw numbers to exactly 11 characters wide FIRST
        u  = sprintf("%-11s", $1)
        n  = sprintf("%-11s", $2)
        sys= sprintf("%-11s", $3)
        io = sprintf("%-11s", $4)
        st = sprintf("%-11s", $5)
        id = sprintf("%-11s", $6)

        # 2. Apply Yellow (70+) or Red (90+) to the pre-padded iowait string
        if (($4 + 0) >= 90) io = r b io rs
        else if (($4 + 0) >= 70) io = y b io rs

        # 3. Apply Yellow (70+) or Green (90+) to the pre-padded idle string
        if (($6 + 0) >= 90) id = g b id rs
        else if (($6 + 0) >= 70) id = y b id rs

        # 4. Print the final assembled line and closing border
        print u n sys io st id
        print "---------------------------------------------------------------------"
    }'
else
    echo -e "${red}iostat command not found. Please install sysstat package.${reset}"
    echo "---------------------------------------------------------------------"
fi
echo ""

# Section: Checking Apache (httpd)
echo "${blue}+===================================+${reset}"
echo "      ${bold}Checking Apache (httpd)${reset}"
echo "${blue}+===================================+${reset}"

# Detect actual Apache process and service name
if pgrep -x "httpd" >/dev/null; then
    APACHE_PROC="httpd"
    APACHE_SVC="httpd"
elif pgrep -x "apache2" >/dev/null; then
    APACHE_PROC="apache2"
    APACHE_SVC="apache2"
else
    APACHE_PROC="httpd"
    APACHE_SVC="httpd"
fi

# Apache Status
if systemctl is-active "$APACHE_SVC" >/dev/null 2>&1; then
    echo "Apache ($APACHE_PROC) is ${green}running${reset}"
else
    echo "Apache ($APACHE_PROC) is ${red}not running${reset}"
fi
echo "-----------------------------------"
echo ""

# PHP-FPM warnings
PHP_FPM_WARNINGS=$(grep -h 'max_children' /opt/cpanel/ea-php*/root/usr/var/log/php-fpm/error.log 2>/dev/null | tail -n 1)
if [[ -n "$PHP_FPM_WARNINGS" ]]; then
    PHP_FPM_WARNINGS_WITH_FILE=$(grep -l 'max_children' /opt/cpanel/ea-php*/root/usr/var/log/php-fpm/error.log 2>/dev/null | xargs grep -H 'max_children' | tail -n 1)
    echo "${bold}PHP-FPM max children reached:${reset} $PHP_FPM_WARNINGS_WITH_FILE"
else
    echo "PHP-FPM no warnings."
fi

echo ""
echo "-----------------------------------"

# MaxRequestWorkers detection
MAX_REQ_WORKERS=$(grep -RE 'MaxRequestWorkers|MaxClients' /etc/apache2/conf/httpd.conf /etc/httpd/conf/httpd.conf /etc/apache2/apache2.conf 2>/dev/null | grep -v "#" | head -n 1 | awk '{print $NF}')
echo "${bold}Apache MaxRequestWorkers:${reset} ${MAX_REQ_WORKERS:-"Not set in config"}"
echo "------------------------------------"

# Memory Calculations logic
# Average RAM per Apache process (MB)
AVG_MEM_APACHE=$(ps -C "$APACHE_PROC" -o rss --no-headers 2>/dev/null | awk '{sum+=$1; ++n} END {if (n>0) printf "%.4f", sum/n/1024; else print 0}')
# Total RAM used by Apache specifically (MB)
TOTAL_RSS_APACHE=$(ps -C "$APACHE_PROC" -o rss --no-headers 2>/dev/null | awk '{sum+=$1} END {printf "%.2f", sum/1024}')
# Total RAM used by ALL processes (MB)
TOTAL_RSS_ALL=$(ps --no-headers -eo rss | awk '{sum+=$1} END {printf "%.2f", sum/1024}')
# Non-Apache memory usage (MB)
NON_APACHE_MEM=$(awk -v total="$TOTAL_RSS_ALL" -v apache="$TOTAL_RSS_APACHE" 'BEGIN {printf "%.2f", total - apache}')
# Total system memory (MB)
TOTAL_SYSTEM_RAM=$(free -m | awk '/^Mem:/ {print $2}')
# Memory available for Apache (MB)
MEM_AVAIL_APACHE=$(awk -v total="$TOTAL_SYSTEM_RAM" -v non="$NON_APACHE_MEM" 'BEGIN {printf "%.2f", total - non}')

# Calculate Optimal MaxRequestWorkers
if awk -v v="$AVG_MEM_APACHE" 'BEGIN {exit !(v>0)}'; then
    OPTIMAL_WORKERS=$(awk -v avail="$MEM_AVAIL_APACHE" -v avg="$AVG_MEM_APACHE" 'BEGIN {printf "%d", avail/avg}')
else
    OPTIMAL_WORKERS="N/A"
fi

echo "Average memory usage per Apache process: ${AVG_MEM_APACHE}MB"
NON_APACHE_GB=$(awk -v m="$NON_APACHE_MEM" 'BEGIN {printf "%.2f", m/1024}')
echo "Estimated memory usage by non-Apache processes: ${NON_APACHE_GB}GB"
MEM_AVAIL_GB=$(awk -v m="$MEM_AVAIL_APACHE" 'BEGIN {printf "%.2f", m/1024}')
echo "Memory available for Apache: ${MEM_AVAIL_GB}GB"
echo "Optimal MaxRequestWorkers: $OPTIMAL_WORKERS"

echo ""
echo "-------- ${bold}Performance Metrics${reset} -----------------------------"
echo ""

# Gather Apache (HTTPD) metrics
HTTPD_PS=$(ps aux | grep "$APACHE_PROC" | grep -v grep)
NUM_HTTPD=$(echo "$HTTPD_PS" | awk 'END {print (NR==1 && $0=="" ? 0 : NR)}')
HTTPD_CPU=$(echo "$HTTPD_PS" | awk '{sum+=$3} END {printf "%.1f", sum}')
HTTPD_MEM_KB=$(echo "$HTTPD_PS" | awk '{sum+=$6} END {printf "%d", sum}')
HTTPD_MEM_MB=$(awk -v kb="$HTTPD_MEM_KB" 'BEGIN {printf "%d", kb/1024}')

echo "${bold}Number of HTTPD Processes:${reset} $NUM_HTTPD"
echo "${bold}HTTPD CPU consumption:${reset} $HTTPD_CPU %"
echo "${bold}HTTPD memory consumption:${reset} $HTTPD_MEM_KB Kilobytes ($HTTPD_MEM_MB Megabytes)"

# CPU Cores
CORES_TOTAL=$(grep -c processor /proc/cpuinfo)
echo "${bold}Number of CPU Cores:${reset} $CORES_TOTAL"

# Gather PHP metrics (CGI, FPM, etc.)
PHP_PS=$(ps aux | grep -E "php|php-fpm|php-cgi" | grep -v grep)
NUM_PHP=$(echo "$PHP_PS" | awk 'END {print (NR==1 && $0=="" ? 0 : NR)}')
PHP_CPU=$(echo "$PHP_PS" | awk '{sum+=$3} END {printf "%.1f", sum}')
PHP_MEM_KB=$(echo "$PHP_PS" | awk '{sum+=$6} END {printf "%d", sum}')
PHP_MEM_MB=$(awk -v kb="$PHP_MEM_KB" 'BEGIN {printf "%d", kb/1024}')

echo "${bold}Number of PHP Processes:${reset} $NUM_PHP"
echo "${bold}PHP CPU consumption:${reset} $PHP_CPU %"
echo "${bold}PHP memory consumption:${reset} $PHP_MEM_KB Kilobytes ($PHP_MEM_MB Megabytes)"

echo ""

echo ""


echo "${blue}+===================================+${reset}"
echo "      ${bold}Checking MYSQL (mysqld)${reset}"
echo "${blue}+===================================+${reset}"

# Detect actual MYSQL process and service name
if pgrep -x "mysqld" >/dev/null; then
    MYSQL_PROC="mysqld"
    MYSQL_SVC="mysqld"
elif pgrep -x "MYSQL2" >/dev/null; then
    MYSQL_PROC="MYSQL2"
    MYSQL_SVC="MYSQL2"
else
    MYSQL_PROC="mysqld"
    MYSQL_SVC="mysqld"
fi

# MYSQL Status
if systemctl is-active "$MYSQL_SVC" >/dev/null 2>&1; then
    echo "MYSQL ($MYSQL_PROC) is ${green}running${reset}"
else
    echo "MYSQL ($MYSQL_PROC) is ${red}not running${reset}"
fi
echo "-----------------------------------"
echo ""

# MySQL Performance Metrics
if systemctl is-active "$MYSQL_SVC" >/dev/null 2>&1; then

    # Helper: run a MySQL query and return single value
    _my() { mysql -Nrs -e "$1" 2>/dev/null; }

    # ── Gather status & variables ────────────────────────────────────────────
    MYSQL_UPTIME=$(_my "SHOW GLOBAL STATUS LIKE 'Uptime';" | awk '{print $2}')
    MYSQL_QUESTIONS=$(_my "SHOW GLOBAL STATUS LIKE 'Questions';" | awk '{print $2}')
    MYSQL_CONNECTIONS=$(_my "SHOW GLOBAL STATUS LIKE 'Connections';" | awk '{print $2}')
    MYSQL_BYTES_SENT=$(_my "SHOW GLOBAL STATUS LIKE 'Bytes_sent';" | awk '{print $2}')
    MYSQL_BYTES_RECV=$(_my "SHOW GLOBAL STATUS LIKE 'Bytes_received';" | awk '{print $2}')
    MYSQL_COM_SELECT=$(_my "SHOW GLOBAL STATUS LIKE 'Com_select';" | awk '{print $2}')
    MYSQL_COM_WRITE=$(_my "SHOW GLOBAL STATUS LIKE 'Com_insert';" | awk '{print $2}')
    MYSQL_COM_UPDATE=$(_my "SHOW GLOBAL STATUS LIKE 'Com_update';" | awk '{print $2}')
    MYSQL_COM_DELETE=$(_my "SHOW GLOBAL STATUS LIKE 'Com_delete';" | awk '{print $2}')
    MYSQL_LOG_BIN=$(_my "SHOW GLOBAL VARIABLES LIKE 'log_bin';" | awk '{print $2}')
    MYSQL_MAX_CONN=$(_my "SHOW GLOBAL VARIABLES LIKE 'max_connections';" | awk '{print $2}')
    MYSQL_MAX_USED_CONN=$(_my "SHOW GLOBAL STATUS LIKE 'Max_used_connections';" | awk '{print $2}')
    MYSQL_ABORTED_CONN=$(_my "SHOW GLOBAL STATUS LIKE 'Aborted_connects';" | awk '{print $2}')
    MYSQL_THREAD_CACHE_SIZE=$(_my "SHOW GLOBAL VARIABLES LIKE 'thread_cache_size';" | awk '{print $2}')
    MYSQL_THREADS_CREATED=$(_my "SHOW GLOBAL STATUS LIKE 'Threads_created';" | awk '{print $2}')
    MYSQL_SLOW_QUERIES=$(_my "SHOW GLOBAL STATUS LIKE 'Slow_queries';" | awk '{print $2}')
    MYSQL_SORT_MERGE_PASSES=$(_my "SHOW GLOBAL STATUS LIKE 'Sort_merge_passes';" | awk '{print $2}')
    MYSQL_SELECT_FULL_JOIN=$(_my "SHOW GLOBAL STATUS LIKE 'Select_full_join';" | awk '{print $2}')
    MYSQL_TMP_DISK_TABLES=$(_my "SHOW GLOBAL STATUS LIKE 'Created_tmp_disk_tables';" | awk '{print $2}')
    MYSQL_TMP_TABLES=$(_my "SHOW GLOBAL STATUS LIKE 'Created_tmp_tables';" | awk '{print $2}')
    MYSQL_TABLE_OPEN_CACHE=$(_my "SHOW GLOBAL VARIABLES LIKE 'table_open_cache';" | awk '{print $2}')
    MYSQL_OPEN_TABLES=$(_my "SHOW GLOBAL STATUS LIKE 'Open_tables';" | awk '{print $2}')
    MYSQL_OPENED_TABLES=$(_my "SHOW GLOBAL STATUS LIKE 'Opened_tables';" | awk '{print $2}')
    MYSQL_TABLE_LOCKS_WAITED=$(_my "SHOW GLOBAL STATUS LIKE 'Table_locks_waited';" | awk '{print $2}')
    MYSQL_TABLE_LOCKS_IMMED=$(_my "SHOW GLOBAL STATUS LIKE 'Table_locks_immediate';" | awk '{print $2}')
    MYSQL_OPEN_FILES=$(_my "SHOW GLOBAL STATUS LIKE 'Open_files';" | awk '{print $2}')
    MYSQL_OPEN_FILES_LIMIT=$(_my "SHOW GLOBAL VARIABLES LIKE 'open_files_limit';" | awk '{print $2}')
    MYSQL_QUERY_CACHE_TYPE=$(_my "SHOW GLOBAL VARIABLES LIKE 'query_cache_type';" | awk '{print $2}')
    MYSQL_PS_MAX_MEM=$(_my "SELECT IFNULL(SUM(CURRENT_NUMBER_OF_BYTES_USED),0) FROM performance_schema.memory_summary_global_by_event_name WHERE EVENT_NAME LIKE 'memory/performance_schema/%';" 2>/dev/null || echo 0)

    # Buffer variables (matching mysqltuner.pl calculation logic)
    MYSQL_INNODB_BUFFER=$(_my "SHOW GLOBAL VARIABLES LIKE 'innodb_buffer_pool_size';" | awk '{print $2}')
    MYSQL_INNODB_LOG_BUFFER=$(_my "SHOW GLOBAL VARIABLES LIKE 'innodb_log_buffer_size';" | awk '{print $2}')
    MYSQL_KEY_BUFFER=$(_my "SHOW GLOBAL VARIABLES LIKE 'key_buffer_size';" | awk '{print $2}')
    MYSQL_QUERY_CACHE_SIZE=$(_my "SHOW GLOBAL VARIABLES LIKE 'query_cache_size';" | awk '{print $2}')
    MYSQL_SORT_BUFFER=$(_my "SHOW GLOBAL VARIABLES LIKE 'sort_buffer_size';" | awk '{print $2}')
    MYSQL_READ_BUFFER=$(_my "SHOW GLOBAL VARIABLES LIKE 'read_buffer_size';" | awk '{print $2}')
    MYSQL_READ_RND_BUFFER=$(_my "SHOW GLOBAL VARIABLES LIKE 'read_rnd_buffer_size';" | awk '{print $2}')
    MYSQL_JOIN_BUFFER=$(_my "SHOW GLOBAL VARIABLES LIKE 'join_buffer_size';" | awk '{print $2}')
    MYSQL_THREAD_STACK=$(_my "SHOW GLOBAL VARIABLES LIKE 'thread_stack';" | awk '{print $2}')
    MYSQL_TMP_TABLE_SIZE=$(_my "SHOW GLOBAL VARIABLES LIKE 'tmp_table_size';" | awk '{print $2}')
    MYSQL_MAX_HEAP_TABLE=$(_my "SHOW GLOBAL VARIABLES LIKE 'max_heap_table_size';" | awk '{print $2}')
    MYSQL_BINLOG_CACHE=$(_my "SHOW GLOBAL VARIABLES LIKE 'binlog_cache_size';" | awk '{print $2}')
    # MariaDB: Aria storage engine page cache (added to global bufs like mysqltuner.pl)
    MYSQL_ARIA_BUFFER=$(_my "SHOW GLOBAL VARIABLES LIKE 'aria_pagecache_buffer_size';" | awk '{print $2}')
    # Table cache hit/miss counters (MySQL 5.6+ / MariaDB 10.0+)
    MYSQL_TBL_CACHE_HITS=$(_my "SHOW GLOBAL STATUS LIKE 'Table_open_cache_hits';" | awk '{print $2}')
    MYSQL_TBL_CACHE_MISSES=$(_my "SHOW GLOBAL STATUS LIKE 'Table_open_cache_misses';" | awk '{print $2}')

    # ── AWK-based calculations ───────────────────────────────────────────────
    eval "$(awk -v uptime="${MYSQL_UPTIME:-1}" \
               -v questions="${MYSQL_QUESTIONS:-0}" \
               -v connections="${MYSQL_CONNECTIONS:-1}" \
               -v bytes_sent="${MYSQL_BYTES_SENT:-0}" \
               -v bytes_recv="${MYSQL_BYTES_RECV:-0}" \
               -v com_select="${MYSQL_COM_SELECT:-0}" \
               -v com_write="${MYSQL_COM_WRITE:-0}" \
               -v com_update="${MYSQL_COM_UPDATE:-0}" \
               -v com_delete="${MYSQL_COM_DELETE:-0}" \
               -v slow="${MYSQL_SLOW_QUERIES:-0}" \
               -v max_conn="${MYSQL_MAX_CONN:-151}" \
               -v max_used="${MYSQL_MAX_USED_CONN:-0}" \
               -v aborted="${MYSQL_ABORTED_CONN:-0}" \
               -v threads_created="${MYSQL_THREADS_CREATED:-0}" \
               -v tmp_disk="${MYSQL_TMP_DISK_TABLES:-0}" \
               -v tmp_total="${MYSQL_TMP_TABLES:-0}" \
               -v open_tables="${MYSQL_OPEN_TABLES:-0}" \
               -v opened_tables="${MYSQL_OPENED_TABLES:-1}" \
               -v tbl_locks_immed="${MYSQL_TABLE_LOCKS_IMMED:-0}" \
               -v tbl_locks_waited="${MYSQL_TABLE_LOCKS_WAITED:-0}" \
               -v open_files="${MYSQL_OPEN_FILES:-0}" \
               -v open_files_limit="${MYSQL_OPEN_FILES_LIMIT:-1}" \
               -v innodb_buf="${MYSQL_INNODB_BUFFER:-134217728}" \
               -v innodb_log_buf="${MYSQL_INNODB_LOG_BUFFER:-8388608}" \
               -v key_buf="${MYSQL_KEY_BUFFER:-8388608}" \
               -v query_cache_size="${MYSQL_QUERY_CACHE_SIZE:-0}" \
               -v sort_buf="${MYSQL_SORT_BUFFER:-262144}" \
               -v read_buf="${MYSQL_READ_BUFFER:-131072}" \
               -v read_rnd_buf="${MYSQL_READ_RND_BUFFER:-262144}" \
               -v join_buf="${MYSQL_JOIN_BUFFER:-262144}" \
               -v thread_stack="${MYSQL_THREAD_STACK:-1048576}" \
               -v tmp_tbl_size="${MYSQL_TMP_TABLE_SIZE:-16777216}" \
               -v max_heap_tbl="${MYSQL_MAX_HEAP_TABLE:-16777216}" \
               -v binlog_cache="${MYSQL_BINLOG_CACHE:-32768}" \
               -v aria_buf="${MYSQL_ARIA_BUFFER:-0}" \
               -v tbl_cache_hits="${MYSQL_TBL_CACHE_HITS:-0}" \
               -v tbl_cache_misses="${MYSQL_TBL_CACHE_MISSES:-0}" \
               -v phys_mem="${TOTAL_SYSTEM_RAM:-1024}" \
               'function hr(b) {
            if (b >= 1073741824) return sprintf("%.1fG", b/1073741824)
            if (b >= 1048576)    return sprintf("%.1fM", b/1048576)
            if (b >= 1024)       return sprintf("%.1fK", b/1024)
            return b "B"
        }
        BEGIN {
        phys_bytes = phys_mem * 1048576

        # uptime pretty
        d=int(uptime/86400); h=int((uptime%86400)/3600); m=int((uptime%3600)/60); s=uptime%60
        if (d>0) up_str=d"d "h"h "m"m "s"s"
        else if (h>0) up_str=h"h "m"m "s"s"
        else if (m>0) up_str=m"m "s"s"
        else up_str=s"s"

        qps = (uptime > 0) ? questions/uptime : 0
        writes = com_write + com_update + com_delete
        reads  = com_select
        total_rw = reads + writes
        pct_read  = (total_rw > 0) ? int(reads*100/total_rw) : 100
        pct_write = (total_rw > 0) ? int(writes*100/total_rw) : 0

        # memory — mirrors mysqltuner.pl server_buffers / per_thread_buffers
        # global: key_buf + innodb_buf + innodb_log_buf + query_cache + max_tmp + aria (MariaDB)
        max_tmp = (tmp_tbl_size < max_heap_tbl) ? tmp_tbl_size : max_heap_tbl
        global_bufs  = key_buf + innodb_buf + innodb_log_buf + query_cache_size + max_tmp + aria_buf
        # per-thread: read + read_rnd + sort + thread_stack + join + binlog_cache (NO tmp_table_size)
        per_thread   = read_buf + read_rnd_buf + sort_buf + thread_stack + join_buf + binlog_cache
        max_possible = global_bufs + (per_thread * max_conn)

        pct_phys     = (phys_bytes > 0) ? max_possible*100/phys_bytes : 0
        peak_mem     = global_bufs + (per_thread * max_used)
        pct_peak     = (phys_bytes > 0) ? peak_mem*100/phys_bytes : 0

        # slow queries
        pct_slow = (questions > 0) ? slow*100/questions : 0

        # connections
        pct_conn = (max_conn > 0) ? max_used*100/max_conn : 0

        # aborted
        pct_abort = (connections > 0) ? aborted*100/connections : 0

        # temp tables
        pct_tmp_disk = (tmp_total > 0) ? tmp_disk*100/tmp_total : 0

        # thread cache
        pct_thread_cache = (connections > 0) ? (1 - threads_created/connections)*100 : 100
        if (pct_thread_cache < 0) pct_thread_cache = 0

        # table cache — use Table_open_cache_hits/misses if available (MySQL 5.6+)
        total_tbl_requests = tbl_cache_hits + tbl_cache_misses
        if (total_tbl_requests > 0)
            pct_table_cache = int(tbl_cache_hits * 100 / total_tbl_requests)
        else
            pct_table_cache = (opened_tables > 0) ? int(open_tables * 100 / opened_tables) : 100

        # table locks
        total_locks = tbl_locks_immed + tbl_locks_waited
        pct_locks_immed = (total_locks > 0) ? tbl_locks_immed*100/total_locks : 100

        # open files
        pct_open_files = (open_files_limit > 0) ? open_files*100/open_files_limit : 0

        # output as shell variable assignments
        printf "MYSQL_UP_STR=\"%s\"\n", up_str
        printf "MYSQL_QPS=\"%.3f\"\n", qps
        printf "MYSQL_PCT_READ=%d\n", pct_read
        printf "MYSQL_PCT_WRITE=%d\n", pct_write
        printf "MYSQL_GLOBAL_BUFS=\"%s\"\n", hr(global_bufs)
        printf "MYSQL_PER_THREAD=\"%s\"\n", hr(per_thread)
        printf "MYSQL_TBL_CACHE_HITS_TOT=%d\n", total_tbl_requests
        printf "MYSQL_TBL_CACHE_HITS_VAL=%d\n", tbl_cache_hits
        printf "MYSQL_PEAK_MEM=\"%s\"\n", hr(peak_mem)
        printf "MYSQL_PCT_PEAK=\"%.2f\"\n", pct_peak
        printf "MYSQL_MAX_POSSIBLE=\"%s\"\n", hr(max_possible)
        printf "MYSQL_PCT_PHYS=\"%.2f\"\n", pct_phys
        printf "MYSQL_PHYS_HR=\"%s\"\n", hr(phys_bytes)
        printf "MYSQL_PCT_SLOW=\"%.0f\"\n", pct_slow
        printf "MYSQL_PCT_CONN=\"%.0f\"\n", pct_conn
        printf "MYSQL_PCT_ABORT=\"%.2f\"\n", pct_abort
        printf "MYSQL_PCT_TMP_DISK=\"%.0f\"\n", pct_tmp_disk
        printf "MYSQL_PCT_THREAD_CACHE=\"%.0f\"\n", pct_thread_cache
        printf "MYSQL_PCT_TABLE_CACHE=\"%.0f\"\n", pct_table_cache
        printf "MYSQL_PCT_LOCKS_IMMED=\"%.0f\"\n", pct_locks_immed
        printf "MYSQL_TOTAL_LOCKS=%d\n", total_locks
        printf "MYSQL_PCT_OPEN_FILES=\"%.0f\"\n", pct_open_files
    }')"

    # ── Icon helpers ─────────────────────────────────────────────────────────
    good_icon="\e[0;32m✔\e[0m"
    bad_icon="\e[0;31m✘\e[0m"
    info_icon="\e[0;34mℹ\e[0m"

    echo ""
    echo "-------- ${bold}MySQL Performance Metrics${reset} -------------------------------------------------------------------"

    # Uptime / QPS line
    _TX=$(numfmt --to=iec ${MYSQL_BYTES_SENT:-0} 2>/dev/null || echo "${MYSQL_BYTES_SENT}B")
    _RX=$(numfmt --to=iec ${MYSQL_BYTES_RECV:-0} 2>/dev/null || echo "${MYSQL_BYTES_RECV}B")
    echo -e " $info_icon  Server up: ${MYSQL_UP_STR} | Queries (q): ${MYSQL_QUESTIONS} total (avg ${MYSQL_QPS} queries/sec (qps)) | Connections (conn): ${MYSQL_CONNECTIONS} total"
    echo -e "             Sent (TX): ${_TX} | Received (RX): ${_RX}"
    echo -e " $info_icon  Workload: ${MYSQL_PCT_READ}% reads (SELECT) / ${MYSQL_PCT_WRITE}% writes (INSERT + UPDATE + DELETE)"

    # Binary logging
    if [[ "${MYSQL_LOG_BIN,,}" == "on" || "${MYSQL_LOG_BIN,,}" == "1" ]]; then
        echo -e " $info_icon  Binary logging is enabled"
    else
        echo -e " $info_icon  Binary logging is disabled"
    fi

    # Memory info
    echo -e " $info_icon  Physical Memory     : ${MYSQL_PHYS_HR}"
    echo -e " $info_icon  Max MySQL memory    : ${MYSQL_MAX_POSSIBLE}"
    echo -e " $info_icon  Total buffers: ${MYSQL_GLOBAL_BUFS} global + ${MYSQL_PER_THREAD} per thread (${MYSQL_MAX_CONN} max threads)"

    # Performance schema memory
    PS_MEM_HR=$(numfmt --to=iec "${MYSQL_PS_MAX_MEM:-0}" 2>/dev/null || echo "0B")
    echo -e " $info_icon  Performance_schema Max memory usage: ${PS_MEM_HR}"

    # Peak memory
    if (( $(echo "${MYSQL_PCT_PEAK:-0} >= 85" | bc -l) )); then
        echo -e " $bad_icon  Maximum reached memory usage: ${MYSQL_PEAK_MEM} (${MYSQL_PCT_PEAK}% of installed RAM)"
    else
        echo -e " $good_icon  Maximum reached memory usage: ${MYSQL_PEAK_MEM} (${MYSQL_PCT_PEAK}% of installed RAM)"
    fi

    # Max possible memory
    if (( $(echo "${MYSQL_PCT_PHYS:-0} >= 85" | bc -l) )); then
        echo -e " $bad_icon  Maximum possible memory usage: ${MYSQL_MAX_POSSIBLE} (${MYSQL_PCT_PHYS}% of installed RAM)"
    else
        echo -e " $good_icon  Maximum possible memory usage: ${MYSQL_MAX_POSSIBLE} (${MYSQL_PCT_PHYS}% of installed RAM)"
    fi

    # Slow queries
    if [[ "$MYSQL_PCT_SLOW" -gt 5 ]]; then
        echo -e " $bad_icon  Slow queries: ${MYSQL_PCT_SLOW}% (${MYSQL_SLOW_QUERIES}/${MYSQL_QUESTIONS})"
    else
        echo -e " $good_icon  Slow queries: ${MYSQL_PCT_SLOW}% (${MYSQL_SLOW_QUERIES}/${MYSQL_QUESTIONS})"
    fi

    # Connection usage
    if [[ "$MYSQL_PCT_CONN" -gt 85 ]]; then
        echo -e " $bad_icon  Highest usage of available connections: ${MYSQL_PCT_CONN}% (${MYSQL_MAX_USED_CONN}/${MYSQL_MAX_CONN})"
    else
        echo -e " $good_icon  Highest usage of available connections: ${MYSQL_PCT_CONN}% (${MYSQL_MAX_USED_CONN}/${MYSQL_MAX_CONN})"
    fi

    # Aborted connections
    if (( $(echo "${MYSQL_PCT_ABORT:-0} >= 3" | bc -l) )); then
        echo -e " $bad_icon  Aborted connections: ${MYSQL_PCT_ABORT}% (${MYSQL_ABORTED_CONN}/${MYSQL_CONNECTIONS})"
    else
        echo -e " $good_icon  Aborted connections: ${MYSQL_PCT_ABORT}% (${MYSQL_ABORTED_CONN}/${MYSQL_CONNECTIONS})"
    fi

    # Query cache
    if [[ "${MYSQL_QUERY_CACHE_TYPE,,}" == "off" || "${MYSQL_QUERY_CACHE_TYPE,,}" == "0" ]]; then
        echo -e " $good_icon  Query cache is disabled by default due to mutex contention on multiprocessor machines."
    else
        echo -e " $info_icon  Query cache type: ${MYSQL_QUERY_CACHE_TYPE}"
    fi

    # Sort requiring temp tables
    if [[ "${MYSQL_SORT_MERGE_PASSES:-0}" -gt 0 ]]; then
        echo -e " $bad_icon  Sorts requiring temporary tables: ${MYSQL_SORT_MERGE_PASSES} sort merge passes"
    else
        echo -e " $good_icon  No Sort requiring temporary tables"
    fi

    # Joins without indexes
    if [[ "${MYSQL_SELECT_FULL_JOIN:-0}" -gt 0 ]]; then
        echo -e " $bad_icon  Joins without indexes: ${MYSQL_SELECT_FULL_JOIN} full join(s) detected"
    else
        echo -e " $good_icon  No joins without indexes"
    fi

    # Temp tables on disk
    TMP_TOTAL="${MYSQL_TMP_TABLES:-0}"
    TMP_DISK="${MYSQL_TMP_DISK_TABLES:-0}"
    if [[ "$MYSQL_PCT_TMP_DISK" -gt 25 ]]; then
        echo -e " $bad_icon  Temporary tables created on disk: ${MYSQL_PCT_TMP_DISK}% (${TMP_DISK} on disk / ${TMP_TOTAL} total)"
    else
        echo -e " $good_icon  Temporary tables created on disk: ${MYSQL_PCT_TMP_DISK}% (${TMP_DISK} on disk / ${TMP_TOTAL} total)"
    fi

    # Thread cache hit rate
    if [[ "$MYSQL_PCT_THREAD_CACHE" -lt 50 ]]; then
        echo -e " $bad_icon  Thread cache hit rate: ${MYSQL_PCT_THREAD_CACHE}% (${MYSQL_THREADS_CREATED} created / ${MYSQL_CONNECTIONS} connections)"
    else
        echo -e " $good_icon  Thread cache hit rate: ${MYSQL_PCT_THREAD_CACHE}% (${MYSQL_THREADS_CREATED} created / ${MYSQL_CONNECTIONS} connections)"
    fi

    # Table cache hit rate
    if [[ "$MYSQL_PCT_TABLE_CACHE" -lt 20 ]]; then
        echo -e " $bad_icon  Table cache hit rate: ${MYSQL_PCT_TABLE_CACHE}% ($(numfmt --to=si ${MYSQL_TBL_CACHE_HITS_VAL:-0} 2>/dev/null || echo ${MYSQL_TBL_CACHE_HITS_VAL:-0}) hits / $(numfmt --to=si ${MYSQL_TBL_CACHE_HITS_TOT:-0} 2>/dev/null || echo ${MYSQL_TBL_CACHE_HITS_TOT:-0}) requests)"
    else
        echo -e " $good_icon  Table cache hit rate: ${MYSQL_PCT_TABLE_CACHE}% ($(numfmt --to=si ${MYSQL_TBL_CACHE_HITS_VAL:-0} 2>/dev/null || echo ${MYSQL_TBL_CACHE_HITS_VAL:-0}) hits / $(numfmt --to=si ${MYSQL_TBL_CACHE_HITS_TOT:-0} 2>/dev/null || echo ${MYSQL_TBL_CACHE_HITS_TOT:-0}) requests)"
    fi

    # Open file limit
    OFL_LIMIT_HR=$(awk "BEGIN{printf \"%dK\", ${MYSQL_OPEN_FILES_LIMIT:-0}/1024}" 2>/dev/null || echo "${MYSQL_OPEN_FILES_LIMIT}")
    if [[ "$MYSQL_PCT_OPEN_FILES" -gt 85 ]]; then
        echo -e " $bad_icon  Open file limit used: ${MYSQL_PCT_OPEN_FILES}% (${MYSQL_OPEN_FILES}/${OFL_LIMIT_HR})"
    else
        echo -e " $good_icon  Open file limit used: ${MYSQL_PCT_OPEN_FILES}% (${MYSQL_OPEN_FILES}/${OFL_LIMIT_HR})"
    fi

    # Table locks
    TOTAL_LOCKS="${MYSQL_TOTAL_LOCKS:-0}"
    if [[ "$MYSQL_PCT_LOCKS_IMMED" -lt 95 && "$TOTAL_LOCKS" -gt 0 ]]; then
        echo -e " $bad_icon  Table locks acquired immediately: ${MYSQL_PCT_LOCKS_IMMED}% (${MYSQL_TABLE_LOCKS_IMMED} immediate / ${TOTAL_LOCKS} locks)"
    else
        echo -e " $good_icon  Table locks acquired immediately: ${MYSQL_PCT_LOCKS_IMMED}% (${MYSQL_TABLE_LOCKS_IMMED} immediate / ${TOTAL_LOCKS} locks)"
    fi

    unset _my
fi

echo ""
echo ""






# MySQL Recommendations
if systemctl is-active "$MYSQL_SVC" >/dev/null 2>&1; then

    echo ""
    echo "-------- ${bold}MySQL Recommendations${reset} -----------------------------------------------------------------------"

    # ── Locate the MySQL error log ────────────────────────────────────────────
    # Step 1: ask MySQL directly (SHOW VARIABLES LIKE 'log_error%' returns log_error + log_error_verbosity)
    MYSQL_ERR_LOG=$(mysql -Nrs -e "SHOW VARIABLES LIKE 'log_error%';" 2>/dev/null \
        | awk '$1 == "log_error" {print $2}')
    # Step 2: fallback — scan datadir for the first .err file
    if [[ -z "$MYSQL_ERR_LOG" || ! -f "$MYSQL_ERR_LOG" ]]; then
        MYSQL_DATADIR=$(mysql -Nrs -e "SHOW VARIABLES LIKE 'datadir';" 2>/dev/null | awk '{print $2}')
        MYSQL_ERR_LOG=$(find "${MYSQL_DATADIR:-/var/lib/mysql}" -maxdepth 1 -name "*.err" 2>/dev/null | head -1)
    fi

    # ── 1. Error log errors ───────────────────────────────────────────────────
    echo ""
    echo -e " ${bold}[1] Error Log: ${MYSQL_ERR_LOG:-not found}${reset}"
    if [[ -n "$MYSQL_ERR_LOG" && -f "$MYSQL_ERR_LOG" ]]; then
        ERR_COUNT=$(grep -ic "error" "$MYSQL_ERR_LOG" 2>/dev/null || echo 0)
        if [[ "$ERR_COUNT" -gt 0 ]]; then
            echo -e " \e[0;31m✘\e[0m  ${ERR_COUNT} error line(s) found — last 5:"
            grep -i "error" "$MYSQL_ERR_LOG" 2>/dev/null \
                | grep -v "^#" \
                | tail -5 \
                | while IFS= read -r line; do
                    echo -e "     \e[0;33m→\e[0m $line"
                  done
        else
            echo -e " \e[0;32m✔\e[0m  No errors found in error log."
        fi
    else
        echo -e " \e[0;34mℹ\e[0m  MySQL error log not accessible or not found."
    fi

    # ── 2. Crashed tables ────────────────────────────────────────────────────
    echo ""
    echo -e " ${bold}[2] Crashed Tables${reset}"
    if [[ -n "$MYSQL_ERR_LOG" && -f "$MYSQL_ERR_LOG" ]]; then
        CRASHED_TABLES=$(tail -1000 "$MYSQL_ERR_LOG" 2>/dev/null \
            | grep -i "marked as crashed" \
            | awk '{print $7}' \
            | awk -F'/' '{print $2}' \
            | sort -u)
        if [[ -n "$CRASHED_TABLES" ]]; then
            echo -e " \e[0;31m✘\e[0m  Crashed table(s) found in error log (last 1000 lines):"
            echo "$CRASHED_TABLES" | head -20 | while IFS= read -r tbl; do
                echo -e "     \e[0;33m→\e[0m ${tbl}  — to repair: ${bold}REPAIR TABLE \`${tbl}\`;${reset}"
            done
        else
            echo -e " \e[0;32m✔\e[0m  No crashed tables found in last 1000 lines of error log."
        fi
    else
        echo -e " \e[0;34mℹ\e[0m  Cannot check for crashed tables — error log not accessible."
    fi

    # ── 3. Table locking analysis ─────────────────────────────────────────────
    echo ""
    echo -e " ${bold}[3] Table Locking Analysis${reset}"
    _TLW=$(mysql -Nrs -e "SHOW GLOBAL STATUS LIKE 'Table_locks_waited';"  2>/dev/null | awk '{print $2}')
    _TLI=$(mysql -Nrs -e "SHOW GLOBAL STATUS LIKE 'Table_locks_immediate';" 2>/dev/null | awk '{print $2}')
    _TLW=${_TLW:-0}; _TLI=${_TLI:-0}

    if [[ "$_TLW" -gt 0 ]]; then
        _LOCK_RATIO=$(( _TLI / _TLW ))
        echo -e " \e[0;34mℹ\e[0m  Lock waits (Table_locks_waited): $_TLW"
        echo -e " \e[0;34mℹ\e[0m  Immediate locks (Table_locks_immediate): $_TLI"
        echo -e " \e[0;34mℹ\e[0m  Current lock wait ratio: 1 : ${_LOCK_RATIO}"
        if [[ "$_LOCK_RATIO" -lt 5000 ]]; then
            echo -e " \e[0;31m✘\e[0m  High lock contention detected! Consider:"
            echo -e "     \e[0;33m→\e[0m Converting frequently-locked MyISAM tables to InnoDB"
            echo -e "     \e[0;33m→\e[0m Setting ${bold}low_priority_updates=1${reset} if you have long SELECT queries with frequent writes"
            echo -e "     \e[0;33m→\e[0m Setting ${bold}concurrent_insert=ALWAYS${reset} for high-concurrency INSERT workloads"
        else
            echo -e " \e[0;32m✔\e[0m  Table locking is fine (1 wait per ${_LOCK_RATIO} immediate locks — ratio is healthy)"
        fi
    else
        echo -e " \e[0;32m✔\e[0m  No table lock waits recorded — locking is not a concern."
        echo -e " \e[0;34mℹ\e[0m  Immediate locks (Table_locks_immediate): $_TLI"
    fi

    echo ""
    echo "-----------------------------------"

fi

echo ""
echo ""

#==================================Step 4==================================
echo "${blue}+===================================+${reset}"
echo "   ${bold}Out Of Memory (OOM) Analysis${reset}"
echo "${blue}+===================================+${reset}"

# Get the last event's date to use as an anchor
LAST_OOM_DATE=$(grep -i "Out of memory" /var/log/messages 2>/dev/null | tail -n 1 | awk '{print $1, $2, $3}')

if [ -n "$LAST_OOM_DATE" ]; then
    # Extract reports for that specific date/timestamp
    OOM_REPORTS=$(grep "$LAST_OOM_DATE" /var/log/messages 2>/dev/null | awk '{sub(/.*kernel: /,""); print}')
    OOM_COUNT=$(grep -i "Out of memory" /var/log/messages 2>/dev/null | wc -l)

    if [[ -n "$OOM_REPORTS" ]]; then
        echo "${red}${bold}OOM killer has been invoked $OOM_COUNT time(s) recently${reset}"
        echo "${yellow}---------------------------------------------------------------------${reset}"
        echo "${bold}Filtering last OOM event on:${reset} $LAST_OOM_DATE"
        echo ""

        # 1. Process the OOM text using AWK (Fixes array clearing and numeric sorting)
        RAW_STATS=$(echo "$OOM_REPORTS" | awk '
            /Tasks state/ { 
                # Safely reset arrays for multiple OOM events (keeps only the most recent crash)
                for (i in user_rss) delete user_rss[i]; 
                for (i in proc_rss) delete proc_rss[i]; 
                for (i in proc_count) delete proc_count[i]; 
            }
            {
                line = $0; 
                gsub(/\[|\]/, "", line); # Strip brackets
                n = split(line, a);
                
                # Ensure it is a valid process line with numbers in the right columns
                if (n >= 9 && a[1] ~ /^[0-9]+$/ && a[2] ~ /^[0-9]+$/ && a[5] ~ /^[0-9]+$/) {
                    uid = a[2]; 
                    rss = a[5]; 
                    name = a[9];
                    # Catch multi-word process names (like "spamd child")
                    for(i=10; i<=n; i++) name = name " " a[i];
                    
                    user_rss[uid] += rss;
                    proc_rss[uid SUBSEP name] += rss;
                    proc_count[uid SUBSEP name]++;
                }
            }
            END {
                # Find the single user consuming the most total RAM
                max_rss = -1; top_uid = "None";
                for (u in user_rss) {
                    if (user_rss[u] > max_rss) { max_rss = user_rss[u]; top_uid = u; }
                }
                print "TOP_UID:" top_uid;
                
                # Print processes with pure numbers in the first column for accurate sorting
                for (idx in proc_rss) {
                    split(idx, p, SUBSEP);
                    # Convert RAM Pages to Megabytes: (pages * 4 / 1024)
                    mb = (proc_rss[idx] * 4) / 1024;
                    printf "%.2f|%s|%s|%d\n", mb, p[1], p[2], proc_count[idx];
                }
            }
        ')

        # 2. Extract the top user variable
        TOP_USER_UID=$(echo "$RAW_STATS" | grep "^TOP_UID:" | cut -d':' -f2)
        TOP_USER_NAME=$(id -nu "$TOP_USER_UID" 2>/dev/null || echo "UID:$TOP_USER_UID")

        # 3. Print the formatted output
        echo "${yellow}=====================================================================${reset}"
        echo "${bold}USER CONSUMING MOST RAM: ${red}${TOP_USER_NAME}${reset} (UID: $TOP_USER_UID)"
        echo "${yellow}=====================================================================${reset}"
        printf "${bold}%-15s %-25s %-10s %s${reset}\n" "USER" "PROCESS NAME" "COUNT" "RAM USED"
        echo "${yellow}---------------------------------------------------------------------${reset}"

        # Filter out the TOP_UID line, sort numerically by column 1, and print top 15
        echo "$RAW_STATS" | grep -v "^TOP_UID:" | sort -t'|' -k1,1nr | head -n 15 | while IFS='|' read -r mb uid name count; do
            username=$(id -nu "$uid" 2>/dev/null || echo "$uid")
            printf "%-15s %-25s %-10s %s MB\n" "$username" "$name" "$count" "$mb"
        done

        #==================================Step 5==================================
        echo ""
        echo "${blue}+===================================+${reset}"
        echo "      ${bold}Step 5: Logs Reports${reset}"
        echo "${blue}+===================================+${reset}"

        # Section: Top 5 domains by request count (Today)
        echo "${bold}Top 5 Domains by Request Count (Today):${reset}"
        printf "${bold}%-55s %s${reset}\n" "DOMAIN" "REQUESTS"
        echo "${yellow}---------------------------------------------------------------------${reset}"

        TOP5_DOMAINS=""
        if [ -d "/usr/local/apache/domlogs" ]; then
            TOP5_DOMAINS=$(for i in $(ls /usr/local/apache/domlogs/*/* 2>/dev/null); do
                grep "$(date +%d/%b)" "$i" 2>/dev/null | awk '{print $1}' | wc -l | xargs echo "$i" | cut -d '/' -f '7-'
            done | sort -nrk2 | head -5)

            if [[ -n "$TOP5_DOMAINS" ]]; then
                echo "$TOP5_DOMAINS" | while IFS= read -r line; do
                    dom=$(echo "$line" | awk '{print $1}')
                    cnt=$(echo "$line" | awk '{print $2}')
                    printf "%-55s %s\n" "$dom" "$cnt"
                done
            else
                echo "No domain log activity found for today."
            fi
        else
            echo "Apache domlogs directory not found."
        fi
        echo "${yellow}---------------------------------------------------------------------${reset}"

        # Section: Top Bot Hits for the top 5 domains
        if [[ -n "$TOP5_DOMAINS" ]]; then
            DAYSAUDIT=2
            NUMRESULTS=5

            echo ""
            echo "${bold}Top Bot Hits for Top 5 Domains (Last $DAYSAUDIT days):${reset}"
            echo "${yellow}---------------------------------------------------------------------${reset}"

            # Date regex construction
            SCAN_DATE=""
            for d in $(seq 1 "$DAYSAUDIT"); do
                SCAN_DATE+=$(date -d "$d days ago" +"\[%d/%b/%Y:|")
            done
            SCAN_DATE=${SCAN_DATE/%|/}

            echo "$TOP5_DOMAINS" | while IFS= read -r line; do
                DOMAIN_FILE=$(echo "$line" | awk '{print $1}')
                DOMAIN_HITS=$(echo "$line" | awk '{print $2}')
                # Strip the log suffix to get the domain name (e.g. domain.com-ssl_log -> domain.com)
                DOMAIN_NAME=$(echo "$DOMAIN_FILE" | sed 's/-ssl_log$//' | sed 's/_log$//')
                # Determine the owning cPanel user from the log path or userdatadomains
                DOMAIN_USER=$(grep "^${DOMAIN_NAME}:" /etc/userdatadomains 2>/dev/null | head -1 | awk -F'==' '{print $1}' | awk -F': ' '{print $2}' | awk '{print $1}')

                echo ""
                echo "${bold}Domain:${reset} $DOMAIN_FILE (${DOMAIN_HITS} hits)"

                if [[ -n "$DOMAIN_USER" ]]; then
                    DOM_HOME=$(getent passwd "$DOMAIN_USER" 2>/dev/null | cut -d: -f6)
                    LOGS_DIR="${DOM_HOME}/logs"
                    ACCLOGS_DIR="${DOM_HOME}/access-logs"

                    CURR_M=$(date +"%b"); CURR_Y=$(date +"%Y")
                    PREV_M=$(date -d "$(date +%Y-%m-15) -1 month" +"%b"); PREV_Y=$(date -d "$(date +%Y-%m-15) -1 month" +"%Y")

                    BOT_OUTPUT=$(zcat -f "${LOGS_DIR}/${DOMAIN_NAME}-${CURR_M}-${CURR_Y}.gz" \
                         "${LOGS_DIR}/${DOMAIN_NAME}-ssl_log-${CURR_M}-${CURR_Y}.gz" \
                         "${LOGS_DIR}/${DOMAIN_NAME}-${PREV_M}-${PREV_Y}.gz" \
                         "${LOGS_DIR}/${DOMAIN_NAME}-ssl_log-${PREV_M}-${PREV_Y}.gz" \
                         "${ACCLOGS_DIR}/${DOMAIN_NAME}" \
                         "${ACCLOGS_DIR}/${DOMAIN_NAME}-ssl_log" 2>/dev/null | \
                    grep -Eh "${SCAN_DATE}" 2>/dev/null | \
                    awk -F\" '{print $6}' | \
                    grep -Ei "bot|crawl" | \
                    sed 's/.*http/http/' | \
                    sort -n | uniq -c | sort -n -k1 -fr | \
                    sed 's/)$//g' | head -"$NUMRESULTS")

                    if [[ -n "$BOT_OUTPUT" ]]; then
                        echo "$BOT_OUTPUT"
                    else
                        echo "  No bot/crawler activity detected."
                    fi
                else
                    echo "  Could not determine domain owner — skipping bot check."
                fi
            done
        fi
        # Section: Log Activity Prior to OOM (10-minute window)
        if [[ -n "$TOP_USER_NAME" && "$TOP_USER_NAME" != "Unknown" ]]; then
            BASE_HOME=$(getent passwd "$TOP_USER_NAME" 2>/dev/null | cut -d: -f6)
            if [[ -d "$BASE_HOME" ]]; then
                ACCLOGS_DIR="${BASE_HOME}/access-logs"
                USER_DOMS=$(grep "$TOP_USER_NAME" /etc/userdatadomains 2>/dev/null | cut -d: -f1)

                echo ""
                echo "${bold}Log Activity 10 Minutes Prior to OOM ($LAST_OOM_DATE):${reset}"
            
                # Parse OOM Date to epoch
                OOM_EPOCH=$(date -d "$LAST_OOM_DATE" +%s 2>/dev/null)
                if [ -n "$OOM_EPOCH" ]; then
                    # Build regex for the 10 preceding minutes
                    OOM_REGEX=""
                    for i in {0..10}; do
                        T=$(($OOM_EPOCH - $i*60))
                        OOM_REGEX+=$(date -d "@$T" +"%d/%b/%Y:%H:%M|")
                    done
                    OOM_REGEX=${OOM_REGEX%|}
                
                    # Build list of active logs to scan
                    LOGS_TO_SCAN=""
                    for DOMAIN in $USER_DOMS; do
                        LOGS_TO_SCAN+=" ${ACCLOGS_DIR}/${DOMAIN} ${ACCLOGS_DIR}/${DOMAIN}-ssl_log"
                    done
                
                    # Check if logs exist
                    VALID_LOGS=""
                    for L in $LOGS_TO_SCAN; do
                        if [ -f "$L" ]; then VALID_LOGS+=" $L"; fi
                    done
                
                    if [ -n "$VALID_LOGS" ]; then
                        # Extract the 10 minute window into a temp variable to avoid multiple disk reads
                        TARGET_TRAFFIC=$(grep -Eh "$OOM_REGEX" $VALID_LOGS 2>/dev/null)
                    
                        if [ -n "$TARGET_TRAFFIC" ]; then
                            echo "${yellow}--- Top 5 IP Addresses ---${reset}"
                            echo "$TARGET_TRAFFIC" | awk -v b="$bold" -v rs="$reset" '{print $1}' | sort | uniq -c | sort -nr | head -n 5 | awk -v b="$bold" -v rs="$reset" '{printf "%s%5s%s hits from %s\n", b, $1, rs, $2}'
                        
                            echo "${yellow}--- Top 5 Requested URLs ---${reset}"
                            echo "$TARGET_TRAFFIC" | awk -F\" '{print $2}' | awk '{print $2}' | sort | uniq -c | sort -nr | head -n 5 | awk -v b="$bold" -v rs="$reset" '{printf "%s%5s%s requests for %.60s...\n", b, $1, rs, $2}'
                        
                            echo "${yellow}--- Top 5 User Agents ---${reset}"
                            echo "$TARGET_TRAFFIC" | awk -F\" '{print $6}' | sort | uniq -c | sort -nr | head -n 5 | awk -v b="$bold" -v rs="$reset" '{
                                count=$1; $1=""; 
                                # trim leading space
                                gsub(/^ /, "", $0);
                                printf "%s%5s%s hits by %.70s...\n", b, count, rs, $0
                            }'
                        else
                            echo "No traffic logged for $TOP_USER_NAME in the 10 minutes prior to OOM."
                        fi
                    else
                        echo "Could not find current access logs for $TOP_USER_NAME to analyze."
                    fi
                else
                    echo "Could not parse OOM Date: $LAST_OOM_DATE"
                fi
            fi
        fi
        
        echo "${yellow}---------------------------------------------------------------------${reset}"
    else
        echo "OOM killer detected in logs ($LAST_OOM_DATE), but detailed report could not be extracted."
        echo "${yellow}---------------------------------------------------------------------${reset}"
    fi
else
    echo "${green}OOM killer has not been invoked recently${reset}"
    echo "${yellow}---------------------------------------------------------------------${reset}"
fi