/usr/local/lp/bin/ Current File: //usr/local/lp/bin/serverweight.sh
#!/bin/bash
export PATH=/sbin:/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
#default to 'today' if arg not given
range="${1:-today}"
format="$2"
opt="$3"
sysstat_data="/var/log/sa"
sar=$(which sar 2> /dev/null)
# weight factor is what number the metric is divided by before reporting
# this can be used to lower the weight of a certain metric in the final score
users_weight_factor='10'
# config file can override default weight settings
conf_file='/etc/serverweight.conf'
settings='users_weight_factor'
if [[ -f "$conf_file" ]]; then
    for setting in $settings; do
        if grep -q "^${setting}=[0-9]*" "$conf_file" 2>/dev/null; then
            setting_value="$(awk -F= "/^${setting}/ {print \$NF}" "$conf_file")"
            if grep -qE '^[0-9]*$' 2>/dev/null <<< "$setting_value"; then
                eval "${setting}=${setting_value}"
            fi
        fi
    done
fi


if [ ! -d "$sysstat_data" ] || [ ! -f "$sar" ]; then
	echo "could not find sysstat data path, is sysstat installed?"
	exit 1
fi

bc=$(which bc 2> /dev/null)
if [ ! -f "$bc" ]; then
	echo "could not find berkley calculator (bc), is bc installed?"
	exit 1
fi

get_weight() {
	safile="$1"
	format="$2"
	opt="$3"

	# how many reboots in the last 72h
	restart_count=0
	for j in $(find "${sysstat_data}/" -type f -regex '.*\/sa[0-9]*' -printf '%T@ %P\n' | sort -n | awk '{print $NF}' | tail -n3); do
		restarts=$(sar -q -f "${sysstat_data}/${j}" 2> /dev/null | grep -Ec RESTART)
		restart_count=$((restarts+restart_count))
	done
	if [ ! "$restart_count" -eq "0" ]; then
		uptime=$(echo "$restart_count * 5" | $bc)
	else
		uptime="0"
	fi

	# cpu idle cycles
	cpu=$(echo "scale=2 ; (100 - $(LC_TIME='POSIX' sar -f "$safile" 2> /dev/null | grep -Ev '^$|Average|idle' | awk '{print$8}' | sort -n | awk 'BEGIN{c=0} {total[c]=$1; c++;} END{print total[int(NR*0.95-0.5)]}' )) / 10" | $bc)
	# weight cpu usage higher at 70, 80 and 90% (30, 20 and 10% idle)
	if [ "$(echo "$cpu" | sed 's/^\./0./' | cut -d'.' -f1)" -gt "9" ]; then
		multi=6
	elif [ "$(echo "$cpu" | sed 's/^\./0./' | cut -d'.' -f1)" -gt "8" ]; then
		multi=5
	elif [ "$(echo "$cpu" | sed 's/^\./0./' | cut -d'.' -f1)" -gt "7" ]; then
		multi=4
	else
		multi=2
	fi
	cpu=$(echo "scale=2 ; $cpu * $multi" | $bc)

	# load 95th, smooth out spikes and noramlize
	load=$(LC_TIME='POSIX' sar -f "$safile" -q 2> /dev/null | grep -Evi "^$|average|linux|ldavg" | awk '{print$4}' | sort -n | awk 'BEGIN{c=0} {total[c]=$1; c++;} END{print total[int(NR*0.95-0.5)]}')

	# iowait
	# we need to weigh iowait heavily as it has a cascading impact on system performance
	iow=$(echo "scale=2 ;  $(LC_TIME='POSIX' sar -f "$safile" 2> /dev/null | tail -n1 | awk '{print$6}')" | $bc)
	if [ "$(echo "$iow" | sed 's/^\./0./' | cut -d'.' -f1)" -ge "6" ]; then
		multi=6
	elif [ "$(echo "$iow" | sed 's/^\./0./' | cut -d'.' -f1)" -ge "4" ]; then
		multi=5
	elif [ "$(echo "$iow" | sed 's/^\./0./' | cut -d'.' -f1)" -ge "2" ]; then
		multi=4
	else
		multi=2
	fi
	iow=$(echo "scale=2 ; $iow * $multi" | $bc)

	# await
	# we add await to help contexualize iowait, we need to weight this less heavily than iowait
	aw=$(LC_TIME='POSIX' sar  -d 2> /dev/null | grep -Evi "^$|average|await|restart|linux" | awk '{print$8}' | sort -n | awk 'BEGIN{c=0} {total[c]=$1; c++;} END{print total[int(NR*0.95-0.5)]}')
	multi=1
	aw=$(echo "scale=2 ; ($aw * $multi) *0.10" | $bc)

	# what does our home usage look like?
	duse=$(df -m / /var /usr /home | grep -E dev | sort -n -k5 | tail -n1 | awk '{print$5}' | tr -d '%')
	if [ "$duse" -ge "90" ]; then
		duse=$(echo "scale=2 ; ($duse * 5)/10" | $bc)
	elif [ "$duse" -ge "80" ]; then
		duse=$(echo "scale=2 ; ($duse * 4)/10" | $bc)
	elif [ "$duse" -ge "70" ]; then
		duse=$(echo "scale=2 ; ($duse * 3)/10" | $bc)
	elif [ "$duse" -ge "60" ]; then
		duse=$(echo "scale=2 ; ($duse * 2)/10" | $bc)
	else
		duse=$(echo "scale=2 ; ($duse * 1)/10" | $bc)
	fi

	# user counts cpanel and non-cpanel
	if [ -d "/var/cpanel/users" ]; then
		users=$(echo "scale=2 ; $(find /var/cpanel/users -maxdepth 1 -type f | wc -l) / ${users_weight_factor}" | $bc)
	else
        # nex users start at uid 60000 so we limit out upper bound so we don't include them
        # these users do not exist in /etc/passwd on systems using freeipa so this only
        # applies on el6 right now
		users=$(echo "scale=2 ; $(awk -F':' '{if($3>=500 && $3<60000){print$1}}' /etc/passwd | wc -l) / ${users_weight_factor}" | $bc)
	fi

	# this is our running processes size 95th, smooth out spikes and normalized
	procs=$(grep -E processor /proc/cpuinfo  -c)
	runsz=$(LC_TIME='POSIX' sar -f "$safile" -q 2> /dev/null | grep -Evi "^$|average|linux|ldavg" | awk '{print$2}' | sort -n | awk 'BEGIN{c=0} {total[c]=$1; c++;} END{print total[int(NR*0.95-0.5)]}')
	if [ "$runsz" -gt "$procs" ]; then
		runsz=$(echo "scale=2 ; $runsz * 1.5" | bc)
	fi
	runq=$(echo "scale=2 ; $runsz / 10" | $bc)

	# memory pressure (commit) and swap use averages, normalized for Cent5 & 6
	if grep -qE '5.[0-9]+' /etc/redhat-release; then
		mempress=$(echo "scale=2 ; $(LC_TIME=POSIX sar -f "$safile" -r 2> /dev/null | grep -Evi '^$|average|linux|commit|memused' | awk '{sum+=$4} END {printf "%.2f", sum/NR}') / 10" | $bc)
		swapuse=$(echo "scale=2 ; $(LC_TIME=POSIX sar -f "$safile" -r 2> /dev/null | grep -Evi '^$|linux|average|swpcad' | awk '{sum+=$9} END {printf "%.2f", sum/NR}') / 10" | $bc)
	else
		memcommit=$(echo "scale=2 ; $(LC_TIME=POSIX sar -f "$safile" -r 2> /dev/null | grep -Evi '^$|average|linux|commit' | awk '{sum+=$8} END {printf "%.2f", sum/NR}') / 10" | $bc)
		memuse=$(echo "scale=2 ; $(LC_TIME=POSIX sar -f "$safile" -r 2> /dev/null | grep -Evi '^$|average|linux|commit' | awk '{sum+=$4} END {printf "%.2f", sum/NR}') / 10" | $bc)
		mempress=$(echo "scale=2 ; $memcommit + $memuse" | $bc)
		swapuse=$(echo "scale=2 ; $(LC_TIME=POSIX sar -f "$safile" -S 2> /dev/null | grep -Evi '^$|linux|average|swpcad' | awk '{sum+=$4} END {printf "%.2f", sum/NR}') / 10" | $bc)
	fi
	if (( $(echo "$mempress >= 100" | bc -l) )); then
		mempress=5
	elif (( $(echo "$swapuse >= 100" | bc -l) )); then
		swapuse=5
	fi

	cpu=${cpu:-0}
	iow=${iow:-0}
	aw=${aw:-0}
	load=${load:-0}
	runq=${runq:-0}
	mempress=${mempress:-0}
	swapuse=${swapuse:-0}
	duse=${duse:-0}
	users=${users:-0}
	uptime=${uptime:-0}

	weight=$(echo "scale=2; $cpu + $iow + $aw + $load + $runq + $mempress + $swapuse + $duse + $users + $uptime" | $bc)
	if [ "$format" == "csv" ]; then
		echo "$date,$HOSTNAME,$cpu,$iow,$aw,$load,$mempress,$swapuse,$runq,$duse,$users,$uptime,$weight"
		elif [ "$format" == "json" ]; then
		echo -e "{\"$HOSTNAME\": { \"date\": \"$date\", \"hostname\": \"$HOSTNAME\", \"cpu\": $cpu, \"iowait\": $iow, \"await\": $aw, \"load95th\": $load, \"memory\": $mempress, \"swap\": $swapuse, \"runsz\": $runq, \"disk\": $duse, \"users\": $users, \"uptime\": $uptime, \"weight\": $weight }}"
	else
		echo "$date $HOSTNAME cpu $cpu iowait $iow await $aw load95th $load memory $mempress swap $swapuse runsz $runq disk $duse users $users uptime $uptime weight $weight"
	fi
}


# we by default run for current day / yesterday weight output
# if $1 = range then trend weight data output
if [ "$range" == "trend" ]; then
	for i in $(find "${sysstat_data}/" -type f -regex '.*\/sa[0-9]*' -printf '%T@ %P\n' | sort -n | awk '{print $NF}'); do
	        date=$(stat -c '%z' "${sysstat_data}/${i}" | awk '{print$1}')
		get_weight "${sysstat_data}/${i}" "$format" "$opt"
		((count++))
		weight_sum=$(echo "scale=2; ${weight_sum:-0} + ${weight:-0}" | bc)
	done
	weight_avg=$(echo "scale=2; $weight_sum / $count" | $bc)
	if [ "$format" == "csv" ]; then
		echo "AVERAGE,$HOSTNAME,weight_avg,$weight_avg"
	elif [ "$format" == "json" ]; then
		echo -e "{\"$HOSTNAME\": { \"average_weight\": \"$weight_avg\" }}"
	else
		echo "AVERAGE $HOSTNAME weight_avg $weight_avg"
	fi
elif [ "$range" == "today" ]; then
	if [ "$(date +'%R' | cut -d ':' -f1)" -lt "12" ]; then
		day="yesterday"
	else
		day="today"
	fi
	safile="$sysstat_data/sa$(date -d $day +'%d')"
	date=$(stat -c '%z' "$safile" | awk '{print$1}')

	get_weight "$safile" "$format" "$opt"
elif [ "$range" == "yesterday" ]; then
	day="$range"
	safile="$sysstat_data/sa$(date -d "$day" +'%d')"
	date=$(stat -c '%z' "$safile" | awk '{print$1}')

	get_weight "$safile" "$format" "$opt"
fi