/usr/local/stat_watch/includes/ Current File: //usr/local/stat_watch/includes/run.shf
#! /bin/bash

function fn_get_direc {
### Given the identifier for a directive, get the value of that directive
	if [[ -n "$f_JOB" ]]; then
		local v_DIREC_IDENT="$1"
		v_DIREC="$( grep -E "^\s*$v_DIREC_IDENT" "$f_JOB" | tail -n1 | sed "s/^[[:blank:]]*$v_DIREC_IDENT[[:blank:]]*//;s/[[:blank:]]*$//" )"
	else
		v_DIREC=
	fi
}

function fn_send_email {
### Given a type of message to send, send that type of message
### "1" - Send that no changes were detected
### "2" - Send Stat Watch results
### "3" - Send that the job ran too long
### "4" - Test message
### Optionally, "$2" can be the address to send to
	local v_TYPE="$1"
	local v_EMAIL=

	### Get the email address
	if [[ -z "$2" ]]; then
		fn_get_direc "Email"; v_EMAIL="$v_DIREC"
	else
		v_EMAIL="$2"
	fi

	### Send the email
	if [[ -n "$v_EMAIL" && -n "$v_TYPE" ]]; then
		if [[ -n "$v_RUN_PRE_E" ]]; then
			eval "$v_RUN_PRE_E"
		fi
		local v_SUBJECT=
		if [[ "$v_TYPE" == "1" ]]; then
			v_SUBJECT="Stat Watch - No changed detected on $(hostname)"
		elif [[ "$v_TYPE" == "2" || "$v_TYPE" == "4" ]]; then
			v_SUBJECT="Stat Watch - File changes on $(hostname)"
		elif [[ "$v_TYPE" == "3" ]]; then
			v_SUBJECT="Stat Watch - Job was terminated on $(hostname)"
		fi
		( 
			if [[ -f "$v_DIR"/"$v_NAME"_message_head.txt ]]; then 
				cat "$v_DIR"/"$v_NAME"_message_head.txt; 
				echo
			fi
			if [[ "$v_TYPE" == "1" ]]; then
				echo "No changes were detected."
			elif [[ "$v_TYPE" == "2" ]]; then
				cat "$f_EMAIL"
			elif [[ "$v_TYPE" == "3" ]]; then
				echo "Job '$v_NAME' was terminated for having run longer then that maximum allowed time of $v_MAX_RUN seconds. You can increase this time by adjusting the 'Max-run' control string in the file $( echo -n "$f_JOB" | "$d_PROGRAM"/scripts/escape.pl )"
			elif [[ "$v_TYPE" == "4" ]]; then
				echo "This is a test message to confirm that mesages from server $(hostname) are reaching the address \"$v_EMAIL\"."
				echo
				echo "If you were not expecting this message, please ignore it as it was likely sent in error."
			fi
			echo
			if [[ -f "$v_DIR"/"$v_NAME"_message_foot.txt ]]; then 
				cat "$v_DIR"/"$v_NAME"_message_foot.txt; 
				echo
			fi
			echo "This output was generated by $( echo -n "$d_PROGRAM/$f_PERL_SCRIPT" | "$d_PROGRAM"/scripts/escape.pl ) and $( echo -n "$d_PROGRAM/$f_PROGRAM" | "$d_PROGRAM"/scripts/escape.pl ) from the job file at $( echo -n "$f_JOB" | "$d_PROGRAM"/scripts/escape.pl )"
			if [[ -f "$v_DIR"/"$v_NAME"_message_end.txt ]]; then 
				echo
				cat "$v_DIR"/"$v_NAME"_message_end.txt; 
			fi
		### $v_EMAIL isn't in quotes here because it's possible that they have specified more than one email address
		) | mail -s "$v_SUBJECT" $v_EMAIL
		if [[ -n "$v_RUN_POST_E" ]]; then
			eval "$v_RUN_POST_E"
		fi
	fi
}

function fn_run {
	f_JOB="$1"
	if [[ $( echo "$f_JOB" | grep -c "/" ) -lt 1 || ! -f "$f_JOB" ]]; then
		if [[ -f "$f_JOBS" ]]; then
			$f_JOB="$( egrep "^$f_JOB:" "$f_JOBS" | cut -d ":" -f2- | sed "s/ - Created .*$//" )"
			if [[ -z "$f_JOB" || ! -f "$f_JOB" ]]; then
				echo "No such file $( echo -n "$f_JOB" | "$d_PROGRAM"/scripts/escape.pl )"
				exit
			fi
		else
			echo "No such file $( echo -n "$f_JOB" | "$d_PROGRAM"/scripts/escape.pl )"
			exit
		fi
	fi
	v_ERROR_OUT="/dev/null"
	if [[ -n "$2" && "$2" == "--errors" ]]; then
		v_ERROR_OUT="/dev/stderr"
	fi
	fn_get_direc "Name"; v_NAME="$v_DIREC"
	if [[ -z "$v_NAME" ]]; then
		echo "Cannot find name in Job file. Exiting"
		exit
	fi

	### Find what directory we should be storing files in
	fn_file_path "$f_JOB"; local v_DIR="$s_DIR"
	fn_get_direc "Working-dir"; local v_DIR2="$v_DIREC"
	if [[ -n "$v_DIR2" && -d "$v_DIR2" ]]; then
		v_DIR="$v_DIR2"
	fi
	fn_get_direc "Log"; f_LOG="$v_DIREC"
	if [[ -z "$f_LOG" ]]; then
		f_LOG="$v_DIR"/"$v_NAME".log
	fi
	fn_get_direc "Expire"; v_EXPIRE="$v_DIREC"
	### Create a directory to stand as an indicator that a job is running
	v_EXIT=false
	mkdir "$v_DIR"/"$v_NAME"_run 2> "$v_ERROR_OUT" || v_EXIT=true
	if [[ $v_EXIT == true ]]; then
		### This won't be 100% effective, but it should prevent most instances of this running twice at the same time
		sleep 0.$(( RANDOM % 10 ))
		v_PID="$( cat "$v_DIR"/"$v_NAME"_run/pid 2> "$v_ERROR_OUT" )"
		if [[ -n "$v_PID" && $( cat /proc/$v_PID/cmdline 2> "$v_ERROR_OUT" | grep -F -c "$d_PROGRAM/$f_PROGRAM" ) -gt 0 ]]; then
			v_EPOCH="$( cat "$v_DIR"/"$v_NAME"_run/epoch 2> "$v_ERROR_OUT" )"
			### check to see if the job has run for too long
			fn_get_direc "Max-run"; v_MAX_RUN2="$v_DIREC"
			if [[ -n "$v_MAX_RUN2" && $( echo "$v_MAX_RUN" | grep -E -c "^[0-9]+$" ) -gt 0 ]]; then
				v_MAX_RUN="$v_MAX_RUN2"
			fi
			### Are there reasons to kill the job?
			v_KILL=false
			if [[ -z "$v_EPOCH" ]]; then
			### We don't know when the job started because the file did not exist or was empts
				v_KILL=true
			elif [[ $( echo "$v_EPOCH" | grep -E -c "^[0-9]+$" ) -lt 1 ]]; then
			### We don't know when the job started because the timestamp is in the wrong format
				v_KILL=true
			elif [[ $(( $( date +%s ) - $v_EPOCH )) -gt "$v_MAX_RUN" ]]; then
			### We know when the job started, and it has ran too long
				v_KILL=true
			fi
			### If so, let's kill it
			if [[ $v_KILL == true ]]; then
				kill -9 $v_PID
				if [[ -n "$v_EPOCH" || $( echo "$v_EPOCH" | grep -E -c "^[0-9]+$" ) -gt 0 ]]; then
					echo "$( date +%Y-%m-%d" "%T" "%z ) - Job killed after running for $(( $( date +%s ) - $v_EPOCH )) seconds" >> "$f_LOG"
				fi
				fn_send_email "3"
				### The logic behind exiting rather than starting a new iteration is as follows:
				### If the reason that the last job took so long is because the server is overloaded, trying to run again now will only continue to impact that
				rm -rf "$v_DIR"/"$v_NAME"_run
				echo "Previous job ran too long. Exiting"
				exit
			else
				echo "An existing iteration of this job is already running"
				exit
			fi
		fi
	fi
	echo "$$" > "$v_DIR"/"$v_NAME"_run/pid
	echo "$( date +%s )" > "$v_DIR"/"$v_NAME"_run/epoch
	if [[ -n "$v_EXPIRE" && $( echo "$v_EXPIRE" | grep -E -c "^[0-9]+$" ) -gt 0 ]]; then
	### If the job was set to expire, see if we still need to run it
		if [[ $( date +%s ) -gt $v_EXPIRE ]]; then
			if [[ $( date --date="now - 15 days" +%s ) -gt $v_EXPIRE ]]; then
			### If we're 15 days past expiration, delete everything
				fn_get_direc "BackupD"; d_BACKUP="$v_DIREC"
				echo "$( date +%Y-%m-%d" "%T" "%z ) - Removing job $( echo -n "$f_JOB" | "$d_PROGRAM"/scripts/escape.pl )" >> "$f_LOG"
				rm -f "$v_DIR"/"$v_NAME"_files.txt "$v_DIR"/"$v_NAME"_files2.txt "$v_DIR"/"$v_NAME"_changes_*.txt "$v_DIR"/"$v_NAME"_message_head.txt "$v_DIR"/"$v_NAME"_message_foot.txt "$v_DIR"/"$v_NAME"_stamp
				### Only remove the backup directory if it's not the default backup directory
				if [[ -d "$d_BACKUP" && "$d_BACKUP" != "$v_BACKUP_DIRECTORY" ]]; then
					rm -rf "$d_BACKUP"
				fi
				rm -f "$f_JOB"
			fi
			rm -rf "$v_DIR"/"$v_NAME"_run
			exit
		fi
	fi

	### Find all of the scripts we might want to run
	fn_get_direc "Run-start"; v_RUN_START="$v_DIREC"
	fn_get_direc "Run-post"; v_RUN_POST="$v_DIREC"
	fn_get_direc "Run-pre-e"; v_RUN_PRE_E="$v_DIREC"
	fn_get_direc "Run-post-e"; v_RUN_POST_E="$v_DIREC"
	fn_get_direc "Run-end"; v_RUN_END="$v_DIREC"
	### Create a variable that makes the email file available to scripts
	export f_EMAIL="$v_DIR"/"$v_NAME"_changes_"$( date +%s )".txt

	fn_get_direc "Nice"; v_NICE2="$v_DIREC"
	if [[ -n "$v_NICE2" && $( echo "$v_NICE2" | grep -E -c "^-?[0-9]+$" ) -gt 0 && $v_NICE2 -ge -20 && $v_NICE2 -le 19 ]]; then
		v_NICE="$v_NICE2"
	fi
	if [[ ! -f "$v_DIR"/"$v_NAME"_files.txt ]]; then
	### If this is the first run, do an initial backup of files
		if [[ -n "$v_RUN_START" ]]; then
			eval "$v_RUN_START"
		fi
		stat -c '%Y' "$f_JOB" > "$v_DIR"/"$v_NAME"_stamp
		nice -n "$v_NICE" "$v_PERL" "$d_PROGRAM"/"$f_PERL_SCRIPT" "${a_PERL_ARGS[@]}" --record --log "$f_LOG" -i "$f_JOB" -o "$v_DIR"/"$v_NAME"_files.txt -v 2> "$v_ERROR_OUT"
		nice -n "$v_NICE" "$v_PERL" "$d_PROGRAM"/"$f_PERL_SCRIPT" "${a_PERL_ARGS[@]}" --backup --log "$f_LOG" -i "$f_JOB" "$v_DIR"/"$v_NAME"_files.txt 2> "$v_ERROR_OUT"
		if [[ -n "$v_RUN_POST" ]]; then
			eval "$v_RUN_POST"
		fi
	else
	### If this is a later run, diff the reports, and if there were changes, email them out
		if [[ -n "$v_RUN_START" ]]; then
			eval "$v_RUN_START"
		fi
		nice -n "$v_NICE" "$v_PERL" "$d_PROGRAM"/"$f_PERL_SCRIPT" "${a_PERL_ARGS[@]}" --record --log "$f_LOG" -i "$f_JOB" -o "$v_DIR"/"$v_NAME"_files2.txt 2> "$v_ERROR_OUT"
		if [[ $( stat -c '%Y' "$f_JOB" ) -gt $( cat "$v_DIR"/"$v_NAME"_stamp 2> "$v_ERROR_OUT" ) ]]; then
			### If the job file has been updated, there's a chance that we need to back up additional files
			### Specifically, if we've seen those files before and they have not changed, but they are now to the list of things that need to be backed up
			nice -n "$v_NICE" "$v_PERL" "$d_PROGRAM"/"$f_PERL_SCRIPT" "${a_PERL_ARGS[@]}" --log "$f_LOG" --backup -i "$f_JOB" "$v_DIR"/"$v_NAME"_files2.txt 2> "$v_ERROR_OUT"
			### update the stat file so that we know not to do that next time
			stat -c '%Y' "$f_JOB" > "$v_DIR"/"$v_NAME"_stamp
		fi
		nice -n "$v_NICE" "$v_PERL" "$d_PROGRAM"/"$f_PERL_SCRIPT" "${a_PERL_ARGS[@]}" --diff --log "$f_LOG" --no-check-retention -i "$f_JOB" "$v_DIR"/"$v_NAME"_files.txt "$v_DIR"/"$v_NAME"_files2.txt --backup -o "$f_EMAIL" --format text 2> "$v_ERROR_OUT"

		### If a db_watch job has been specified, run that
		fn_get_direc "Db-watch"; v_DB_WATCH="$v_DIREC"
		if [[ -n "$v_DB_WATCH" && -f "$v_DB_WATCH" ]]; then
			mkdir -p "$v_DIR"/"$v_NAME"_db_watch
			"$d_PROGRAM"/scripts/db_watch.sh --run "$v_DB_WATCH" --out-dir "$v_DIR"/"$v_NAME"_db_watch --out "$f_EMAIL"_
			if [[ -f "$f_EMAIL"_ ]]; then
				### If there were results, append them to the end of the email
				echo >> "$f_EMAIL"
				cat "$f_EMAIL"_ >> "$f_EMAIL"
				### Get rid of the original file
				rm -f "$f_EMAIL"_
			fi
		fi

		if [[ -n "$v_RUN_POST" ]]; then
			eval "$v_RUN_POST"
		fi
		### This file should contain at least two lines - one indicating it's processing and one for the file or directory.
		if [[ $( wc -l "$v_DIR"/"$v_NAME"_files2.txt 2> "$v_ERROR_OUT" | cut -d " " -f1 ) -gt 1 ]]; then
			mv -f "$v_DIR"/"$v_NAME"_files2.txt "$v_DIR"/"$v_NAME"_files.txt
			if [[ $( wc -l "$f_EMAIL" 2> "$v_ERROR_OUT" | cut -d " " -f1 ) -lt 2 ]]; then
				### This is the one instances where we're getting a directive from the job file without fn_get_direc
				v_NO_CHANGE="$( grep -E -c "^\s*Email-no-changes\s*$" "$f_JOB" )"
				if [[ $v_NO_CHANGE -gt 1 ]]; then
					fn_send_email "1"
				else
					rm -f "$f_EMAIL"
				fi
			else
			### If there were changes, check if we have an email address and then send a message to it
				fn_send_email "2"
			fi
		else
			### If the file didn't contain at least two lines, then there was something broken about the output. Remove it rather than replace it
			rm -f "$v_DIR"/"$v_NAME"_files2.txt
		fi

		##### Instead of pruning based on a random chance, if we're already storing a file at "$d_BACKUP"/__last_prune, why not just base it on the amount of time elapsed since that time?
		### Check to see if the user set the prune variables to soemthing different
		fn_get_direc "Prune-max"; v_PRUNE_MAX2="$v_DIREC"
		if [[ -n "$v_PRUNE_MAX2" && $( echo "$v_PRUNE_MAX2" | grep -E -c "^[0-9]+$" ) -gt 0 ]]; then
			v_PRUNE_MAX="$v_PRUNE_MAX2"
		fi
		fn_get_direc "Prune-chance"; v_PRUNE_CHANCE2="$v_DIREC"
		if [[ -n "$v_PRUNE_CHANCE2" && $( echo "$v_PRUNE_CHANCE2" | grep -E -c "^[0-9]+$" ) -gt 0 ]]; then
			v_PRUNE_CHANCE="$v_PRUNE_CHANCE2"
		fi

		### Determine whether or not we're purning old backups
		if [[ $(( $v_PRUNE_CHANCE + RANDOM % $v_PRUNE_MAX )) -le "$v_PRUNE_CHANCE" ]]; then
			fn_get_direc "BackupD"; d_BACKUP="$v_DIREC"
			v_LAST_PRUNE=0;
			### If there's a default backup directory, prevent multiple jobs from pruning it all the time
			if [[ "$v_BACKUP_DIRECTORY" == "$d_BACKUP" && -d "$d_BACKUP" && -f "$d_BACKUP"/__last_prune ]]; then
				v_LAST_PRUNE="$( cat "$d_BACKUP"/__last_prune )"
			fi
			if [[ $(( v_LAST_PRUNE + 10800 )) -le "$( date +%s )" ]]; then
				nice -n "$v_NICE" "$v_PERL" "$d_PROGRAM"/"$f_PERL_SCRIPT" "${a_PERL_ARGS[@]}" --prune --log "$f_LOG" -i "$f_JOB" 2> "$v_ERROR_OUT"
			fi
		fi

		### Remove old email messages
		fn_get_direc "Email-retain"; v_EMAIL_RETAIN2="$v_DIREC"
		if [[ -n "$v_EMAIL_RETAIN2" && $( echo "$v_EMAIL_RETAIN2" | grep -E -c "^[0-9]+$" ) -gt 0 ]]; then
			v_EMAIL_RETAIN="$v_EMAIL_RETAIN2"
		fi
		if [[ "$v_EMAIL_RETAIN" -ne 0 ]]; then
			v_EMAIL_RETAIN="$(( $v_EMAIL_RETAIN * 86400 ))"
			for i in $( \ls -1 "$v_DIR" | egrep "^$v_NAME""_changes_" | sed -E "s/^.*[^0-9]([0-9]+).txt$/\1/" ); do
				if [[ $(( $( date +%s ) - $v_EMAIL_RETAIN )) -gt "$i" ]]; then
					rm -f "$v_DIR"/"$v_NAME"_changes_"$i".txt
				fi
			done
		fi
	fi

	### Trim lines from the beginning of the log if necessary
	fn_get_direc "Log-max"; v_LOG_MAX2="$v_DIREC"
	if [[ -n "$v_LOG_MAX2" && $( echo "$v_LOG_MAX2" | grep -E -c "^[0-9]+$" ) -gt 0 && "$v_LOG_MAX2" -gt 0 ]]; then
		v_LOG_MAX="$v_LOG_MAX2"
	fi
	if [[ $( stat -c %s "$f_LOG" ) -gt $v_LOG_MAX ]]; then
		### Goodbye first thousand lines
		sed -i "1,1000d" "$f_LOG" 2> "$v_ERROR_OUT"
	fi

	if [[ -n "$v_RUN_END" ]]; then
		eval "$v_RUN_END"
	fi
	rm -rf "$v_DIR"/"$v_NAME"_run
	exit
}