#!/usr/bin/env bash

folder=serial
nodelist=compute01,compute04,compute07,compute10,compute11,compute12

TIMESTR=`date +"%s"`
DBNAME="${folder}_${TIMESTR}"
NODES=(1)
ESIZE=(A B C AB AC BC ABC)

pwd=`pwd`

thrds=1

QSIZE=1

for e in "${ESIZE[@]}"
do
	echo "----------------------- ${e} Exploits -----------------------" >> ./slurm_reports/"$folder"/sync_data.txt

	SKIP_FLAG=0
	#Dispatch the job and get the output ("Submitted batch job <jobid>
	JOB=`sbatch --nodelist="$nodelist" --exclusive ag_run.sh "$DBNAME" "$e"`
	#Pull the jobid
	JOB=`echo "$JOB" | grep -oP '(?<=job )[^ ]*'`	
	
	#Output will be stored as this format
	FILE="./slurm_reports/job.${JOB}.out"
	ERR="./slurm_reports/job.${JOB}.err"

	NOW=`date +"%H:%M"`
	
	echo "Synchronous test with "$e" exploit set with dispatched at "$NOW" with jobid "$JOB"" >> ./slurm_reports/"$folder"/sync_data.txt
		
	#Wait until job is finished
	while squeue | grep "$JOB" > /dev/null 2>&1; do	
		#Slurm occasionally has an issue where the prog finishes but doesn't leave slurm queue
		#Check if program finished by checking the output file for the final line, delay by 5 seconds for safety, then cancel the job
		if test -f "$FILE" && (cat "$FILE" | grep "total run time" > /dev/null 2>&1) ; then
			for j in {1..5}
				do
					sleep 1
				done
			SKIP_FLAG=1
			echo "Job done, but slurm hung." >> ./slurm_reports/"$folder"/sync_data.txt
			tmp=`scancel "$JOB"`
			sleep 3
			break	
		fi
		sleep 1
	done

	#Wait for the delay between job finish and file write
	while ! test -f  "$FILE"; do
		sleep 1
	done
	
	#Get runtime and states from the output
	#RUNTIME=`cat "$FILE" | grep -oP '(?<=total run time is )[^ ]*'`
	RUNTIME=`cat "$FILE" | grep -oP '(?<=AG TOOK )[^ ]*'`
	DUP=`cat "$FILE" | grep -oP '(?<=Duplicated States that had to be removed: )[^ ]*'`
	STATES=`cat "$FILE" | grep -oP '(?<=Total States: )[^ ]*'`
	EDGES=`cat "$FILE" | grep -oP '(?<=Total Edges: )[^ ]*'`

			
	if ! test -s "$ERR" || test "$SKIP_FLAG" -eq 1 ; then
		printf "${e} Exploit Set, ${RUNTIME} runtime. ${STATES} states, ${EDGES} edges  \n\n" >> ./slurm_reports/"$folder"/sync_data.txt
		str="$e","$STATES","$EDGES"
		echo $str >> ./slurm_reports/"$folder"/sync_data.csv 	
	else
		printf "Errors occurred. Please see err file for more details. \n\n" >> ./slurm_reports/"$folder"/sync_data.txt
	fi

done