// ag_gen.cpp contains the methods for building an attack graph and generating
// an attack graph's exploits and printing them

#include <algorithm>
#include <chrono>
#include <iostream>
#include <vector>
#include <tuple>
#include <unordered_map>
#include <omp.h>
#include <sys/time.h>
#include <string.h>
#include <map>
#include <random>
#include <unordered_set>
#include <set>

#include "ag_gen.h"

#include "../util/odometer.h"
#include "../util/db_functions.h"
#include "../util/avail_mem.h"
#include "../mpi/tasks.h"

#include <boost/archive/tmpdir.hpp>
#include <boost/archive/text_iarchive.hpp>
#include <boost/archive/text_oarchive.hpp>
#include <boost/function_output_iterator.hpp>

#include <boost/serialization/base_object.hpp>
#include <boost/serialization/utility.hpp>
#include <boost/serialization/list.hpp>
#include <boost/serialization/assume_abstract.hpp>
#include <boost/serialization/string.hpp>
#include <boost/serialization/vector.hpp>
#include <boost/serialization/deque.hpp>

#include <boost/mpi.hpp>
#include <boost/mpi/environment.hpp>
#include <boost/mpi/communicator.hpp>
#include <boost/mpi/collectives.hpp>

#include <boost/serialization/is_bitwise_serializable.hpp>
#include <boost/range/irange.hpp>

namespace mpi = boost::mpi;

#ifdef REDIS

/**
* @brief Constructor for generator
* @details Builds a generator for creating attack graphs.
*
* @param _instance The initial information for generating the graph
*/
AGGen::AGGen(AGGenInstance &_instance, RedisManager &_rman) : instance(_instance), rman(&_rman) {
    rman->clear();
    auto init_quals = instance.initial_qualities;
    auto init_topos = instance.initial_topologies;
    NetworkState init_state(init_quals, init_topos);
    init_state.set_id();
    int init_id = init_state.get_id();
    FactbaseItems init_items =
    	make_tuple(make_tuple(init_quals, init_topos), init_id);
    instance.factbases.push_back(init_state.get_factbase());
    instance.factbase_items.push_back(init_items);
    std::string hash = std::to_string(init_state.get_hash(instance.facts));
    // std::cout << "before init insertion" << std::endl;
    rman->insert_factbase(hash, init_id);
    // rman->insert_facts(hash, init_quals, init_topos);
    rman->commit();
    // std::cout << "after init insertion" << std::endl;
    // hash_map.insert(std::make_pair(init_state.get_hash(instance.facts), init_id));
    frontier.push_back(init_state);
    use_redis = true;
}

#endif

AGGen::AGGen(AGGenInstance &_instance) : instance(_instance) {
    auto init_quals = instance.initial_qualities;
    auto init_topos = instance.initial_topologies;
    NetworkState init_state(init_quals, init_topos);//instantiate an obj init_state with initial input
    init_state.set_id();
    int init_id = init_state.get_id();
    FactbaseItems init_items =
    	make_tuple(make_tuple(init_quals, init_topos), init_id);
    instance.factbases.push_back(init_state.get_factbase());
    instance.factbase_items.push_back(init_items);
    std::string hash = std::to_string(init_state.get_hash(instance.facts));
    hash_map.insert(std::make_pair(init_state.get_hash(instance.facts), init_id));
    frontier.push_back(init_state);
    use_redis = false;
    instance.duplicates = 0;
}

/**
* @brief Generates exploit postconditions
* @details When an exploit is known to apply to a set of assets,
* the postconditions must be generated. This is done by iterating
* through each parameterized fact and inserting the applicable
* assets.
*
* @param group A tuple containing the exploit and applicable assets
* @return A tuple containing the "real" qualities and "real" topologies
*/
std::tuple<std::vector<std::tuple<ACTION_T, Quality>>, std::vector<std::tuple<ACTION_T, Topology>>>
createPostConditions(std::tuple<Exploit, AssetGroup> &group, Keyvalue &facts) 
{
    auto ex = std::get<0>(group);
    auto ag = std::get<1>(group);

    auto perm = ag.get_perm();

    auto param_postconds_q = ex.postcond_list_q();
    auto param_postconds_t = ex.postcond_list_t();

    std::vector<std::tuple<ACTION_T, Quality>> postconds_q;
    std::vector<std::tuple<ACTION_T, Topology>> postconds_t;

    for (auto &postcond : param_postconds_q) {
        auto action = std::get<0>(postcond);
        auto fact = std::get<1>(postcond);
        	
        Quality q(perm[fact.get_param_num()], fact.name, fact.op,
        	  fact.value, facts);
        postconds_q.emplace_back(action, q);
    }

    for (auto &postcond : param_postconds_t) {
        auto action = std::get<0>(postcond);
        auto fact = std::get<1>(postcond);

        auto dir = fact.get_dir();
        auto prop = fact.get_property();
        auto op = fact.get_operation();
        auto val = fact.get_value();

        Topology t(perm[fact.get_from_param()],
        	   perm[fact.get_to_param()], dir, prop, op, val, facts);
        postconds_t.emplace_back(action, t);
    }
    return make_tuple(postconds_q, postconds_t);
}

/**
* @brief Generate attack graph
* @details Begin the generation of the attack graph. The algorithm is as
* follows:
*
*      1. Fetch next factbase to expand from the frontier
*      2. Fetch all exploits
*      3. Loop over each exploit to determine if it is applicable.
*          a. Fetch preconditions of the exploit
*          b. Generate all permutations of assets using the Odometer utility
*          c. Apply each permutation of the assets to the preconditions.
*          d. Check if ALL generated preconditions are present in the current
* factbase. 4a. If all preconditions are found, apply the matching asset group
* to the postconditions of the exploit. 4b. If not all preconditions are found,
* break and continue checking with the next exploit.
*      5. Push the new network state onto the frontier to be expanded later.
*/
AGGenInstance &AGGen::tasking_generate(bool batch_process, int batch_size, int numThrd, int initQSize,\
double mem_threshold, mpi::communicator &world) 
{

    //Init all Nodes with these variables
    std::vector<Exploit> exploit_list = instance.exploits;

    //Create a vector that contains all the groups of exploits to be fired synchonously
    std::vector<std::string> ex_groups;
    for (const auto &ex : exploit_list) {
        //If the group isn't already in the vector  
        if(!(std::find(ex_groups.begin(), ex_groups.end(), ex.get_group()) !=ex_groups.end())) {
            //Don't include the "no" group
            if(ex.get_group()!="null")
        	   ex_groups.emplace_back(ex.get_group());                        
        }      
    }

    int last_known_id = 0;
    //Print out the groups if desired
    if (world.rank() == 0){
        std::cout <<"\nGenerating through MPI Tasking" << std::endl;
        std::cout <<"There are "<<ex_groups.size()<<" groups: ";
        for(int i=0; i<ex_groups.size(); i++){
            std::cout<<ex_groups[i] << ". ";
        }
        std::cout<<"\n";
    }

    auto counter = 0;
    auto start = std::chrono::system_clock::now();

    unsigned long esize = exploit_list.size();
    if (world.rank() ==0){
       printf("esize: %lu\n", esize);
    }
    bool save_queued = false;


    std::unordered_map<size_t, PermSet<size_t>> od_map;
    size_t assets_size = instance.assets.size();
    for (const auto &ex : exploit_list) {
        size_t num_params = ex.get_num_params();
        if (od_map.find(num_params) == od_map.end()) {
            Odometer<size_t> od(num_params, assets_size);
            od_map[num_params] = od.get_all();
        }
    }

    int frt_size=frontier.size();
    if (world.rank() ==0){
        printf("The actual QSize to start using multiple threads is %d\n",frt_size); 
    }

    double total_t=0.0;
    //unti:ms
    double total_task0, total_task1, total_task2, total_task3, total_task4, total_task5, total_task6 = 0.0;

    struct timeval t1,t2;
    gettimeofday(&t1,NULL);

    int num_tasks = 6;
    #pragma omp parallel for num_threads(numThrd) default(none) shared(esize,counter,\
    exploit_list,od_map,frt_size,total_t,t1,t2,std::cout, mem_threshold, num_tasks,\
    ex_groups, world, total_task0, total_task1, total_task2, total_task3, total_task4, total_task5, last_known_id) schedule(dynamic,1)
    //auto ag_start = std::chrono::system_clock::now();
    for(int k=0;k<frt_size;k++){
        int mpi_exit = 0;
        double f_alpha = 0.0;
        int zero_queue = 0;
        auto tot_sys_mem = getTotalSystemMemory();
        std::deque<NetworkState> localFrontier;
        localFrontier.emplace_front(frontier[k]);

        if (mpi_exit == 1){
            #pragma omp cancel for
        }

        //Task 1 Node Allocating
        int alloc;
        double ceil_val;
        if(world.size() <= num_tasks)
            alloc = 1;
        else{
            ceil_val = ((double)world.size()-(double)num_tasks)/2.0;
            alloc = 1 + ceil(ceil_val);
        }

        //Task 2 Node Allocating
        int reduc_factor = 0;
        int two_alloc = alloc;
        if(world.size() % 2 != 0 && world.size() > num_tasks){
            two_alloc = alloc-1;
            reduc_factor = 1;
        }

        //Create Communicators
        //Note, these each return 2 handles!
        mpi::communicator tcomm = world.split(world.rank() > 0 && world.rank() <= alloc);
        mpi::communicator ttwo_comm;
        if(world.size() == 2){
            ttwo_comm = world.split(world.rank() == send_check(world, alloc) && world.rank() <= (2*two_alloc));
        }
        else
            ttwo_comm = world.split(world.rank() >= send_check(world, alloc) && world.rank() <= (alloc+two_alloc));

        while (!localFrontier.empty() || !unex_empty() || world.rank() > 0){//while starts

            if (world.rank() == 0){
        	   int dummy = 1;
        	   for (int w = 1; w < world.size(); w++){
        	        mpi::request dum_req = world.isend(w, 14, dummy);
        	        dum_req.wait();
        	    }
            }
            else {
            	//If we don't have the go-ahead, check for the Finalize message.
            	while(!world.iprobe(0, 14)){
            	    if(world.iprobe(0, 15)){
            		  mpi_exit = 1;
            		  break;
            	    }
            	}
	            if(mpi_exit == 1)
	               break;
            	//Receive the message so it's not just sitting in the queue.
            	int dummy;
            	world.recv(0, 14, dummy);
            }

            //We need to refill the localFrontier with states from the database if it's empty
            //TODO: Investigate if it's faster to read from DB directly into Node 0
        	//Or if faster to have worker node read from DB, then comm to Node 0
            if(localFrontier.empty() && world.rank() == 0) {
            	struct timeval t01,t02;
            	gettimeofday(&t01,NULL);
            	task_zero(instance, localFrontier, mem_threshold);
            	gettimeofday(&t02,NULL);
            	total_task0+=(t02.tv_sec-t01.tv_sec)*1000.0+(t02.tv_usec-t01.tv_usec)/1000.0;
            }

            //Task 0 to Task 1 Communication
            if(world.rank() == 0)
            {
    			auto current_state = localFrontier.back();
    			localFrontier.pop_back();
    			for(int l=0; l < alloc; l++){
    				mpi::request cs_req = world.isend(send_check(world, l), 20, current_state); 
    				cs_req.wait();
    			}	
            }

            //Execute Task 1
            if (world.rank() > 0 && world.rank() <= alloc){
            	NetworkState current_state;
            	world.recv(mpi::any_source, 20, current_state);

            	struct timeval t11,t12;
            	gettimeofday(&t11,NULL);
            	task_one(instance, current_state, exploit_list, od_map, alloc, two_alloc, reduc_factor, num_tasks, world, tcomm);
            	gettimeofday(&t12,NULL);
            	total_task1+=(t12.tv_sec-t11.tv_sec)*1000.0+(t12.tv_usec-t11.tv_usec)/1000.0;
            }

            //Execute Task 2
            //Edge case: 2 nodes: This prevents node 1 from accessing the execution of task 2
            if((world.size() == 2 && world.rank() == 0) || world.size() > 2)
            {
        	   if(world.rank() >= send_check(world, alloc) && world.rank() <= (alloc+two_alloc))
	           {
            	    //Execute Task 2
            	    struct timeval t21,t22;
            	    gettimeofday(&t21,NULL);
            	    task_two(instance, alloc, two_alloc, world, localFrontier, mem_threshold,\
            		ttwo_comm, ex_groups, hash_map, edge_map);

            	    //Wait for all Task 2 nodes to finish
            	    while(ttwo_comm.iprobe(mpi::any_source, 3) || ttwo_comm.iprobe(mpi::any_source, 4)){
                		NetworkState new_state;
                		Quality fact;

                		ttwo_comm.recv(mpi::any_source, 3, fact);
                		ttwo_comm.recv(mpi::any_source, 4, new_state);

                		instance.facts.hash_table[new_state.compound_assign(fact)]=instance.facts.size();
                		instance.facts.length++;
                		instance.facts.str_vector.push_back(new_state.compound_assign(fact));
            	    }
            	    
                    ttwo_comm.barrier();
            	    gettimeofday(&t22,NULL);
            	    total_task2+=(t22.tv_sec-t21.tv_sec)*1000.0+(t22.tv_usec-t21.tv_usec)/1000.0;

            	    //Have the 0th Task 2 node tell the other world nodes that it's done
            	    if(ttwo_comm.rank() == 0){
                		for (int w = 0; w < world.size(); w++)
                		{
                		    //2 Node Edge case
                		    if(world.size() == 2){
                    			mpi::request tdone_req = world.isend(1, 2, 1);
                    			//tdone_req.wait();
                    			break;
                		    }

                		    else if(w < world.rank() || w > alloc+two_alloc)
                		    {
                			    mpi::request tdone_req = world.isend(w, 2, 1);
                			    //tdone_req.wait();
                		    }

                		}
            	    }
	            }
    
            	else{
            	    //If we haven't been told that task 2 is finished, and if we still more facts or states to update:        
            	    int finished_signal = 0;
            	    if(world.iprobe(mpi::any_source, 2)){
            		    world.recv(mpi::any_source, 2, finished_signal);
            	    }

	                while(!finished_signal || world.iprobe(mpi::any_source, 3) || world.iprobe(mpi::any_source, 4) || world.iprobe(mpi::any_source, 5))
	                {
                		//Check for the finished signal
                		if(world.iprobe(mpi::any_source, 2)){
                		    //Prevent double-pulling
                		    if(!finished_signal){
                    			world.recv(mpi::any_source, 2, finished_signal);
                		    }
               			}

                		//If we get a new fact and new state, update
                		if(world.iprobe(mpi::any_source, 3) && world.iprobe(mpi::any_source, 4)){
                		    NetworkState new_state;
                		    Quality fact;

                		    world.recv(mpi::any_source, 3, new_state);
                		    world.recv(mpi::any_source, 4, fact);

                		    instance.facts.hash_table[new_state.compound_assign(fact)]=instance.facts.size();
                		    instance.facts.length++;
                		    instance.facts.str_vector.push_back(new_state.compound_assign(fact));
                		}

                		if(world.rank() == 0){
                		    if(world.iprobe(mpi::any_source, 5)){
                    			NetworkState critical_state;
                    			NetworkState current_state;
                    			Exploit exploit;
                    			AssetGroup assetGroup;
                    			world.recv(mpi::any_source, 5, critical_state);
                    			world.recv(mpi::any_source, 6, current_state);
                    			world.recv(mpi::any_source, 10, exploit);
                    			world.recv(mpi::any_source, 11, assetGroup);

            					if(instance.factbases.size() == 0){
                                	last_known_id = 0;
                            	}
                			    else last_known_id = instance.factbases.size()-1;
                			    if(get_num_states() > 1)
                    			    last_known_id += get_num_states()-1;

                    			struct timeval t31,t32;
                    			gettimeofday(&t31,NULL);
                    			task_three(instance, critical_state, localFrontier, mem_threshold, world,\
                    			    alloc, two_alloc, current_state, exploit, assetGroup, hash_map, last_known_id, edge_map);
                    			gettimeofday(&t32,NULL);
                    			total_task3+=(t32.tv_sec-t31.tv_sec)*1000.0+(t32.tv_usec-t31.tv_usec)/1000.0;
                		    }
                		} //Node 0 Task 3 wait ends
	                } //while probe/instance update ends
	            } //else wait ends
            }
            //Task Four
            if (world.rank() == send_check(world, alloc+two_alloc)){
            	if(world.iprobe(0, 7) || world.iprobe(0, 8)){
            	    std::vector<Factbase> factbases_dump;
            	    std::vector<Edge> edges_dump;
            	    world.recv(0, 7, factbases_dump);
            	    world.recv(0, 8, edges_dump);

            	    instance.factbases = factbases_dump;
            	    instance.edges = edges_dump;
            	    struct timeval t41,t42;
            	    gettimeofday(&t41,NULL);
            	    save_ag_to_db(instance, true);
            	    std::vector<Factbase>().swap(instance.factbases);
            	    std::vector<FactbaseItems>().swap(instance.factbase_items);
            	    std::vector<Edge>().swap(instance.edges);
            	    gettimeofday(&t42,NULL);
            	    total_task4+=(t42.tv_sec-t41.tv_sec)*1000.0+(t42.tv_usec-t41.tv_usec)/1000.0;
            	}
            }

            //Task Five
            if (world.rank() == 0){
        	   double f_alpha;
        	   if (!localFrontier.empty())
        	       f_alpha = (static_cast<double>(localFrontier.size()) * (localFrontier.back().get_size()))/tot_sys_mem;
        	   else
        	       f_alpha = 0.0;

            	if (f_alpha >= (mem_threshold/2)) {
            	    int q_eles = localFrontier.size();
            	    for (int q = 0; q < (q_eles/2); q++) {
                		auto rem_state = localFrontier.front();
                		localFrontier.pop_front();
                		mpi::request rem_req = world.isend(send_check(world, alloc+two_alloc+1), 50, rem_state);
                		rem_req.wait();
            	    }
            	    if (!localFrontier.empty())
            		    f_alpha = (static_cast<double>(localFrontier.size()) * (localFrontier.back().get_size()))/tot_sys_mem;
            	    else
            		    f_alpha = 0;
            	}
            }

            if(world.rank() == send_check(world, alloc+two_alloc+1)){
                struct timeval t51,t52;
                gettimeofday(&t51,NULL);
            	while(world.iprobe(0, 50)){
            	    NetworkState save_state;
            	    world.recv(0, 50, save_state);
            	    save_unexplored_to_db(save_state);
            	}
                gettimeofday(&t52,NULL);
                total_task5+=(t52.tv_sec-t51.tv_sec)*1000.0+(t52.tv_usec-t51.tv_usec)/1000.0;
            }
        } //while ends

        if(world.rank() == 0){
            for (int w = 1; w < world.size(); w++){
        	    mpi::request proc_req = world.isend(w, 15, 1);
        	    proc_req.wait();
        	}
        }

        if (mpi_exit == 1){
            #pragma omp cancel for
        }

        auto ag_end = std::chrono::system_clock::now();
    }//OpenMP block ends

    //Wait for db ops to finish
    world.barrier();
    if(world.rank() == 0){
        gettimeofday(&t2,NULL);
        total_t+=(t2.tv_sec-t1.tv_sec)*1000.0+(t2.tv_usec-t1.tv_usec)/1000.0;
        printf("AG TOOK %lf ms.\n", total_t);
    }

    auto end = std::chrono::system_clock::now();

    if(world.rank() != 0){
        if (total_task1 > 0.0)
    	   world.isend(0, 24, total_task1);
        if (total_task2 > 0.0)
    	   world.isend(0, 25, total_task2);
        if (total_task3 > 0.0)
    	   world.isend(0, 26, total_task3);
        if (total_task4 > 0.0)
    	   world.isend(0, 27, total_task4);
        if (total_task5 > 0.0)
            world.isend(0, 28, total_task5);
    }
    world.barrier();
    if(world.rank() == 0){
        double task1_count, task2_count, task3_count, task4_count, task5_count = 0;
        while(world.iprobe(mpi::any_source, 24)){
            double tmp;
            world.recv(mpi::any_source, 24, tmp);
            total_task1 += tmp;
            task1_count++;
        }
        while(world.iprobe(mpi::any_source, 25)){
            double tmp;
            world.recv(mpi::any_source, 25, tmp);
            total_task2 += tmp;
            task2_count++;
        }
        while(world.iprobe(mpi::any_source, 26)){
            double tmp;
            world.recv(mpi::any_source, 26, tmp);
            total_task3 += tmp;
            task3_count++;
        }
        while(world.iprobe(mpi::any_source, 27)){
            double tmp;
            world.recv(mpi::any_source, 27, tmp);
            total_task4 += tmp;
            task4_count++;
        }
        while(world.iprobe(mpi::any_source, 28)){
            double tmp;
            world.recv(mpi::any_source, 28, tmp);
            total_task5 += tmp;
            task5_count++;
        }

        std::cout << "Average Task 0 time: " << total_task0 << "ms" << std::endl;
        std::cout << "Average Task 1 time: " << total_task1/task1_count << "ms" << std::endl;
        std::cout << "Average Task 2 time: " << total_task2/task2_count << "ms" << std::endl;
        std::cout << "Average Task 3 time: " << total_task3/task3_count << "ms" << std::endl;
        std::cout << "Average Task 4 time: " << total_task4/task4_count << "ms" << std::endl;
        std::cout << "Average Task 5 time: " << total_task5/task5_count << "ms" << std::endl;

        //edge_adjustment(instance, hash_map);
    }

    std::chrono::duration<double> elapsed_seconds = end - start;
    instance.elapsed_seconds = elapsed_seconds;

    return instance;
}

AGGenInstance &AGGen::omp_generate(bool batch_process, int batch_num, int numThrd,\
int initQSize, double mem_threshold, mpi::communicator &world)
{
    std::vector<Exploit> exploit_list = instance.exploits;

    //Create a vector that contains all the groups of exploits to be fired synchonously
    std::vector<std::string> ex_groups;
    for (const auto &ex : exploit_list) {
    //If the group isn't already in the vector  
        if(!(std::find(ex_groups.begin(), ex_groups.end(), ex.get_group()) !=ex_groups.end())) {
            //Don't include the "no" group
            if(ex.get_group()!="null")
        	   ex_groups.emplace_back(ex.get_group());                        
        }         
    }

    //Print out the groups if desired
    std::cout <<"\nThere are "<<ex_groups.size()<<" groups: ";
    for(int i=0; i<ex_groups.size(); i++){
        std::cout<<ex_groups[i] << ". ";
    }
    std::cout<<"\n";

    auto start = std::chrono::system_clock::now();

    unsigned long esize = exploit_list.size();
    printf("esize: %lu\n", esize);
    bool save_queued = false;

    if(world.rank() == 0)
        std::cout << "Generating Attack Graph through the OMP/Serial Approach" << std::endl;

    std::unordered_map<size_t, PermSet<size_t>> od_map;
    size_t assets_size = instance.assets.size();
    for (const auto &ex : exploit_list) {
        size_t num_params = ex.get_num_params();
        if (od_map.find(num_params) == od_map.end()) {
            Odometer<size_t> od(num_params, assets_size);
            od_map[num_params] = od.get_all();
        }
    }

    //might be where to apply parallelization.
    while (frontier.size()<initQSize){//while starts, test multiple thread case
    //while (frontier.size()!=0){//while starts, test single thread case
        auto current_state = frontier.back();
        auto current_hash = current_state.get_hash(instance.facts);
        frontier.pop_back();
        std::vector<std::tuple<Exploit, AssetGroup>> appl_exploits;
        for (size_t i = 0; i < esize; i++){//for loop for applicable exploits starts
            auto e = exploit_list.at(i);
            size_t num_params = e.get_num_params();
            auto preconds_q = e.precond_list_q();
            auto preconds_t = e.precond_list_t();
            auto perms = od_map[num_params];
            std::vector<AssetGroup> asset_groups;
            for (auto perm : perms) {
            	std::vector<Quality> asset_group_quals;
            	std::vector<Topology> asset_group_topos;
            	asset_group_quals.reserve(preconds_q.size());
            	asset_group_topos.reserve(preconds_t.size());
            	for (auto &precond : preconds_q) {
            	    asset_group_quals.emplace_back(
            		  perm[precond.get_param_num()], precond.name, precond.op,
            		  precond.value, instance.facts);
            	}
            	for (auto &precond : preconds_t) {
            	    auto dir = precond.get_dir();
            	    auto prop = precond.get_property();
            	    auto op = precond.get_operation();
            	    auto val = precond.get_value();
            	    asset_group_topos.emplace_back(
            		  perm[precond.get_from_param()],
            		  perm[precond.get_to_param()], dir, prop, op, val, instance.facts);
            	}
        	    asset_groups.emplace_back(asset_group_quals, asset_group_topos, perm);
            }
            auto assetgroup_size = asset_groups.size();
            for (size_t j = 0; j < assetgroup_size; j++) {
            	auto asset_group = asset_groups.at(j);
            	for (auto &quality : asset_group.get_hypo_quals()) {
            	    if (!current_state.get_factbase().find_quality(quality)) {
            		    goto LOOPCONTINUE;
            	    }
            	}
            	for (auto &topology : asset_group.get_hypo_topos()) {
            	    if (!current_state.get_factbase().find_topology(topology)) {
            		  goto LOOPCONTINUE;
            	    }
        	    }

                {
        	    auto new_appl_exploit = std::make_tuple(e, asset_group);
        	    appl_exploits.push_back(new_appl_exploit);
                }
                LOOPCONTINUE:;      	
            }
        } //for loop for applicable exploits ends

        auto appl_expl_size = appl_exploits.size();
        for (size_t j = 0; j < appl_expl_size; j++) { //for loop for new states starts
            auto e = appl_exploits.at(j);
            auto exploit = std::get<0>(e);
            auto assetGroup = std::get<1>(e);
            auto postconditions = createPostConditions(e, instance.facts);
            auto qualities = std::get<0>(postconditions);
            auto topologies = std::get<1>(postconditions);
            NetworkState new_state{current_state};
            for(auto &qual : qualities) {
            	auto action = std::get<0>(qual);
            	auto fact = std::get<1>(qual);
            	switch(action) {
                	case ADD_T:
                	    new_state.add_quality(fact);
                	    break;
                	case UPDATE_T:
                	    new_state.update_quality(fact);
                	    break;
                	case DELETE_T:
                	    new_state.delete_quality(fact);
                	    break;
            	}
            }
            for(auto &topo : topologies) {
            	auto action = std::get<0>(topo);
            	auto fact = std::get<1>(topo);
            	switch(action) {
                	case ADD_T:
                	    new_state.add_topology(fact);
                	    break;
                	case UPDATE_T:
                	    new_state.update_topology(fact);
                	    break;
                	case DELETE_T:
                	    new_state.delete_topology(fact);
                	    break;
            	}
            }

            auto hash_num = new_state.get_hash(instance.facts);
            if (hash_num == current_hash)
        	   continue;
            if (hash_map.find(hash_num) == hash_map.end()) {
        	    new_state.set_id();
        	    auto facts_tuple = new_state.get_factbase().get_facts_tuple();
        	    FactbaseItems new_items =
        		std::make_tuple(facts_tuple, new_state.get_id());
        	    instance.factbase_items.push_back(new_items);
        	    instance.factbases.push_back(new_state.get_factbase());
        	    hash_map.insert(std::make_pair(new_state.get_hash(instance.facts), new_state.get_id()));
        	    frontier.emplace_front(new_state);
        	    Edge ed(current_state.get_id(), new_state.get_id(), exploit, assetGroup);
        	    ed.set_id();
                edge_adjustment(instance, hash_map, edge_map, ed);
            }   
            else {
        	    int id = hash_map[hash_num];
        	    Edge ed(current_state.get_id(), id, exploit, assetGroup);
        	    ed.set_id();
                edge_adjustment(instance, hash_map, edge_map, ed);
            }
        } //for loop for new states ends
    } //while ends

    //int numThrd=32;
    if(world.rank() == 0){
        printf("The number of threads used is %d\n",numThrd);
        printf("The initial QSize is %d\n",initQSize); 
    }

    int frt_size=frontier.size();
    if(world.rank() ==0)
        printf("The actual QSize to start using multiple threads is %d\n",frt_size); 


    double total_t=0.0;
    struct timeval t1,t2;
    gettimeofday(&t1,NULL);
    //#pragma omp parallel for num_threads(numThrd) default(none) shared(esize,counter,exploit_list,od_map,frt_size,total_t,t1,t2) schedule(dynamic,1)
    #pragma omp parallel for num_threads(numThrd) default(none) shared(esize,exploit_list,od_map,frt_size,total_t,t1,t2,std::cout, mem_threshold, ex_groups) schedule(dynamic,1)
    //auto ag_start = std::chrono::system_clock::now();
    for(int k=0;k<frt_size;k++){

        double f_alpha = 0.0;
        auto tot_sys_mem = getTotalSystemMemory();
        std::deque<NetworkState> localFrontier;
        localFrontier.emplace_front(frontier[k]);
        while (!localFrontier.empty() || !unex_empty()){//while starts
            //We need to refill the localFrontier with states from the database if it's empty
            if(localFrontier.empty()) {
            	double total_tt = 0.0;
            	struct timeval tt1,tt2;
            	gettimeofday(&tt1,NULL);
            	int retrv_counter = 0;
    	
            	//TODO: One (or a few) larger queries to pull in new states, rather than single queries that pull states one-by-one
            	do {
            	    NetworkState db_new_state = fetch_unexplored(instance.facts);
            	    localFrontier.emplace_front(db_new_state);
            	    //alpha = get_alpha();
            	    f_alpha = (static_cast<double>(localFrontier.size()) * (localFrontier.back().get_size()))/tot_sys_mem;
            	    retrv_counter += 1;
            	}
            	//Leave a 30% buffer in alpha
            	while((f_alpha <= (mem_threshold * 0.7)) && !unex_empty());

            	gettimeofday(&tt2,NULL);
            	total_tt+=(tt2.tv_sec-tt1.tv_sec)*1000.0+(tt2.tv_usec-tt1.tv_usec)/1000.0;
            	//printf("Retrieving from db took %lf s.\n", total_tt);
            }

            auto current_state = localFrontier.back();
            auto current_hash = current_state.get_hash(instance.facts);
            localFrontier.pop_back();
            std::vector<std::tuple<Exploit, AssetGroup>> appl_exploits;
            for (size_t i = 0; i < esize; i++) {//for loop for applicable exploits starts
            	auto e = exploit_list.at(i);
            	size_t num_params = e.get_num_params();
            	auto preconds_q = e.precond_list_q();
            	auto preconds_t = e.precond_list_t();
            	auto perms = od_map[num_params];
            	std::vector<AssetGroup> asset_groups;
            	for (auto perm : perms) {
            	    std::vector<Quality> asset_group_quals;
            	    std::vector<Topology> asset_group_topos;
            	    asset_group_quals.reserve(preconds_q.size());
            	    asset_group_topos.reserve(preconds_t.size());

	                for (auto &precond : preconds_q) {
               		    asset_group_quals.emplace_back(
            		    perm[precond.get_param_num()], precond.name, precond.op,
            		    precond.value, instance.facts);
            	    }
	               for (auto &precond : preconds_t) {
                	    auto dir = precond.get_dir();
                		auto prop = precond.get_property();
                		auto op = precond.get_operation();
                		auto val = precond.get_value();
                		asset_group_topos.emplace_back(
                		    perm[precond.get_from_param()],
                		    perm[precond.get_to_param()], dir, prop, op, val, instance.facts);
                	}
	                asset_groups.emplace_back(asset_group_quals, asset_group_topos, perm);
	            }
	            auto assetgroup_size = asset_groups.size();
	            for (size_t j = 0; j < assetgroup_size; j++) {
	                auto asset_group = asset_groups.at(j);
	                for (auto &quality : asset_group.get_hypo_quals()) {
                		if (!current_state.get_factbase().find_quality(quality)) {
                		    goto LOOPCONTINUE1;
                		}
	                }
	                for (auto &topology : asset_group.get_hypo_topos()) {
                		if (!current_state.get_factbase().find_topology(topology)) {
                		    goto LOOPCONTINUE1;
                		}
	                }
	    
                    {
            		auto new_appl_exploit = std::make_tuple(e, asset_group);
            		appl_exploits.push_back(new_appl_exploit);
	                }
	                LOOPCONTINUE1:;
	            }
            } //for loop for creating applicable exploits ends

            std::map<std::string, int> group_fired; //Map to hold fired status per group
            std::map<std::string, std::vector<std::tuple<Exploit, AssetGroup>>> sync_vectors; //Map to hold all group exploits

            for (auto map_group : ex_groups)
            {
        	group_fired.insert(std::pair<std::string, int> (map_group, 0));
            }

            //Build up the map of synchronous fire exploits
            for(auto itr=appl_exploits.begin(); itr!=appl_exploits.end(); itr++){
            	//auto e = appl_exploits.at(itr);
            	auto e = *itr;
            	auto egroup = std::get<0>(e).get_group();

            	if (egroup != "null"){
            	    sync_vectors[egroup].push_back(e);
            	}
            }  

            //loop through the vector
            for(auto itr=appl_exploits.begin(); itr!=appl_exploits.end(); itr++){
            	auto e = *itr;
            	auto exploit = std::get<0>(e);
            	auto assetGroup = std::get<1>(e);
            	//std::cout<<exploit.get_name()<<std::endl;

            	auto egroup=exploit.get_group();

            	if ((egroup != "null" && group_fired[egroup] == 0) || egroup == "null"){
            	    NetworkState new_state{current_state};
            	    std::vector<std::tuple<Exploit, AssetGroup>> sync_exploits;

            	    if (egroup == "null")
            		    sync_exploits.push_back(e);

            	    else {
            		    sync_exploits = sync_vectors[egroup];

                		//TODO: Does not work if only some assets belong to a group. This only works if
                		//all assets are in the group
                		if(sync_exploits.size() < instance.assets.size()){
                		    continue;
                		}
	                }

            	    for(auto sync_itr=sync_exploits.begin(); sync_itr!=sync_exploits.end(); sync_itr++){
                		e = *sync_itr;
                		exploit = std::get<0>(e);
                		egroup=exploit.get_group();
                		assetGroup = std::get<1>(e);
                		group_fired[egroup] = 1;

                		auto postconditions = createPostConditions(e, instance.facts);
                		auto qualities = std::get<0>(postconditions);
                		auto topologies = std::get<1>(postconditions);
		
                		for(auto &qual : qualities) {
                		    auto action = std::get<0>(qual);
                		    auto fact = std::get<1>(qual);
                		    switch(action) {
                    			case ADD_T:
                    			    new_state.add_quality(fact);
                    				break;
                    			case UPDATE_T:
                    			    new_state.update_quality(fact);
			                        break;
                    			case DELETE_T:
                    			    new_state.delete_quality(fact);
                    			    break;
                    		}
		                }

                		for(auto &topo : topologies) {
                		    auto action = std::get<0>(topo);
                		    auto fact = std::get<1>(topo);
                		    switch(action) {
                			case ADD_T:
                			    new_state.add_topology(fact);
                			    break;
                			case UPDATE_T:
                			    new_state.update_topology(fact);
                			    break;
                			case DELETE_T:
                			    new_state.delete_topology(fact);
                			    break;
                		    }
                		}    
	                }//Sync. Fire for            

	                auto hash_num = new_state.get_hash(instance.facts);
	
	                if (hash_num == current_hash)
		                continue;

	                #pragma omp critical
                	    //although local frontier is updated, the global hash is also updated to avoid testing on explored states.
                	    if (hash_map.find(hash_num) == hash_map.end()) {
                		    new_state.set_id();
                    		auto facts_tuple = new_state.get_factbase().get_facts_tuple();
                    		FactbaseItems new_items = std::make_tuple(facts_tuple, new_state.get_id());
                    		instance.factbase_items.push_back(new_items);
                    		instance.factbases.push_back(new_state.get_factbase());
                    		hash_map.insert(std::make_pair(new_state.get_hash(instance.facts), new_state.get_id()));
                		    
                    		//See memory usage. If it exceeds the threshold, store new states in the DB
                    		double i_alpha = 0.0;
                    		//Get the most recent Factbase's size * total number of factbases, rough approximation of *2 to account for factbase_items
                    		double i_usage = instance.factbases.back().get_size() * instance.factbases.size() * 2 + sizeof(instance.edges[0]) * instance.edges.size();

                    		i_alpha = i_usage/tot_sys_mem;
                    		double f_alpha;
                    		if (!localFrontier.empty())
                    		    f_alpha = (static_cast<double>(localFrontier.size()) * (localFrontier.back().get_size()))/tot_sys_mem;
                    		else
                    		    f_alpha = 0.0;

                    		if (f_alpha >= (mem_threshold/2)) {
                    		    save_unexplored_to_db(new_state);
                    		    if (!localFrontier.empty())
                    			f_alpha = (static_cast<double>(localFrontier.size()) * (localFrontier.back().get_size()))/tot_sys_mem;
                    		    else
                    			f_alpha = 0;
                    		}

                    		//Store new state in database to ensure proper ordering of the FIFO queue
                    		else if (!unex_empty()){
                    		    save_unexplored_to_db(new_state);
                    		}

                    		//Otherwise, we can just store in memory
                    		else {
                    		    localFrontier.emplace_front(new_state);
                    		}

                    		if (i_alpha >= mem_threshold/2){
                                //edge_adjustment(instance, hash_map);
                    		    save_ag_to_db(instance, true);

                    		    //Clear vectors and free memory
                    		    std::vector<Factbase>().swap(instance.factbases);
                    		    std::vector<FactbaseItems>().swap(instance.factbase_items);
                    		    std::vector<Edge>().swap(instance.edges);
                    		    
                    		    i_usage = (sizeof(instance.factbases) + (sizeof(instance.factbases[0]) * instance.factbases.size()) +\
                    				sizeof(instance.factbase_items) + (sizeof(instance.factbase_items[0]) * instance.factbase_items.size()) +\
                    				sizeof(instance.edges) + (sizeof(instance.edges[0]) * instance.edges.size()));
                    		    i_alpha = i_usage/tot_sys_mem;
                    		}
				    
                    		Edge ed(current_state.get_id(), new_state.get_id(), exploit, assetGroup);
                    		ed.set_id();
                    		instance.edges.push_back(ed);
                    	} //END if (hash_map.find(hash_num) == hash_map.end())   

	                   else {
                		    int id = hash_map[hash_num];
                		    Edge ed(current_state.get_id(), id, exploit, assetGroup);
                		    ed.set_id();
                		    instance.edges.push_back(ed);
                	    }    
	            } //sync fire if
	            else
	                continue;
            } //for loop for new states ends
        } //while frontier ends
        auto ag_end= std::chrono::system_clock::now();
    }//OpenMP block ends
    gettimeofday(&t2,NULL);
    total_t+=(t2.tv_sec-t1.tv_sec)*1000.0+(t2.tv_usec-t1.tv_usec)/1000.0;
    if(world.rank() == 0)
        printf("AG TOOK %lf ms.\n", total_t);

    auto end = std::chrono::system_clock::now();

    std::chrono::duration<double> elapsed_seconds = end - start;
    instance.elapsed_seconds = elapsed_seconds;

    return instance;
}

AGGenInstance &AGGen::sg_generate(bool use_bcl, bool batch_process, int batch_num, int numThrd,\
int initQSize, double mem_threshold, mpi::communicator &world, int state_limit, int bclhash_size)
{
    //Init all Nodes with these variables
    std::vector<Exploit> exploit_list = instance.exploits;

    //Distributed Hash maps for factbase hashes and explored hashes
    //NOTE: At this time, BCL does not support realloc or resizing its containers. This should ideally be changed once support is added.
    BCL::HashMap<size_t, int> BCLmap(bclhash_size);
    BCL::HashMap<size_t, int> BCLexpl(bclhash_size);

    //Create a vector that contains all the groups of exploits to be fired synchonously
    std::vector<std::string> ex_groups;
    for (const auto &ex : exploit_list) {
        //If the group isn't already in the vector  
        if(!(std::find(ex_groups.begin(), ex_groups.end(), ex.get_group()) !=ex_groups.end())) {
            //Don't include the "no" group
            if(ex.get_group()!="null")
        	   ex_groups.emplace_back(ex.get_group());                        
        }      
    }

    //Print out the groups if desired
    if (world.rank() == 0){
        std::cout <<"\nThere are "<<ex_groups.size()<<" groups: ";
        for(int i=0; i<ex_groups.size(); i++){
            std::cout<<ex_groups[i] << ". ";
        }
        std::cout<<"\n";
    }

    auto counter = 0;
    auto start = std::chrono::system_clock::now();

    unsigned long esize = exploit_list.size();
    if (world.rank() ==0){
        printf("esize: %lu\n", esize);
    }
    bool save_queued = false;

    if (world.rank() == 0){
        if (use_bcl)
            std::cout << "Generating Attack Graph Using MPI Subgraphing with a DHT" << std::endl;
        else
            std::cout << "Generating Attack Graph Using MPI Subgraphing without a DHT" << std::endl;
    }

    std::unordered_map<size_t, PermSet<size_t>> od_map;
    size_t assets_size = instance.assets.size();
    for (const auto &ex : exploit_list) {
        size_t num_params = ex.get_num_params();
        if (od_map.find(num_params) == od_map.end()) {
            Odometer<size_t> od(num_params, assets_size);
            od_map[num_params] = od.get_all();
        }
    }

    int frt_size=frontier.size();
    if (world.rank() == 0) {
        printf("The actual QSize to start using multiple threads is %d\n",frt_size); 
    }

    double total_t = 0.0;
    //unit:ms
    double total_task0, total_task1, total_task2, total_task3, total_task4 = 0.0;

    struct timeval t1,t2;
    gettimeofday(&t1,NULL);

    std::deque<NetworkState> localFrontier;
    if(world.rank() == 0){
        double f_alpha = 0.0;
        auto tot_sys_mem = getTotalSystemMemory();
        localFrontier.emplace_front(frontier[0]);
    }

    int finished_signal = 0;
    int send_msg = 0;
    int state_count = 0;
    int ns_inc = 0;
    int first_loop = 1;
    int msg_sends = 0;
  
    std::unordered_set<size_t> front_expl;

    std::random_device rd; // obtain a random number from hardware
    std::mt19937 gen(rd()); // seed the generator

    //Make separate comm to not sync with db node
    mpi::communicator work_comm = world.split(world.rank() != 1);

    //std::unordered_set<NetworkState> localFrontier_seen;
    int last_known_id = 0;
    //Send new Network State to all worker nodes, if we have enough unex states to do so
    if(world.rank() == 0){
        for (int w = 0; w < std::min((int)world.size(), (int)localFrontier.size()); w++){
		    //last_known_id = localFrontier.front().get_id();
            if(world.size() > 1 && w == 0){
                continue;
            }
            if(world.size() >= 3 && w == 1){
                    continue;
            }   

            mpi::request state_req = world.isend(w, 1, localFrontier.front());
            state_req.wait();

            localFrontier.pop_front();
	        msg_sends++;
        }	
    }

    //Array of MPI Status requests that we'll use for waiting on non-blocking sends.
    mpi::request requests[20];

    //Main Work Loop - Just make this a do while instead of first_loop
    while(!localFrontier.empty() || finished_signal == 0 || world.rank() > 0 || first_loop == 1){
        first_loop = 0;
	    //Refill localFrontier if needed
        if(localFrontier.empty() && world.rank() == 0 && !unex_empty()) {
            task_zero(instance, localFrontier, mem_threshold);
        }

        //Worker Nodes
        if (world.rank() > 1 || world.size() == 1 || (world.size() == 2 && world.rank() == 1)){

            world.probe(0, mpi::any_tag);

            //Check for finished signal
            if(world.iprobe(0, 99)){
                break;
            }

            //We get a new state and need to do work
            if (world.iprobe(0, 1)){
                //Start Fresh each iteration
                localFrontier.clear();
                std::vector<Factbase>().swap(instance.factbases);       
                std::vector<Edge>().swap(instance.edges);
                std::vector<FactbaseItems>().swap(instance.factbase_items);

                NetworkState current_state;
		        //current_state.reset_curr_id();
                world.recv(0, 1, current_state);
                state_count = 0;
                send_msg = 1;
		        localFrontier.emplace_front(current_state);
                while(!localFrontier.empty()){
		            if (state_count < state_limit){
                        //Do work
                        auto current_state = localFrontier.front();
                        auto current_hash = current_state.get_hash(instance.facts);
                        localFrontier.pop_front();
                        if (use_bcl){
                            if (BCLexpl.find(current_hash) != BCLexpl.end())
                                continue;
                        }
                        std::vector<std::tuple<Exploit, AssetGroup>> appl_exploits;
                        for (size_t i = 0; i < esize; i++) {//for loop for applicable exploits starts
                            auto e = exploit_list.at(i);
                            size_t num_params = e.get_num_params();
                            auto preconds_q = e.precond_list_q();
                            auto preconds_t = e.precond_list_t();
                            auto perms = od_map[num_params];
                            std::vector<AssetGroup> asset_groups;
                            for (auto perm : perms) {
                                std::vector<Quality> asset_group_quals;
                                std::vector<Topology> asset_group_topos;
                                asset_group_quals.reserve(preconds_q.size());
                                asset_group_topos.reserve(preconds_t.size());
                          
                                for (auto &precond : preconds_q) {
                                    asset_group_quals.emplace_back(
                                    perm[precond.get_param_num()], precond.name, precond.op,
                                    precond.value, instance.facts);
                                }
                                for (auto &precond : preconds_t) {
                                    auto dir = precond.get_dir();
                                    auto prop = precond.get_property();
                                    auto op = precond.get_operation();
                                    auto val = precond.get_value();
                                    asset_group_topos.emplace_back(
                                        perm[precond.get_from_param()],
                                        perm[precond.get_to_param()], dir, prop, op, val, instance.facts);
                                }
                                asset_groups.emplace_back(asset_group_quals, asset_group_topos,
                                                          perm);
                            }
                            auto assetgroup_size = asset_groups.size();
                            for (size_t j = 0; j < assetgroup_size; j++) {
                                auto asset_group = asset_groups.at(j);
                                for (auto &quality : asset_group.get_hypo_quals()) {
                                    if (!current_state.get_factbase().find_quality(quality)) {
                                        goto LOOPCONTINUE1;
                                    }
                                }
                                for (auto &topology : asset_group.get_hypo_topos()) {
                                    if (!current_state.get_factbase().find_topology(topology)) {
                                        goto LOOPCONTINUE1;
                                    }
                                }
                                {
                                    auto new_appl_exploit = std::make_tuple(e, asset_group);
                                    appl_exploits.push_back(new_appl_exploit);
                                }
                            LOOPCONTINUE1:;
                            }
                        } //for loop for creating applicable exploits ends

                        std::map<std::string, int> group_fired; //Map to hold fired status per group
                        std::map<std::string, std::vector<std::tuple<Exploit, AssetGroup>>> sync_vectors; //Map to hold all group exploits

                        for (auto map_group : ex_groups)
                        {
                            group_fired.insert(std::pair<std::string, int> (map_group, 0));
                        }

                        //Build up the map of synchronous fire exploits
                        for(auto itr=appl_exploits.begin(); itr!=appl_exploits.end(); itr++){
                            //auto e = appl_exploits.at(itr);
                            auto e = *itr;
                            auto egroup = std::get<0>(e).get_group();

                            if (egroup != "null"){
                                sync_vectors[egroup].push_back(e);
                            }
                        }  

                        //loop through the vector
                        for(auto itr=appl_exploits.begin(); itr!=appl_exploits.end(); itr++){

                            auto e = *itr;
                            auto exploit = std::get<0>(e);
                            auto assetGroup = std::get<1>(e);
                            //std::cout<<exploit.get_name()<<std::endl;

                            auto egroup=exploit.get_group();

                            if ((egroup != "null" && group_fired[egroup] == 0) || egroup == "null"){
                                NetworkState new_state{current_state};
                                std::vector<std::tuple<Exploit, AssetGroup>> sync_exploits;

                                if (egroup == "null")
                                    sync_exploits.push_back(e);

                                else {
                                    sync_exploits = sync_vectors[egroup];

                                    //TODO: Does not work if only some assets belong to a group. This only works if
                                    //all assets are in the group
                                    if(sync_exploits.size() < instance.assets.size()){
                                        state_count++;
                                        continue;
                                    }
                                }

                                for(auto sync_itr=sync_exploits.begin(); sync_itr!=sync_exploits.end(); sync_itr++){
                                    e = *sync_itr;
                                    exploit = std::get<0>(e);
                                    egroup=exploit.get_group();
                                    assetGroup = std::get<1>(e);
                                    group_fired[egroup] = 1;

                                    auto postconditions = createPostConditions(e, instance.facts);
                                    auto qualities = std::get<0>(postconditions);
                                    auto topologies = std::get<1>(postconditions);
                                    
                                    for(auto &qual : qualities) {
                                        auto action = std::get<0>(qual);
                                        auto fact = std::get<1>(qual);
                                        switch(action) {
                                            case ADD_T:
                                                new_state.add_quality(fact);
                                                    break;
                                            case UPDATE_T:
                                                new_state.update_quality(fact);
                                                break;
                                            case DELETE_T:
                                                new_state.delete_quality(fact);
                                                break;
                                        }
                                    }

                                    for(auto &topo : topologies) {
                                        auto action = std::get<0>(topo);
                                        auto fact = std::get<1>(topo);
                                        switch(action) {
                                            case ADD_T:
                                                new_state.add_topology(fact);
                                                break;
                                            case UPDATE_T:
                                                new_state.update_topology(fact);
                                                break;
                                            case DELETE_T:
                                                new_state.delete_topology(fact);
                                                break;
                                        }
                                    }    
                                }//Sync. Fire for            

                                auto hash_num = new_state.get_hash(instance.facts);
                            
                                if (hash_num == current_hash){
                                    state_count++;
                                    continue;
                                }

                               #pragma omp critical
                                //although local frontier is updated, the global hash is also updated to avoid testing on explored states.
                                //if (hash_map.find(hash_num) == hash_map.end()) {
                                if (use_bcl){
                                    if (BCLmap.find(hash_num) == BCLmap.end()) {
                                        //new_state.set_id();
                                        new_state.force_set_id(BCLmap.capacity());
                                        instance.factbases.push_back(new_state.get_factbase());
                                        //hash_map.insert(std::make_pair(new_state.get_hash(instance.facts), new_state.get_id()));
                                        BCLmap[hash_num] = new_state.get_id();

                                        localFrontier.emplace_front(new_state);
                                        //Edge ed(current_state.get_id(), new_state.get_id(), exploit, assetGroup);
                                        Edge ed(current_hash, hash_num, exploit, assetGroup);
                                        ed.set_id();
                                        instance.edges.push_back(ed);
                                    } //END if (hash_map.find(hash_num) == hash_map.end())
                                    else {
                                        //auto id = hash_map[hash_num];
                                        //Edge ed(current_state.get_id(), id, exploit, assetGroup);
                                        Edge ed(current_hash, hash_num, exploit, assetGroup);
                                        ed.set_id();
                                        instance.edges.push_back(ed);
                                    }
                                    BCLexpl[current_hash] = BCLexpl.capacity();
                                }
                                else{
                                    if (hash_map.find(hash_num) == hash_map.end()) {
                                        new_state.set_id();
                                        instance.factbases.push_back(new_state.get_factbase());
                                        hash_map.insert(std::make_pair(new_state.get_hash(instance.facts), new_state.get_id()));
                                        
                                        localFrontier.emplace_front(new_state);
                                        Edge ed(current_hash, hash_num, exploit, assetGroup);
                                        ed.set_id();
                                        instance.edges.push_back(ed);
                                    } //END if (hash_map.find(hash_num) == hash_map.end())
                                    else {
                                        //auto id = hash_map[hash_num];
                                        //Edge ed(current_state.get_id(), id, exploit, assetGroup);
                                        Edge ed(current_hash, hash_num, exploit, assetGroup);
                                        ed.set_id();
                                        instance.edges.push_back(ed);
                                    }
                                }
                            } //sync fire if
                            else
                                continue;
                        } //for loop for new states ends		
                        state_count++;
                    }
                    else{
                        break;
                    }
                }
            }

            //Let root node we finished, but only if we have done work since our last message
            if(send_msg == 1){
                send_msg = 0;

                if(!localFrontier.empty()){
                    requests[2] = world.isend(0, 3, localFrontier);
                }
                //Send new states and edges, then clear worker instance
		        if(!instance.factbases.empty()){	
        			requests[3] = world.isend(0, 10, instance.factbases);
		        }
		        if(!instance.edges.empty()){
        			requests[4] = world.isend(0, 11, instance.edges);
	            }

                //Finish Signal
		        requests[5] = world.isend(0, 2, 1);
            }

	        mpi::wait_all(requests, requests+5);
        } //end worker nodes

        //Database Node
        if (world.rank() == 1 || world.size() == 1){

            world.probe(0, mpi::any_tag);
            //Check for finished signal, assuming we have no more storage requests
            if(world.iprobe(0, 99) && !world.iprobe(0,7) && !world.iprobe(0,8)){
		        break;
            }
            
            //Check for instance storage requests
            if(world.iprobe(0, 7) || world.iprobe(0, 8)){
                std::vector<Factbase> factbases_dump;
                std::vector<Edge> edges_dump;
                world.recv(0, 7, factbases_dump);
                world.recv(0, 8, edges_dump);
                
                instance.factbases = factbases_dump;
                instance.edges = edges_dump;
                save_ag_to_db(instance, true);

                //Clear vectors and free memory
                std::vector<Factbase>().swap(instance.factbases);
                std::vector<FactbaseItems>().swap(instance.factbase_items);
                std::vector<Edge>().swap(instance.edges);
            }

            //Check for frontier storage requests
            while(world.iprobe(0, 50)){
                NetworkState save_state;
                world.recv(0, 50, save_state);
                save_unexplored_to_db(save_state);
            }
            
        } //end world rank =1

        //Root Node
        if (world.rank() == 0){
            std::map<int, int> deque_marker;
            int finish_count = 0;
            std::map<int, std::vector<Factbase>> node_factbases;
            std::map<int, std::vector<Edge>> node_edges;
            //Receive states and edges when nodes finish
            while(finish_count != msg_sends){
                while(world.iprobe(mpi::any_source, 10) || world.iprobe(mpi::any_source, 11)){
                    for(int w = 0; w < world.size(); w++){
    		            if(world.iprobe(w, 10)){
    	                    world.recv(w, 10, node_factbases[w]);
    		            }
    		            if(world.iprobe(w, 11)){
    	                    world.recv(w, 11, node_edges[w]);
    		            }
                        if(!node_factbases[w].empty() || !node_edges[w].empty()){
                            state_merge(node_factbases[w], node_edges[w], hash_map, instance, mem_threshold,\
                             world, edge_map);
                        }
                    }
                }

                //Nodes finish
                for(int w = 0; w < world.size(); w++){
                    int dummy_finish = 0;
                    if(world.iprobe(w, 2)){
                        world.recv(w, 2, dummy_finish);
                        finish_count++;
                    }
                }
            }

            //Check for factbases or nodes, and add them to our network
            while(world.iprobe(mpi::any_source, 10) || world.iprobe(mpi::any_source, 11)){
                for(int w = 0; w < world.size(); w++){
                    if(world.iprobe(w, 10)){
                        world.recv(w, 10, node_factbases[w]);
                    }
                    if(world.iprobe(w, 11)){
                        world.recv(w, 11, node_edges[w]);
                    }
                    if(!node_factbases[w].empty() || !node_edges[w].empty()){
                        state_merge(node_factbases[w], node_edges[w], hash_map, instance, mem_threshold,\
                         world, edge_map);
                    }
                }
            }

	        //Receive node frontiers and merge them into root frontier
            for(int w = 0; w < world.size(); w++){
                //If we have a big enough world, dedicate a node soley for db ops and a root
                if(world.size() >= 3){
                    if(w == 0 || w == 1){
                        continue;
                    }
                }
                if(world.iprobe(w, 3)){
                    std::deque<NetworkState> nodeFrontier; 
                    world.recv(w, 3, nodeFrontier);
                    std::unordered_set<size_t> tmp_expl;
                    std::vector<std::tuple<NetworkState, size_t>> insertFrontier;

                    /*
                    //Maintain a list of explored states
                    for (Factbase fb : node_factbases[w]){
                        auto fb_hash = fb.hash(instance.facts);
                        //Incorrectly declare that we've explored all states in our factbase vector
                        tmp_expl.insert(fb_hash);
                    }

                    for (NetworkState ns : nodeFrontier){
                        auto ns_hash = ns.get_hash(instance.facts);
                        //If we have a state on our local frontier that's in our local explored, we know we haven't actually explored it
                        if(tmp_expl.count(ns_hash) || front_expl.count(ns_hash)){
                            localFrontier.emplace_back(ns);
                            tmp_expl.erase(ns_hash);
                        }
                    }
                    
                
                    //Add our tmp_expl to our front_expl
                    for (auto tmp_hash : tmp_expl){
                        front_expl.insert(tmp_hash);
                    }
                    */
                    localFrontier.insert(std::end(localFrontier), std::begin(nodeFrontier), std::end(nodeFrontier));
                    deque_marker[w] = (int)(localFrontier.size()-1);
                }

                //If a node doesn't have a specific state to pull, randomly assign it one
                if (!deque_marker.count(w)){
                    //Randomly assign a state index to pop
                    if(!localFrontier.empty()){
                    	std::uniform_int_distribution<> distr(0, localFrontier.size()-1); // define the range
                    	deque_marker[w] = distr(gen);
    		        }
    		        else{
    			        deque_marker[w] = 0;
    		        }
                }
            }
		    	
		    msg_sends = 0;
            //Send new Network State to all worker nodes, if we have enough unex states to do so
            for (int w = 0; w < world.size(); w++){
                if(world.size() >= 3){
                    if(w == 0 || w == 1){
                        continue;
                    }
                }
                else if(world.size() == 2){
                    if(w == 0){
                        continue;
                    }
                }
                int proceed = 0;
                while(proceed == 0 && !localFrontier.empty()){
		            if(deque_marker[w] >= localFrontier.size()){
			            if(localFrontier.empty()){
			                //deque_marker[w] = 0;
			                break;
			            }
			            deque_marker[w] = localFrontier.size()-1;
		            }
                    if(!localFrontier.empty()){
                        auto deque_access = deque_marker[w];

                        NetworkState send_state = localFrontier.at(deque_access);
                        //Don't explore on states we already have explored
                        if(front_expl.find(send_state.get_hash(instance.facts)) != front_expl.end()){
                            if(localFrontier.size() == 1){
				                localFrontier.clear();
				                deque_marker[w] = 0;
			                }
			                else{
			    	            localFrontier.erase(localFrontier.begin()+deque_access);
                            	deque_marker[w] = deque_marker[w]-1;
			                }
                        }
			            else{
			                proceed = 1;   
                       	} 

                        if(proceed == 1){
			                //Reset State back to ID 0
			                send_state.force_set_id(0);
                            mpi::request state_req = world.isend(w, 1, send_state);
                            state_req.wait();

			                if(localFrontier.size() == 1){
				                localFrontier.clear();
			                }
			                else{
			                    localFrontier.erase(localFrontier.begin()+deque_access);
			                }
			                msg_sends++;
                        }
                    }
		
		            //Else localFrontier size = 0
		            else{
			            break;
		            }
                }
            }
	        //if LocalFrontier is empty at this point and we did not send any messages, we're done
            if(localFrontier.empty() && msg_sends == 0 && !world.iprobe(mpi::any_source, mpi::any_tag)){
                finished_signal = 1;
            }  
        } //end world rank 0


    } // end main work loop

    //Tell all nodes to finish
    if(world.rank() == 0){
        for(int w = 1; w < world.size(); w++){
            world.isend(w, 99, 1);
        }
    }

    world.barrier();
    if(world.rank() == 0){
        gettimeofday(&t2,NULL);
        total_t+=(t2.tv_sec-t1.tv_sec)*1000.0+(t2.tv_usec-t1.tv_usec)/1000.0;
        printf("AG TOOK %lf ms.\n", total_t);
        auto end = std::chrono::system_clock::now();

        std::chrono::duration<double> elapsed_seconds = end - start;
        instance.elapsed_seconds = elapsed_seconds;
    }
  
    return instance;
}
