UTulsa-Research/ag_gen
1
0
Fork 1
Code Issues Pull Requests Projects Releases 1 Wiki Activity

MPI Subgraphing

Browse Source
This commit is contained in:
hammer 2022-02-11 02:02:45 -06:00
parent 52aed47d00
commit daf5d6bbbc
16 changed files with 16648 additions and 2453 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
build/ag_gen
View File

Binary file not shown.

2
build/run.sh
View File

@ -58,7 +58,7 @@ if [ "$TYPE" == "$strval1" ]; then
if [ "$(dnsdomainname)" = "hammer.esg.utulsa.edu" ]; then
#4 Exploit Option
mpiexec --mca btl_openib_allow_ib 1 --mca btl openib,self,vader --mca opal_warn_on_missing_libcuda 0 --bind-to numa --map-by numa -np "$NODES" --timeout 129600 ./ag_gen -n ../Oct_2021/nm_files/"$CARS"_car_timeline_maintenance.nm -x ../Oct_2021/Sync/4_Exploits/"$NUM_SERV"_Serv/sync_timeline_maintenance.xp -t "$NUM_THREADS" -q 1 -p -a 0.6 -z "$DBNAME" -s -l 20
mpiexec --mca btl_openib_allow_ib 1 --mca btl openib,self,vader --mca opal_warn_on_missing_libcuda 0 --bind-to numa --map-by numa -np "$NODES" --timeout 129600 ./ag_gen -n ../Oct_2021/nm_files/"$CARS"_car_timeline_maintenance.nm -x ../Oct_2021/Sync/4_Exploits/"$NUM_SERV"_Serv/sync_timeline_maintenance.xp -t "$NUM_THREADS" -q 1 -p -a 0.6 -z "$DBNAME" -l 40 -s
#6 Exploit Option
#mpiexec --mca btl_openib_allow_ib 1 --mca btl openib,self,vader --mca opal_warn_on_missing_libcuda 0 --bind-to numa --map-by numa -np "$NODES" --timeout 129600 ./ag_gen -n ../Oct_2021/nm_files/"$CARS"_car_timeline_maintenance.nm -x ../Oct_2021/Sync/6_Exploits/"$NUM_SERV"_Serv/sync_timeline_maintenance.xp -t "$NUM_THREADS" -q 1 -p -a 0.6 -z "$DBNAME" -s -l 20
else

2
build/schema.sql
View File

@ -308,7 +308,7 @@ ALTER SEQUENCE public.factbase_id_seq OWNED BY public.factbase.id;
CREATE TABLE public.factbase_item (
factbase_id integer NOT NULL,
f bigint NOT NULL,
f integer NOT NULL,
type text NOT NULL
);

16574
build/states.txt
View File

File diff suppressed because it is too large Load Diff

2363
src/ag_gen/ag_gen.cpp
View File

File diff suppressed because it is too large Load Diff

14
src/ag_gen/edge.cpp
View File

@ -20,7 +20,7 @@
//Edge::Edge(int iFrom, int iTo, Exploit &ex, AssetGroup &ag)
// : from_node(iFrom), to_node(iTo), exploit(ex), assetGroup(ag), deleted(false) {}
Edge::Edge(size_t iFrom, size_t iTo, Exploit &ex, AssetGroup &ag)
Edge::Edge(int iFrom, int iTo, Exploit &ex, AssetGroup &ag)
: from_node(iFrom), to_node(iTo), exploit(ex), assetGroup(ag), deleted(false) {}
Edge::Edge()
@ -44,7 +44,7 @@ bool Edge::is_deleted() { return deleted; }
//}
size_t Edge::get_from_id()
int Edge::get_from_id()
{
return from_node;
}
@ -56,7 +56,7 @@ size_t Edge::get_from_id()
//}
size_t Edge::get_to_id()
int Edge::get_to_id()
{
return to_node;
}
@ -86,6 +86,14 @@ int Edge::set_id() {
return id;
}
void Edge::force_from_id(int i) {
from_node = i;
}
void Edge::force_to_id(int i) {
to_node = i;
}
int Edge::edge_current_id = 0;
/**

12
src/ag_gen/edge.h
View File

@ -62,8 +62,8 @@ class Edge {
class Edge {
static int edge_current_id;
int id;
size_t from_node;
size_t to_node;
int from_node;
int to_node;
Exploit exploit;
AssetGroup assetGroup;
bool deleted;
@ -77,7 +77,7 @@ class Edge {
}
public:
Edge(size_t, size_t, Exploit &, AssetGroup &);
Edge(int, int, Exploit &, AssetGroup &);
Edge();
std::string get_query();
@ -85,8 +85,10 @@ class Edge {
int get_id();
int set_id();
size_t get_from_id();
size_t get_to_id();
int get_from_id();
int get_to_id();
void force_from_id(int i);
void force_to_id(int i);
int get_exploit_id();
void set_deleted();
bool is_deleted();

14
src/ag_gen/factbase.cpp
View File

@ -49,12 +49,20 @@ void Factbase::force_set_id(size_t i) {
/**
* @return The current Factbase ID.
*/
size_t Factbase::get_id() const { return id; }
int Factbase::get_id() const { return id; }
std::tuple<std::vector<Quality>, std::vector<Topology>> Factbase::get_facts_tuple() const {
return std::make_tuple(qualities, topologies);
}
void Factbase::reset_curr_id() {
current_id = 0;
}
void Factbase::reset_curr_id(int i){
current_id = i;
}
/**
* @brief Searches for a Quality in the Factbase.
* @details Returns true if the Quality is found and false otherwise.
@ -168,13 +176,13 @@ void Factbase::delete_topology(Topology &t) {
* @param factlist The current Keyvalue
* @return The hash of the Factbase
*/
size_t Factbase::hash(Keyvalue &factlist) const {
size_t Factbase::hash(Keyvalue &factlist){
// size_t hash = 0xf848b64e; // Random seed value
// size_t seed = 0x0c32a12fe19d2119;
size_t seed = 0;
std::set<size_t> factset_q;
std::transform(qualities.begin(), qualities.end(), std::inserter(factset_q, factset_q.end()),
[&](const Quality &q) -> size_t {
[&](Quality &q) -> size_t {
return q.encode(factlist).enc;});
std::for_each(factset_q.begin(), factset_q.end(),

15
src/ag_gen/factbase.h
View File

@ -29,10 +29,10 @@ class NetworkState;
*/
class Factbase {
static int current_id;
//int current_id;
friend std::ostream & operator << (std::ostream &os, const Factbase &fb);
friend class boost::serialization::access;
size_t id;
int id;
int qsize;
int tsize;
@ -46,8 +46,8 @@ class Factbase {
template<class Archive>
void serialize(Archive &ar, const unsigned int /* file_version */){
ar & qualities & topologies & id;
//ar & current_id & id & qsize & tsize & qualities & topologies;
//ar & qualities & topologies & id;
ar & current_id & id & qsize & tsize & qualities & topologies;
//ar & qualities;
//ar & topologies;
}
@ -74,9 +74,10 @@ class Factbase {
void set_id();
void force_set_id(int i);
void force_set_id(size_t i);
size_t get_id() const;
size_t hash(Keyvalue &factlist) const;
void reset_curr_id();
void reset_curr_id(int i);
int get_id() const;
size_t hash(Keyvalue &factlist);
int get_size();
};

11
src/ag_gen/network_state.cpp
View File

@ -42,7 +42,7 @@ void NetworkState::force_set_id(int i) { factbase.force_set_id(i); }
/**
* @return The ID of the NetworkState
*/
size_t NetworkState::get_id() { return factbase.get_id(); }
int NetworkState::get_id() { return factbase.get_id(); }
int NetworkState::get_size() { return factbase.get_size(); }
@ -57,10 +57,17 @@ const Factbase &NetworkState::get_factbase() const { return factbase; }
* @param factlist The current Keyvalue
* @return The hash of the Factbase
*/
size_t NetworkState::get_hash(Keyvalue &factlist) const {
size_t NetworkState::get_hash(Keyvalue &factlist){
return factbase.hash(factlist);
}
void NetworkState::reset_curr_id(){
factbase.reset_curr_id();
}
void NetworkState::reset_curr_id(int i){
factbase.reset_curr_id(i);
}
/**
* @brief Adds all unique Quality elements of a vector to the Factbase
*

9
src/ag_gen/network_state.h
View File

@ -53,13 +53,14 @@ class NetworkState {
NetworkState();
const Factbase &get_factbase() const;
size_t get_hash(Keyvalue &factlist) const;
size_t get_hash(Keyvalue &factlist);
void set_id();
void force_set_id(int i);
size_t get_id();
int get_id();
int get_size();
void reset_curr_id();
void reset_curr_id(int i);
void add_qualities(std::vector<Quality> q);
void add_topologies(std::vector<Topology> t);
@ -73,7 +74,7 @@ class NetworkState {
void delete_quality(Quality &q);
void delete_topology(Topology &t);
//bool operator==(NetworkState& foo) {return get_id() == foo.get_id();}
//bool operator<(const NetworkState& foo) const {return get_hash(Keyvalue &factlist) < foo.get_hash(Keyvalue &factlist);}
};
BOOST_SERIALIZATION_ASSUME_ABSTRACT(NetworkState)

43
src/ag_gen/quality.cpp
View File

@ -16,13 +16,13 @@
//ORIGINAL QUALITY
//Quality::Quality(int asset, std::string qualName, std::string o, std::string qualValue, Keyvalue &facts)
// : asset_id(asset), name(std::move(qualName)), op(std::move(o)), value(std::move(qualValue)), encoded(encode(facts).enc) {}
Quality::Quality(int asset, std::string qualName, std::string o, std::string qualValue, Keyvalue &facts)
: asset_id(asset), name(std::move(qualName)), op(std::move(o)), value(std::move(qualValue)), encoded(encode(facts).enc) {}
//TESTING ENCODING ERROR
Quality::Quality(int asset, std::string qualName, std::string o, std::string qualValue, Keyvalue &facts)
: asset_id(asset), name(std::move(qualName)), op(std::move(o)), value(std::move(qualValue)), encoded((size_t)facts.size()) {}
//Quality::Quality(int asset, std::string qualName, std::string o, std::string qualValue, Keyvalue &facts)
// : asset_id(asset), name(std::move(qualName)), op(std::move(o)), value(std::move(qualValue)), encoded((size_t)facts.size()) {}
Quality::Quality()
{
@ -80,37 +80,20 @@ void Quality::print() const {
*
* @return The EncodedQuality
*/
EncodedQuality Quality::encode(const Keyvalue &kv_facts) const {
EncodedQuality Quality::encode(Keyvalue &kv_facts) {
EncodedQuality qual{};
qual.dec.asset_id = asset_id;
qual.dec.attr = kv_facts[name];
//std::cout<<"DONE PART 2"<<std::endl;
//std::cout<<"ASSET ID " <<asset_id<<std::endl;
//std::cout<<" NAME" <<kv_facts[name]<<std::endl;
//std::cout<<"VALUE " <<value<<std::endl;
//THIS IS THE ONE THAT THROWS THE OUT OF RANGE FOR THE UNORDERED MAP
// std::cout<<"Break"<<std::endl;
//std::cout<<"ATTEMPTING TO FIND " <<value<<std::endl;
//std::cout<<"SIZE FROM ENCODE " <<kv_facts.size() << std::endl;
//std::unordered_map<std::string,int>::const_iterator got = kv_facts.hash_table.find(value);
//if(got==kv_facts.hash_table.end())
// std::cout<< " NOT FOUND"<<std::endl;
//else
// std::cout<<" YES FOUND " <<std::endl;
std::unordered_map<std::string,int>::const_iterator got = kv_facts.hash_table.find(value);
if(got == kv_facts.hash_table.end()){
kv_facts.hash_table[value] = kv_facts.length;
kv_facts.str_vector.push_back(value);
kv_facts.length++;
}
//std::cout<<"NAME "<<name<<"VALUE " <<value<< "KEY OR W/E "<< kv_facts[value]<<std::endl;
// std::cout<<"SIZE FROM ENCODE "<<kv_facts.size()<<std::endl;
//try {
qual.dec.val = kv_facts[value];
//}
//catch (std::out_of_range outofrange)
// kv_facts.hash_table.insert(value, kv_facts.size()+1);
//kv_facts.length+=1;
//std::cout<<"YES" << std::endl;
//qual.dec.val=kv_facts[value];
//std::cout<<"DONE PART 3"<<std::endl;
return qual;
}

2
src/ag_gen/quality.h
View File

@ -89,7 +89,7 @@ class Quality {
size_t encoded;
EncodedQuality encode(const Keyvalue &kv_facts) const;
EncodedQuality encode(Keyvalue &kv_facts);
friend class Factbase;

13
src/main.cpp
View File

@ -452,11 +452,12 @@ int main(int argc, char *argv[]) {
bool use_redis = false;
bool use_postgres = false;
bool mpi_subgraphing = false;
bool mpi_tasking = false;
double alpha = 0.5;
int opt;
while ((opt = getopt(argc, argv, "rb:g:dhc:l:n:x:t:q:pa:sm:z:")) != -1) {
while ((opt = getopt(argc, argv, "rb:g:dhc:l:n:x:t:q:pa:sem:z:")) != -1) {
switch (opt) {
case 'g':
should_graph = true;
@ -503,6 +504,9 @@ int main(int argc, char *argv[]) {
case 's':
mpi_subgraphing = true;
break;
case 'e':
mpi_tasking = true;
break;
case 'm':
mpi_nodes = atoi(optarg);
break;
@ -688,12 +692,12 @@ int main(int argc, char *argv[]) {
if (mpi_subgraphing && world.size() > 3)
postinstance = gen.sg_generate(batch_process, batch_size, thread_count, init_qsize, alpha, world, depth_limit); //The method call to generate the attack graph, defined in ag_gen.cpp.
else if (world.size() > 1)
else if (mpi_tasking && world.size() > 1)
postinstance = gen.generate(batch_process, batch_size, thread_count, init_qsize, alpha, world); //The method call to generate the attack graph, defined in ag_gen.cpp.
else
postinstance = gen.single_generate(batch_process, batch_size, thread_count, init_qsize, alpha, world); //The method call to generate the attack graph, defined in ag_gen.cpp.
world.barrier();
//world.barrier();
//Serialization Unit Testing on Postinstance Data
//serialization_unit_testing(postinstance, world);
world.barrier();
@ -704,8 +708,9 @@ int main(int argc, char *argv[]) {
//std::cout << "# of factbase_item " <<postinstance.factbase_items.size()<<std::endl;
if(world.rank() == 0){
std::cout << "Saving to Database." << std::endl;
save_ag_to_db(postinstance, true);
std::cout << "PostInstance states: " << postinstance.factbases.size() << std::endl;
std::cout << "Total Edges: " << get_num_edges() << std::endl;
std::cout << "Total Time: " << postinstance.elapsed_seconds.count() << " seconds\n";

24
src/mpi/tasks.cpp
View File

@ -508,19 +508,35 @@ int send_check(boost::mpi::communicator &world, int curr_node){
}
void state_merge(std::vector<Factbase> node_factbases, std::vector<Edge> node_edges,\
std::unordered_map<size_t, int> &hash_map, AGGenInstance &instance, double mem_threshold, mpi::communicator &world){
std::unordered_map<size_t, int> &hash_map, AGGenInstance &instance, double mem_threshold,\
mpi::communicator &world, int last_known_id){
auto tot_sys_mem = getTotalSystemMemory();
for(auto fb : node_factbases){
//auto fb = ns.get_factbase();
//std::cout << "Started Task 3." << std::endl;
auto hash_num = fb.get_id();
auto hash_num = fb.hash(instance.facts);
//although local frontier is updated, the global hash is also updated to avoid testing on explored states.
if (hash_map.find(hash_num) == hash_map.end()) {
auto old_id = fb.get_id();
fb.reset_curr_id(last_known_id+1);
fb.set_id();
auto new_id = fb.get_id();
last_known_id = new_id;
for(auto ed: node_edges){
if (ed.get_from_id() == old_id){
ed.force_from_id(new_id);
}
if (ed.get_to_id() == old_id){
ed.force_to_id(new_id);
}
}
instance.factbases.push_back(fb);
hash_map.insert(std::make_pair(fb.get_id(), fb.get_id()));
hash_map.insert(std::make_pair(fb.hash(instance.facts), fb.get_id()));
//See memory usage. If it exceeds the threshold, store new states in the DB
double i_alpha = 0.0;
@ -549,7 +565,7 @@ void state_merge(std::vector<Factbase> node_factbases, std::vector<Edge> node_ed
}
}
//This does add duplicate edges
//This does add duplicate edges - taken care of through graphviz' "strict" graphing
for (auto ed : node_edges){
instance.edges.push_back(ed);
}

3
src/mpi/tasks.h
View File

@ -24,6 +24,7 @@ void task_four(NetworkState &new_state);
int send_check(boost::mpi::communicator &world, int curr_node);
void state_merge(std::vector<Factbase> node_factbases, std::vector<Edge> node_edges,\
std::unordered_map<size_t, int> &hash_map, AGGenInstance &instance, double mem_threshold, mpi::communicator &world);
std::unordered_map<size_t, int> &hash_map, AGGenInstance &instance, double mem_threshold,\
mpi::communicator &world, int last_known_id);
#endif //TASKS_H