1091 lines
37 KiB
C++
Executable File
1091 lines
37 KiB
C++
Executable File
|
|
#include <vector>
|
|
#include <string>
|
|
#include <unordered_map>
|
|
#include <algorithm>
|
|
#include <iostream>
|
|
#include <sstream>
|
|
#include <array>
|
|
#include <iterator>
|
|
#include <sys/time.h>
|
|
#include <omp.h>
|
|
|
|
#include "db_functions.h"
|
|
|
|
#include "avail_mem.h"
|
|
#include "keyvalue.h"
|
|
#include "db.h"
|
|
|
|
#include "../ag_gen/exploit.h"
|
|
#include "../ag_gen/edge.h"
|
|
#include "../ag_gen/quality.h"
|
|
#include "../ag_gen/topology.h"
|
|
#include "../ag_gen/asset.h"
|
|
#include "../ag_gen/factbase.h"
|
|
#include "../ag_gen/network_state.h"
|
|
|
|
#include <fstream>
|
|
|
|
static DB db;
|
|
//static DB db2;
|
|
//static DB db3;
|
|
//static DB db4;
|
|
|
|
void init_db(std::string connect_str) {
|
|
db.connect(connect_str);
|
|
//db.connect("dbname=ag_parallel user=mingli password=N7SBaxkXdoA5venS");
|
|
//db2.connect("postgresql://catherine:1234@localhost:5432/ag_parallel");
|
|
//db3.connect("postgresql://mingli:f5410116@localhost:5432/ag_parallel");
|
|
//db4.connect("postgresql://rosemary:110726@localhost:5432/ag_parallel");
|
|
/*db.exec("BEGIN;");
|
|
db.execAsync("DELETE FROM factbase_item WHERE TRUE;");
|
|
db.execAsync("DELETE FROM edge_asset_binding WHERE TRUE;");
|
|
db.execAsync("DELETE FROM edge WHERE TRUE;");
|
|
db.execAsync("DELETE FROM factbase WHERE TRUE;");
|
|
db.execAsync("DELETE FROM keyvalue WHERE TRUE;");
|
|
db.execAsync("DELETE FROM exploit_postcondition WHERE TRUE;");
|
|
db.execAsync("DELETE FROM exploit_precondition WHERE TRUE;");
|
|
db.execAsync("DELETE FROM exploit WHERE TRUE;");
|
|
db.execAsync("DELETE FROM quality WHERE TRUE;");
|
|
db.execAsync("DELETE FROM topology WHERE TRUE;");
|
|
db.execAsync("DELETE FROM asset WHERE TRUE;");
|
|
db.execAsync("COMMIT;");*/
|
|
}
|
|
|
|
void import_models(std::string nm, std::string xp) {
|
|
db.exec(nm);
|
|
db.exec(xp);
|
|
}
|
|
|
|
int get_max_factbase_id() {
|
|
std::vector<Row> res = db.exec("SELECT MAX(id) FROM factbase;");
|
|
return stoi(res[0][0]);
|
|
}
|
|
|
|
std::vector<std::string> fetch_keyvalues() {
|
|
std::vector<Row> rows = db.exec("SELECT property FROM keyvalue;");
|
|
std::vector<std::string> kvs{};
|
|
|
|
for(auto &row : rows) {
|
|
kvs.push_back(row[0]);
|
|
}
|
|
|
|
return kvs;
|
|
}
|
|
|
|
Keyvalue fetch_kv() {
|
|
auto kv_raw = fetch_keyvalues();
|
|
Keyvalue kv{};
|
|
kv.populate(kv_raw);
|
|
return kv;
|
|
}
|
|
|
|
void delete_edges(std::vector<int> edge_ids) {
|
|
std::ostringstream ss;
|
|
|
|
ss << "(";
|
|
|
|
std::copy(edge_ids.begin(), edge_ids.end() - 1, std::ostream_iterator<int>(ss, ", "));
|
|
ss << edge_ids.back() << ")";
|
|
|
|
std::string ids = ss.str();
|
|
|
|
std::string sql = "DELETE FROM edge_asset_binding WHERE edge_id IN " + ids
|
|
+ "; DELETE FROM edge WHERE id IN" + ids + ";";
|
|
|
|
db.exec(sql);
|
|
}
|
|
|
|
GraphInfo fetch_graph_info() {
|
|
std::vector<Row> factbase_rows = db.exec("SELECT id FROM factbase ORDER BY id;");
|
|
std::vector<Row> edge_rows = db.exec("SELECT * FROM edge ORDER BY id;");
|
|
|
|
std::vector<int> factbase_ids;
|
|
factbase_ids.resize(factbase_rows.size());
|
|
|
|
std::transform(factbase_rows.begin(), factbase_rows.end(), factbase_ids.begin(),
|
|
[](Row row){ return std::stoi(row[0]); });
|
|
|
|
std::vector<std::array<int, 4>> edges;
|
|
|
|
for (auto& row : edge_rows) {
|
|
int id = std::stoi(row[0]);
|
|
int from_node = std::stoi(row[1]);
|
|
int to_node = std::stoi(row[2]);
|
|
int exploit_id = std::stoi(row[3]);
|
|
|
|
std::array<int, 4> arr = {id, from_node, to_node, exploit_id};
|
|
edges.push_back(arr);
|
|
}
|
|
|
|
return std::make_pair(factbase_ids, edges);
|
|
}
|
|
|
|
std::vector<std::vector<std::pair<size_t, std::string>>> fetch_all_factbase_items() {
|
|
std::vector<std::vector<std::pair<size_t, std::string>>> fi;
|
|
std::vector<Row> firows = db.exec("SELECT * FROM factbase_item;");
|
|
if (firows.empty())
|
|
throw CustomDBException();
|
|
|
|
int current_index = -1;
|
|
for (auto firow : firows) {
|
|
int index = stoi(firow[0]);
|
|
if (index != current_index) {
|
|
current_index = index;
|
|
fi.emplace_back();
|
|
}
|
|
size_t st;
|
|
sscanf(firow[1].c_str(), "%zu", &st);
|
|
fi[index].push_back(make_pair(st, firow[2]));
|
|
}
|
|
|
|
return fi;
|
|
}
|
|
|
|
std::vector<std::pair<size_t, std::string>> fetch_one_factbase_items(int index) {
|
|
std::vector<std::pair<size_t, std::string>> fi;
|
|
std::vector<Row> firows = db.exec("SELECT f,type FROM factbase_item WHERE factbase_id=" + std::to_string(index) + ";");
|
|
if (firows.empty())
|
|
throw CustomDBException();
|
|
|
|
for (auto firow : firows) {
|
|
size_t st;
|
|
sscanf(firow[0].c_str(), "%zu", &st);
|
|
fi.push_back(std::make_pair(st, firow[1]));
|
|
}
|
|
|
|
return fi;
|
|
}
|
|
|
|
std::vector<std::string> fetch_unique_values() { //from the database to fetch all the values and properties and put them in a string vector
|
|
std::string keyvaluesql = "SELECT property FROM quality\n"
|
|
"UNION DISTINCT\n"
|
|
"SELECT value FROM quality\n"
|
|
"UNION DISTINCT\n"
|
|
//"SELECT op FROM quality\n"
|
|
//"UNION DISTINCT\n"
|
|
"SELECT property FROM exploit_postcondition\n"
|
|
"UNION DISTINCT\n"
|
|
"SELECT value FROM exploit_postcondition\n"
|
|
"UNION DISTINCT\n"
|
|
"SELECT property FROM topology\n"
|
|
"UNION DISTINCT\n"
|
|
"SELECT value FROM topology;";//all the values and properties will be distinct from each other
|
|
|
|
std::vector<Row> rows = db.exec(keyvaluesql);
|
|
std::vector<std::string> attrs; //this will be a set of values and properties
|
|
|
|
for(auto &row : rows) {
|
|
attrs.push_back(row[0]);
|
|
}
|
|
|
|
return attrs;
|
|
}
|
|
|
|
/**
|
|
* @brief Fetches all possible quality attributes.
|
|
*
|
|
* @return Returns a vector of strings with all possible quality attributes.
|
|
*/
|
|
std::vector<std::string> fetch_quality_attributes() {
|
|
std::vector<std::string> attrs;
|
|
std::vector<Row> qrows = db.exec("SELECT DISTINCT property FROM quality;");
|
|
std::vector<Row> erows =
|
|
db.exec("SELECT DISTINCT property FROM exploit_postcondition;");
|
|
|
|
for (auto &row : qrows) {
|
|
std::string prop = row[0];
|
|
attrs.push_back(prop);
|
|
}
|
|
|
|
for (auto &row : erows) {
|
|
std::string prop = row[0];
|
|
attrs.push_back(prop);
|
|
}
|
|
|
|
return attrs;
|
|
}
|
|
|
|
/**
|
|
* @brief Fetches all possible quality values.
|
|
*
|
|
* @return Returns
|
|
*/
|
|
std::vector<std::string> fetch_quality_values() {
|
|
std::vector<std::string> vals;
|
|
std::vector<Row> qrows = db.exec("SELECT DISTINCT value FROM quality;");
|
|
std::vector<Row> erows =
|
|
db.exec("SELECT DISTINCT value FROM exploit_postcondition;");
|
|
|
|
for (auto &row : qrows) {
|
|
std::string val = row[0];
|
|
vals.push_back(val);
|
|
}
|
|
|
|
for (auto &row : erows) {
|
|
std::string val = row[0];
|
|
vals.push_back(val);
|
|
}
|
|
|
|
return vals;
|
|
}
|
|
|
|
std::vector<std::string> fetch_topology_attributes() {
|
|
std::vector<std::string> attrs;
|
|
std::vector<Row> rows = db.exec("SELECT DISTINCT property FROM topology;");
|
|
|
|
for (auto &row : rows) {
|
|
std::string prop = row[0];
|
|
attrs.push_back(prop);
|
|
}
|
|
|
|
return attrs;
|
|
}
|
|
|
|
/**
|
|
* @brief Fetches all possible topology values.
|
|
*
|
|
* @return Returns a vector of strings with all possible topology values
|
|
*/
|
|
std::vector<std::string> fetch_topology_values() {
|
|
std::vector<std::string> vals;
|
|
std::vector<Row> rows = db.exec("SELECT DISTINCT value FROM topology;");
|
|
|
|
for (auto &row : rows) {
|
|
std::string val = row[0];
|
|
vals.push_back(val);
|
|
}
|
|
|
|
return vals;
|
|
}
|
|
|
|
/**
|
|
* @brief Fetches the preconditions of an Exploit from the database.
|
|
*/
|
|
std::unordered_map<
|
|
int, std::tuple<std::vector<ParameterizedQuality>, std::vector<ParameterizedTopology>>>
|
|
fetch_exploit_preconds() {
|
|
std::vector<Row> rows = db.exec("SELECT * FROM exploit_precondition;");
|
|
|
|
std::unordered_map<
|
|
int, std::tuple<std::vector<ParameterizedQuality>, std::vector<ParameterizedTopology>>>
|
|
precond_map;
|
|
|
|
int curr_id = -1;
|
|
std::vector<ParameterizedQuality> preconds_q;
|
|
std::vector<ParameterizedTopology> preconds_t;
|
|
for (auto &row : rows) {
|
|
int type = stoi(row[2]);
|
|
int exploit_id = stoi(row[1]);
|
|
|
|
if (exploit_id != curr_id) {
|
|
if (curr_id != -1) {
|
|
std::tuple<std::vector<ParameterizedQuality>,
|
|
std::vector<ParameterizedTopology>>
|
|
tup{preconds_q, preconds_t};
|
|
precond_map[curr_id] = tup;
|
|
|
|
preconds_q.clear();
|
|
preconds_t.clear();
|
|
}
|
|
|
|
curr_id = exploit_id;
|
|
}
|
|
|
|
if (type == 0) {//type 0 is quality-typed precondition
|
|
int param1 = stoi(row[3]);
|
|
std::string property = row[5];
|
|
std::string value = row[6];
|
|
std::string op = row[7];
|
|
|
|
ParameterizedQuality qual{param1, property, value, op};
|
|
preconds_q.push_back(qual);
|
|
} else { //type 1 is topology-typed precondition
|
|
int param1 = stoi(row[3]);
|
|
int param2 = stoi(row[4]);
|
|
std::string property = row[5];
|
|
std::string value = row[6];
|
|
std::string op = row[7];
|
|
std::string dir_str = row[8];
|
|
|
|
DIRECTION_T dir;
|
|
if(dir_str == "->") {
|
|
dir = FORWARD_T;
|
|
} else if (dir_str == "<-") {
|
|
dir = BACKWARD_T;
|
|
} else if (dir_str == "<->") {
|
|
dir = BIDIRECTION_T;
|
|
} else {
|
|
std::cerr << "Unknown direction '" << dir_str << "'" << std::endl;
|
|
exit(1);
|
|
}
|
|
|
|
ParameterizedTopology topo{param1, param2, dir,
|
|
property, op, value}; //parameterized into a struct
|
|
preconds_t.push_back(topo);
|
|
}
|
|
}
|
|
|
|
std::tuple<std::vector<ParameterizedQuality>, std::vector<ParameterizedTopology>> tup{
|
|
preconds_q, preconds_t};
|
|
precond_map[curr_id] = tup;//the last exploit's preconditions (quality and topology) also needs to put in the unordered map
|
|
|
|
return precond_map; //the unordered map use the exploit id as the key, the value of each item is the parameterized quality and topology
|
|
}
|
|
|
|
/**
|
|
* @brief Fetches the postconditions of an Exploit from the database.
|
|
*/
|
|
std::unordered_map<
|
|
int, std::tuple<std::vector<PostconditionQ>, std::vector<PostconditionT>>>
|
|
fetch_exploit_postconds() {
|
|
std::vector<Row> rows = db.exec("SELECT * FROM exploit_postcondition;");
|
|
|
|
std::unordered_map<
|
|
int, std::tuple<std::vector<PostconditionQ>, std::vector<PostconditionT>>>
|
|
postcond_map;
|
|
|
|
int curr_id = -1;
|
|
std::vector<PostconditionQ> postconds_q; //PostconditionQ and PostconditionT are defined in exploit.h
|
|
std::vector<PostconditionT> postconds_t;
|
|
for (auto &row : rows) {
|
|
int type = stoi(row[2]);//type 0 change on quality, 1 change on topology(seldom)
|
|
int exploit_id = stoi(row[1]);
|
|
|
|
std::string action_str = row[9];
|
|
ACTION_T action;
|
|
|
|
// std::cout << action_str << "\n";
|
|
|
|
if(action_str == "add" || action_str == "insert") {
|
|
action = ADD_T;
|
|
} else if (action_str == "update") {
|
|
action = UPDATE_T;
|
|
} else if (action_str == "delete") {
|
|
action = DELETE_T;
|
|
} else {
|
|
std::cout << "Bad Action '" << action_str << "'" << std::endl;
|
|
exit(1);
|
|
}
|
|
|
|
if (exploit_id != curr_id) {
|
|
if (curr_id != -1) {
|
|
std::tuple<std::vector<PostconditionQ>,
|
|
std::vector<PostconditionT>>
|
|
tup{postconds_q, postconds_t};
|
|
postcond_map[curr_id] = tup;
|
|
|
|
postconds_q.clear();
|
|
postconds_t.clear();
|
|
}
|
|
|
|
curr_id = exploit_id;
|
|
}
|
|
|
|
if (type == 0) {
|
|
int param1 = stoi(row[3]);
|
|
std::string property = row[5];
|
|
std::string value = row[6];
|
|
std::string op = row[7];
|
|
|
|
ParameterizedQuality qual{param1, property, value, op};
|
|
postconds_q.push_back(std::make_tuple(action, qual));
|
|
} else {
|
|
int param1 = stoi(row[3]);
|
|
int param2 = stoi(row[4]);
|
|
std::string property = row[5];
|
|
std::string value = row[6];
|
|
std::string op = row[7];
|
|
std::string dir_str = row[8];
|
|
|
|
DIRECTION_T dir;
|
|
if(dir_str == "->") {
|
|
dir = FORWARD_T;
|
|
} else if (dir_str == "<-") {
|
|
dir = BACKWARD_T;
|
|
} else if (dir_str == "<->") {
|
|
dir = BIDIRECTION_T;
|
|
} else {
|
|
std::cerr << "Unknown direction '" << dir_str << "'" << std::endl;
|
|
exit(1);
|
|
}
|
|
|
|
ParameterizedTopology topo{param1, param2, dir,
|
|
property, op, value};
|
|
postconds_t.push_back(std::make_tuple(action, topo));
|
|
}
|
|
}
|
|
|
|
std::tuple<std::vector<PostconditionQ>, std::vector<PostconditionT>> tup{
|
|
postconds_q, postconds_t};
|
|
postcond_map[curr_id] = tup;
|
|
|
|
return postcond_map;
|
|
}
|
|
|
|
std::vector<Exploit> fetch_all_exploits() {
|
|
std::vector<Exploit> exploits;
|
|
std::vector<Row> rows = db.exec("SELECT * FROM exploit;");
|
|
|
|
auto pre = fetch_exploit_preconds();
|
|
auto post = fetch_exploit_postconds();
|
|
|
|
for (auto &row : rows) {
|
|
int id = std::stoi(row[0]);
|
|
std::string name = row[1];
|
|
int num_params = std::stoi(row[2]);
|
|
std::string group = row[3];
|
|
|
|
auto preconds = pre[id];
|
|
auto postconds = post[id];
|
|
//Use Exploit class's obj exploit to store each exploit's complete info, especially pre-and-post conditions
|
|
Exploit exploit(id, name, num_params, group, preconds, postconds);
|
|
exploits.push_back(exploit);//push into exploits vector
|
|
}
|
|
|
|
return exploits; //the vector will be returned to the calling function.
|
|
}
|
|
|
|
/**
|
|
* @brief Gets all of the qualities for the Asset
|
|
* @details Grabs all of the qualities in the database associated with
|
|
* the Asset's ID and gives them to the Asset
|
|
*/
|
|
std::unordered_map<int, std::vector<Quality>> fetch_asset_qualities(Keyvalue &facts) {
|
|
std::vector<Row> rows = db.exec("SELECT * FROM quality;");
|
|
|
|
std::unordered_map<int, std::vector<Quality>> qmap;
|
|
|
|
int curr_id = -1;
|
|
std::vector<Quality> qualities;
|
|
for (auto &row : rows) {
|
|
int asset_id = std::stoi(row[0]);
|
|
std::string property = row[1];
|
|
std::string op = row[2];
|
|
std::string value = row[3];
|
|
// cout << "op: " << op << endl;
|
|
|
|
if (asset_id != curr_id) {
|
|
if (curr_id != -1) { //a new asset's quality has been reached
|
|
qmap[curr_id] = qualities; //all the old qualities are put in the old asset's element in the unordered map
|
|
qualities.clear(); //clear the qualities for the new asset
|
|
}
|
|
curr_id = asset_id; //unconditional update curr_id to asset_id whenever a new asset is reached
|
|
}
|
|
// Quality qual(asset_id, property, "=", value);
|
|
// qualities.push_back(qual);
|
|
|
|
qualities.emplace_back(asset_id, property, op, value, facts);//each asset's qualities will be put in a dedicated Quality vector
|
|
}
|
|
|
|
qmap[curr_id] = qualities;//the last asset's qualities must be put in the unordered map too
|
|
|
|
return qmap;
|
|
}
|
|
|
|
/**
|
|
* @brief Gets all of the Assets under the network
|
|
* @details Grabs all of the Assets in the database under the network given in
|
|
* the argument and returns a vector of those Assets
|
|
*
|
|
* @param network Name of the network to grab from
|
|
*/
|
|
std::vector<Asset> fetch_all_assets(Keyvalue &facts) {
|
|
std::vector<Row> rows = db.exec("SELECT * FROM asset;");
|
|
std::vector<Asset> new_assets; //use class Asset (in asset.h) to define a vector, each element will be an object
|
|
|
|
auto qmap = fetch_asset_qualities(facts); //combine each asset with all its qualities
|
|
|
|
for (auto &row : rows) {
|
|
int id = std::stoi(row[0]);
|
|
std::string name = row[1];
|
|
auto q = qmap[id]; //this returns a Quality vector, which includes all the records in the quality table related to the current id.
|
|
new_assets.emplace_back(name, q);
|
|
}
|
|
|
|
return new_assets;
|
|
}
|
|
|
|
std::vector<Quality> fetch_all_qualities(Keyvalue &facts) {
|
|
std::vector<Quality> qualities;
|
|
std::vector<Row> rows = db.exec("SELECT * FROM quality;");
|
|
|
|
for (auto &row : rows) {
|
|
int asset_id = std::stoi(row[0]); //stoi means string to integer, since rows is a vector of string vectors. The smallest elements in rows are string type.
|
|
std::string property = row[1];
|
|
std::string op = row[2];
|
|
std::string value = row[3];
|
|
|
|
Quality qual(asset_id, property, op, value, facts); //associate each record from quality table with the facts(the hash_table and fact string vector).
|
|
qualities.push_back(qual);
|
|
}
|
|
|
|
return qualities;
|
|
}
|
|
|
|
std::vector<Topology> fetch_all_topologies(Keyvalue &facts) {
|
|
std::vector<Topology> topologies;
|
|
|
|
std::vector<Row> rows = db.exec("SELECT * FROM topology;");
|
|
for (auto &row : rows) {
|
|
int from_asset = std::stoi(row[0]);
|
|
int to_asset = std::stoi(row[1]);
|
|
std::string dir_str = row[2];
|
|
|
|
DIRECTION_T dir; //DIRECTION_T is a user defined enum type (in topology.h), it has three different values: FORWARD_T 0, BACKWARD_T 1, and BIDIRECTION_T 2
|
|
if(dir_str == "->") {
|
|
dir = FORWARD_T;
|
|
} else if (dir_str == "<-") {
|
|
dir = BACKWARD_T;
|
|
} else if (dir_str == "<->") {
|
|
dir = BIDIRECTION_T;
|
|
} else {
|
|
std::cerr << "Unknown direction '" << dir_str << "'" << std::endl;
|
|
exit(1);
|
|
}
|
|
|
|
std::string property = row[3];
|
|
std::string op = row[4];
|
|
std::string value = row[5];
|
|
|
|
Topology t(from_asset, to_asset, dir, property, op, value, facts); //associate each record from topology table with the facts
|
|
topologies.push_back(t);
|
|
}
|
|
|
|
return topologies;
|
|
}
|
|
|
|
Keyvalue fetch_facts() { //Keyvalue is a class name(defined in Keyvalue.h), this function will return an Keyvalue obj
|
|
Keyvalue initfacts;
|
|
|
|
// initfacts.populate(fetch_quality_attributes());
|
|
// initfacts.populate(fetch_quality_values());
|
|
// initfacts.populate(fetch_topology_attributes());
|
|
// initfacts.populate(fetch_topology_values());
|
|
initfacts.populate(fetch_unique_values());
|
|
|
|
return initfacts;
|
|
}
|
|
|
|
// void save_ag_to_db(std::vector<FactbaseItems> &factbase_items,
|
|
// std::vector<Factbase> &factbases, std::vector<Edge> &edges,
|
|
// Keyvalue &factlist) {
|
|
void save_ag_to_db(AGGenInstance &instance, bool save_keyvalue){
|
|
std::vector<FactbaseItems>& factbase_items = instance.factbase_items;
|
|
std::vector<Factbase>& factbases = instance.factbases;
|
|
std::vector<Edge>& edges = instance.edges;
|
|
auto tot_sys_mem = getTotalSystemMemory();
|
|
|
|
Keyvalue& factlist = instance.facts;
|
|
|
|
struct timeval t1,t2;
|
|
|
|
/** backup cout buffer and redirect to states.txt **/
|
|
std::ofstream out("states.txt");
|
|
auto *coutbuf = std::cout.rdbuf();
|
|
std::cout.rdbuf(out.rdbuf());
|
|
|
|
//this part takes 0.3s
|
|
gettimeofday(&t1,NULL);
|
|
db.exec("BEGIN;");
|
|
if (!factbases.empty()){
|
|
//printf("The size of the factbases is %ld\n",factbases.size());
|
|
|
|
std::string factbase_sql_query = "INSERT INTO factbase VALUES ";
|
|
int factbase_counter = 0;
|
|
int flag = 0;
|
|
double fbsize = 0;
|
|
|
|
for (int i = 0; i < factbases.size(); ++i) {
|
|
|
|
|
|
/** redirect state info printing to states.txt file **/
|
|
factbases[i].print();
|
|
|
|
if (i == 0 || flag == 1) {
|
|
factbase_sql_query += "(" + std::to_string(factbases[i].get_id()) +
|
|
",'" +
|
|
std::to_string(factbases[i].hash(factlist)) + "')";
|
|
flag = 0;
|
|
} else {
|
|
factbase_sql_query += ",(" + std::to_string(factbases[i].get_id()) +
|
|
",'" +
|
|
std::to_string(factbases[i].hash(factlist)) + "')";
|
|
|
|
}
|
|
//Break up query due to memory constraints. Suboptimal approach.
|
|
factbase_counter += 1;
|
|
fbsize = sizeof(factbase_sql_query);
|
|
if (fbsize/tot_sys_mem >= 0.1){
|
|
factbase_sql_query += ";";
|
|
db.execAsync(factbase_sql_query);
|
|
flag = 1;
|
|
factbase_sql_query = "INSERT INTO factbase VALUES ";
|
|
db.exec("COMMIT;");
|
|
db.exec("BEGIN;");
|
|
|
|
}
|
|
}
|
|
factbase_sql_query += ";";
|
|
db.execAsync(factbase_sql_query);
|
|
}
|
|
db.execAsync("COMMIT;");
|
|
|
|
/** reset cout buffer **/
|
|
std::cout.rdbuf(coutbuf);
|
|
|
|
//gettimeofday(&t2,NULL);
|
|
//printf("The preparation of factbases took %lf ms\n",(t2.tv_sec-t1.tv_sec)*1000.0+(t2.tv_usec-t1.tv_usec)/1000.0);
|
|
|
|
|
|
|
|
//this part takes 1.5s
|
|
//gettimeofday(&t1,NULL);
|
|
//potentially bug in forming the string for db storage, check if the first record has a comma before it
|
|
/*FACTBASE ITEMS
|
|
if (!factbase_items.empty()) {
|
|
int fis=factbase_items.size();
|
|
//fis=4;
|
|
//#pragma omp parallel for num_threads(4)\
|
|
//default(none) shared(factbase_items,fis,db,db2,db3,db4,std::cout) schedule(static,1)
|
|
#pragma omp parallel for num_threads(1)\
|
|
default(none) shared(factbase_items,fis,db,std::cout)
|
|
//for(int k=0;k<4;k++){
|
|
for(int k=0; k<1; k++) {
|
|
std::string item_sql_query = "INSERT INTO factbase_item VALUES ";
|
|
std::string quality_sql_query = "";
|
|
std::string topology_sql_query = "";
|
|
int sql_index=0;
|
|
*///FACTBASE ITEMS
|
|
|
|
/*
|
|
for (int j = 0; j<fis/4+((k==3)?(fis%4):0); j++){
|
|
//for (int j = 0, sql_index = 0; j < factbase_items.size(); ++j) {
|
|
auto fbi = factbase_items[j+k*(fis/4)];
|
|
int id = std::get<1>(fbi);
|
|
auto items = std::get<0>(fbi);
|
|
auto quals = std::get<0>(items);
|
|
auto topo = std::get<1>(items);
|
|
for (auto qi : quals) {
|
|
*/
|
|
//UNCOMMENT FOR FACTBASE ITEM STORAGE
|
|
/*
|
|
int fbi_counter = 0;
|
|
int fbi_q_flag = 0;
|
|
int fbi_t_flag = 0;
|
|
for (int j = 0, sql_index = 0; j < factbase_items.size(); ++j) {
|
|
auto fbi = factbase_items[j];
|
|
int id = std::get<1>(fbi);
|
|
auto items = std::get<0>(fbi);
|
|
auto quals = std::get<0>(items);
|
|
auto topo = std::get<1>(items);
|
|
for (auto qi : quals) {
|
|
if (sql_index == 0 || fbi_q_flag == 1){
|
|
quality_sql_query += "(" + std::to_string(id) + "," +
|
|
std::to_string(qi.get_encoding()) +
|
|
",'quality')";
|
|
fbi_q_flag = 0;
|
|
}
|
|
|
|
else
|
|
quality_sql_query += ",(" + std::to_string(id) + "," +
|
|
std::to_string(qi.get_encoding()) +
|
|
",'quality')";
|
|
sql_index++;
|
|
}
|
|
|
|
for (auto ti : topo) {
|
|
if (sql_index == 0 || fbi_t_flag == 1) {
|
|
topology_sql_query += "(" + std::to_string(id) + "," +
|
|
std::to_string(ti.get_encoding()) +
|
|
",'topology')";
|
|
fbi_t_flag = 0;
|
|
}
|
|
|
|
else
|
|
topology_sql_query += ",(" + std::to_string(id) + "," +
|
|
std::to_string(ti.get_encoding()) +
|
|
",'topology')";
|
|
sql_index++;
|
|
}
|
|
//Break up query due to memory constraints. Suboptimal approach.
|
|
fbi_counter += 1;
|
|
if (fbi_counter >= 500){
|
|
item_sql_query += quality_sql_query + topology_sql_query + "ON CONFLICT DO NOTHING;";
|
|
db.exec("BEGIN;");
|
|
db.execAsync(item_sql_query);
|
|
db.execAsync("COMMIT;");
|
|
fbi_q_flag = 1;
|
|
fbi_t_flag = 1;
|
|
item_sql_query = "INSERT INTO factbase_item VALUES ";
|
|
quality_sql_query = "";
|
|
topology_sql_query = "";
|
|
}
|
|
|
|
}
|
|
|
|
item_sql_query += quality_sql_query + topology_sql_query + "ON CONFLICT DO NOTHING;";
|
|
int thrd_num=omp_get_thread_num();
|
|
if(thrd_num==0){
|
|
db.exec("BEGIN;");
|
|
db.execAsync(item_sql_query);
|
|
db.execAsync("COMMIT;");
|
|
}
|
|
else if(thrd_num==1){
|
|
//db2.exec("BEGIN;");
|
|
//db2.execAsync(item_sql_query);
|
|
//db2.execAsync("COMMIT;");
|
|
}
|
|
else if(thrd_num==2){
|
|
//db3.exec("BEGIN;");
|
|
//db3.execAsync(item_sql_query);
|
|
//db3.execAsync("COMMIT;");
|
|
}
|
|
else {
|
|
//db4.exec("BEGIN;");
|
|
//db4.execAsync(item_sql_query);
|
|
//db4.execAsync("COMMIT;");
|
|
}
|
|
}
|
|
}
|
|
gettimeofday(&t2,NULL);
|
|
//printf("The saving of factbase and items took %lf ms\n",(t2.tv_sec-t1.tv_sec)*1000.0+(t2.tv_usec-t1.tv_usec)/1000.0);
|
|
*///FACTBASE ITEMS
|
|
|
|
//this part takes 42.0s
|
|
//probably where to apply parallelization, this part takes most of the db saving time (>=80%)
|
|
gettimeofday(&t1,NULL);
|
|
int edge_counter = 0;
|
|
int edge_flag = 0;
|
|
if (!edges.empty()) {
|
|
std::vector<std::string> edge_queries;
|
|
edge_queries.resize(edges.size());
|
|
//convert each edge into a string which will be used by db insert
|
|
std::transform(edges.begin(), edges.end(), edge_queries.begin(), to_query);//to_query is a unary operation on all the elements in edges. It is defined in db_function.
|
|
|
|
struct timeval t3,t4;
|
|
//---first way to build the map
|
|
std::unordered_map<std::string, int> eq;
|
|
gettimeofday(&t3,NULL);
|
|
auto ei = edge_queries.begin(); //returns an iterator, not the 1st element!
|
|
//std::cout<<"The first element should be "<<edge_queries.at(0)<<std::endl;
|
|
int j;
|
|
for (j = 0; ei != edge_queries.end(); j++, ei++) //this for loop takes only 76.5ms, it gives each edge string an integer value and stores in unordered map (like a dictionary)
|
|
{
|
|
eq.insert({*ei, j});//all keys must be different, which means not all key-value pairs are inserted!!!
|
|
}
|
|
|
|
//----first method to store into db
|
|
//---------------------------------
|
|
//---------------------------------
|
|
//eq is dictionary
|
|
//edges an Edge vector as the member of input postinstance
|
|
std::string edge_sql_query = "INSERT INTO edge VALUES ";
|
|
std::string edge_assets_sql_query = "INSERT INTO edge_asset_binding VALUES ";
|
|
int ii = 0;
|
|
double esql = 0;
|
|
double easql = 0;
|
|
for (auto ei : eq) {
|
|
int i = ei.second;
|
|
|
|
int eid = edges[i].get_id();
|
|
//int eid = edges[i].set_id();
|
|
|
|
if (ii == 0 | edge_flag == 1) {
|
|
edge_sql_query += "(" + std::to_string(eid) + "," + ei.first;
|
|
edge_assets_sql_query += edges[i].get_asset_query();
|
|
edge_flag = 0;
|
|
} else {
|
|
edge_sql_query += ",(" + std::to_string(eid) + "," + ei.first;
|
|
edge_assets_sql_query += "," + edges[i].get_asset_query();
|
|
}
|
|
|
|
++ii;
|
|
//Break up query due to memory constraints. Suboptimal approach.
|
|
edge_counter += 1;
|
|
esql = sizeof(edge_sql_query);
|
|
easql = sizeof(edge_assets_sql_query);
|
|
|
|
if (esql/tot_sys_mem >= 0.1 || easql/tot_sys_mem >= 0.1){
|
|
edge_sql_query += "ON CONFLICT DO NOTHING;";
|
|
edge_assets_sql_query += "ON CONFLICT DO NOTHING;";
|
|
db.execAsync(edge_sql_query);
|
|
db.execAsync(edge_assets_sql_query);
|
|
edge_flag = 1;
|
|
edge_sql_query = "INSERT INTO edge VALUES ";
|
|
edge_assets_sql_query = "INSERT INTO edge_asset_binding VALUES ";
|
|
}
|
|
}
|
|
edge_sql_query += " ON CONFLICT DO NOTHING;";
|
|
edge_assets_sql_query += " ON CONFLICT DO NOTHING;";
|
|
struct timeval t5,t6,t7;
|
|
gettimeofday(&t5,NULL);
|
|
db.execAsync(edge_sql_query);//33.7s
|
|
gettimeofday(&t6,NULL);
|
|
db.execAsync(edge_assets_sql_query); //7.6s
|
|
gettimeofday(&t7,NULL);
|
|
// printf("The saving of edges took %lf ms\n",(t6.tv_sec-t5.tv_sec)*1000.0+(t6.tv_usec-t5.tv_usec)/1000.0);
|
|
//printf("The saving of edges_assets_bingding took %lf ms\n",(t7.tv_sec-t6.tv_sec)*1000.0+(t7.tv_usec-t6.tv_usec)/1000.0);
|
|
|
|
//----second method to store into db
|
|
//----------------------------------
|
|
//----------------------------------
|
|
/*
|
|
std::vector<std::string> unique_eq;
|
|
int unique_idx[eq.size()];
|
|
int jj=0;
|
|
for(auto ei: eq) {
|
|
unique_eq.push_back(ei.first);
|
|
unique_idx[jj]=ei.second;
|
|
jj++;
|
|
}
|
|
gettimeofday(&t4,NULL);
|
|
//printf("The preparation of eq took %lf ms\n",(t4.tv_sec-t3.tv_sec)*1000.0+(t4.tv_usec-t3.tv_usec)/1000.0);
|
|
|
|
struct timeval t5,t6;
|
|
gettimeofday(&t5,NULL);
|
|
//jj=6;
|
|
//#pragma omp parallel for num_threads(2)\
|
|
//default(none) shared(jj,unique_idx,edges,unique_eq,std::cout,db,db2,db3,db4)
|
|
#pragma omp parallel for num_threads(1)\
|
|
default(none) shared(jj,unique_idx,edges,unique_eq,std::cout,db)
|
|
for(int k=0;k<2;k++)
|
|
{
|
|
std::string edge_sql_query = "INSERT INTO edge VALUES ";
|
|
std::string edge_assets_sql_query = "INSERT INTO edge_asset_binding VALUES ";
|
|
std::cout << "TEST CONDITION: " << ((jj/2)+(k==1)?(jj%2):0);
|
|
for(int i=0;i<jj/2+(k==1)?(jj%2):0;i++){
|
|
int idx=unique_idx[i+k*(jj/2)];
|
|
int eid = edges[idx].get_id();
|
|
if(i==0){
|
|
edge_sql_query += "(" + std::to_string(eid) + "," + unique_eq[i+k*(jj/2)];
|
|
edge_assets_sql_query += edges[idx].get_asset_query();
|
|
}
|
|
else{
|
|
edge_sql_query += ",(" + std::to_string(eid) + "," + unique_eq[i+k*(jj/2)];
|
|
edge_assets_sql_query += "," + edges[idx].get_asset_query();
|
|
}
|
|
}
|
|
edge_sql_query += ";";
|
|
edge_assets_sql_query += ";";
|
|
int thrd_num=omp_get_thread_num();
|
|
if(thrd_num==0){
|
|
db.exec("BEGIN;");
|
|
db.execAsync(edge_sql_query);//33.7s
|
|
db.execAsync(edge_assets_sql_query); //7.6s
|
|
db.execAsync("COMMIT;");//this part only takes 0.5ms
|
|
}
|
|
else if(thrd_num==1){
|
|
//db2.exec("BEGIN;");
|
|
//db2.execAsync(edge_sql_query);//33.7s
|
|
//db2.execAsync(edge_assets_sql_query); //7.6s
|
|
//db2.execAsync("COMMIT;");//this part only takes 0.5ms
|
|
}
|
|
else if(thrd_num==2){
|
|
//db3.exec("BEGIN;");
|
|
//db3.execAsync(edge_sql_query);//33.7s
|
|
//db3.execAsync(edge_assets_sql_query); //7.6s
|
|
//db3.execAsync("COMMIT;");//this part only takes 0.5ms
|
|
}
|
|
else{
|
|
//db4.exec("BEGIN;");
|
|
//db4.execAsync(edge_sql_query);//33.7s
|
|
//db4.execAsync(edge_assets_sql_query); //7.6s
|
|
//db4.execAsync("COMMIT;");//this part only takes 0.5ms
|
|
}
|
|
}
|
|
gettimeofday(&t6,NULL);
|
|
//printf("The parallel part took %lf ms\n",(t6.tv_sec-t5.tv_sec)*1000.0+(t6.tv_usec-t5.tv_usec)/1000.0);
|
|
*/
|
|
}
|
|
|
|
gettimeofday(&t2,NULL);
|
|
//printf("The saving of edge and edge_asset_binding took %lf ms\n",(t2.tv_sec-t1.tv_sec)*1000.0+(t2.tv_usec-t1.tv_usec)/1000.0);//42.0s
|
|
|
|
//this part takes around 8.6s
|
|
gettimeofday(&t1,NULL);
|
|
db.exec("BEGIN;");
|
|
int cnt1=0;
|
|
if (save_keyvalue) {
|
|
std::ostringstream out;
|
|
out << "INSERT INTO keyvalue VALUES ";
|
|
std::vector<std::string> keyvalue_vector = factlist.get_str_vector();
|
|
size_t count = 0;
|
|
for(auto &value : keyvalue_vector) {
|
|
if(count == 0)
|
|
out << "(" << std::to_string(count++) << ",'" << value << "')";
|
|
else
|
|
out << ",(" << std::to_string(count++) << ",'" << value << "')";
|
|
cnt1=cnt1+1;
|
|
}
|
|
out << ";";
|
|
db.execAsync(out.str());
|
|
}
|
|
db.execAsync("COMMIT;");
|
|
gettimeofday(&t2,NULL);
|
|
//printf("The saving of keyvalue took %lf ms\n",(t2.tv_sec-t1.tv_sec)*1000.0+(t2.tv_usec-t1.tv_usec)/1000.0);
|
|
}
|
|
|
|
void save_unexplored_to_db(NetworkState netstate) {
|
|
int fid = netstate.get_id();
|
|
const Factbase fb = netstate.get_factbase();
|
|
|
|
auto facts_tuple = fb.get_facts_tuple();
|
|
auto qualities = std::get<0>(facts_tuple);
|
|
auto topologies = std::get<1>(facts_tuple);
|
|
|
|
std::string qual_sql_query = "INSERT INTO unex_state_q VALUES ";
|
|
std::string top_sql_query = "INSERT INTO unex_state_t VALUES ";
|
|
|
|
int i = 0;
|
|
|
|
for(auto qual : qualities) {
|
|
auto asset_id = qual.get_asset_id();
|
|
auto name = qual.get_name();
|
|
auto op = qual.get_op();
|
|
auto val = qual.get_value();
|
|
|
|
if (i==0) {
|
|
qual_sql_query += "(" + std::to_string(fid) + "," \
|
|
+ std::to_string(asset_id) + "," + "'" + name + "'" + ","\
|
|
+ "'" + op +"'" + "," + "'" + val + "'" + ")";
|
|
|
|
}
|
|
else {
|
|
qual_sql_query += ",(" + std::to_string(fid) + "," \
|
|
+ std::to_string(asset_id) + "," + "'" + name + "'" + ","\
|
|
+ "'" + op +"'" + "," + "'" + val + "'" + ")";
|
|
}
|
|
|
|
++i;
|
|
}
|
|
|
|
i = 0;
|
|
|
|
for (auto ti : topologies) {
|
|
auto from_id = ti.get_from_asset_id();
|
|
auto to_id = ti.get_to_asset_id();
|
|
auto dir = ti.get_dir();
|
|
auto prop = ti.get_property();
|
|
auto op = ti.get_op();
|
|
auto val = ti.get_value();
|
|
|
|
if (i == 0) {
|
|
top_sql_query += "(" + std::to_string(fid) + "," \
|
|
+ std::to_string(from_id) + ","\
|
|
+ std::to_string(to_id) + ","\
|
|
+ std::to_string(dir) + ","\
|
|
+ "'" + prop + "'" + ","\
|
|
+ "'" + op + "'" + ","\
|
|
+ "'" + val + "'" + ")";
|
|
}
|
|
else {
|
|
top_sql_query += ",(" + std::to_string(fid) + "," \
|
|
+ std::to_string(from_id) + ","\
|
|
+ std::to_string(to_id) + ","\
|
|
+ std::to_string(dir) + ","\
|
|
+ "'" + prop + "'" + ","\
|
|
+ "'" + op + "'" + ","\
|
|
+ "'" + val + "'" + ")";
|
|
}
|
|
|
|
++i;
|
|
}
|
|
|
|
qual_sql_query += " ON CONFLICT DO NOTHING;";
|
|
top_sql_query += " ON CONFLICT DO NOTHING;";
|
|
|
|
struct timeval t1,t2;
|
|
gettimeofday(&t1,NULL);
|
|
|
|
db.execAsync(qual_sql_query);
|
|
db.execAsync(top_sql_query);
|
|
|
|
gettimeofday(&t2,NULL);
|
|
//printf("The saving of unexplored states took %lf ms\n",(t2.tv_sec-t1.tv_sec)*1000.0+(t2.tv_usec-t1.tv_usec)/1000.0);
|
|
}
|
|
|
|
//Fetch Qualities, Topologies, reform into a factbase, then return the factbase
|
|
NetworkState fetch_unexplored(Keyvalue &facts){
|
|
std::vector<Quality> qualities;
|
|
std::vector<Topology> topologies;
|
|
int id;
|
|
|
|
//Pulls the fid from the first row in the unex_state Table, deletes all rows with this fid, then returns all deleted rows
|
|
std::string qdel_str = "DELETE FROM unex_state_q WHERE fid=(SELECT fid FROM unex_state_q LIMIT 1) RETURNING *;";
|
|
std::string tdel_str = "DELETE FROM unex_state_t WHERE fid=(SELECT fid FROM unex_state_t LIMIT 1) RETURNING *;";
|
|
|
|
std::vector<Row> qrows = db.exec(qdel_str);
|
|
|
|
for (auto &row : qrows) {
|
|
int fid = std::stoi(row[0]);
|
|
int asset_id = std::stoi(row[1]);
|
|
std::string property = row[2];
|
|
std::string op = row[3];
|
|
std::string value = row[4];
|
|
|
|
Quality qual(asset_id, property, op, value, facts); //associate each record from unex_state table with the facts(the hash_table and fact string vector).
|
|
qualities.push_back(qual);
|
|
id = fid;
|
|
}
|
|
|
|
std::vector<Row> trows = db.exec(tdel_str);
|
|
for (auto &row : trows) {
|
|
//int fid = std::stoi(row[0]);
|
|
int from_asset = std::stoi(row[1]);
|
|
int to_asset = std::stoi(row[2]);
|
|
std::string dir_str = row[3];
|
|
|
|
DIRECTION_T dir; //DIRECTION_T is a user defined enum type (in topology.h), it has three different values: FORWARD_T 0, BACKWARD_T 1, and BIDIRECTION_T 2
|
|
if(dir_str == "->" || dir_str == "0") {
|
|
dir = FORWARD_T;
|
|
} else if (dir_str == "<-" || dir_str == "1") {
|
|
dir = BACKWARD_T;
|
|
} else if (dir_str == "<->" || dir_str == "2") {
|
|
dir = BIDIRECTION_T;
|
|
} else {
|
|
std::cerr << "Unknown direction '" << dir_str << "'" << std::endl;
|
|
exit(1);
|
|
}
|
|
|
|
std::string property = row[4];
|
|
std::string op = row[5];
|
|
std::string value = row[6];
|
|
|
|
Topology t(from_asset, to_asset, dir, property, op, value, facts); //associate each record from topology table with the facts
|
|
topologies.push_back(t);
|
|
}
|
|
|
|
NetworkState fetched_state(qualities, topologies);
|
|
fetched_state.force_set_id(id);
|
|
|
|
return fetched_state;
|
|
}
|
|
|
|
bool unex_empty()
|
|
{
|
|
std::vector<Row> row = db.exec("SELECT count(*) FROM unex_state_q LIMIT 1;");
|
|
for (auto &r : row){
|
|
auto count = std::stoi(r[0]);
|
|
if (count == 0)
|
|
return true;
|
|
else
|
|
return false;
|
|
}
|
|
}
|
|
|
|
int get_num_states()
|
|
{
|
|
std::vector<Row> row = db.exec("SELECT count(*) AS exact_count FROM factbase;");
|
|
for (auto &r : row) {
|
|
return std::stoi(r[0]);
|
|
}
|
|
}
|
|
|
|
int get_num_edges()
|
|
{
|
|
std::vector<Row> row = db.exec("SELECT count(*) AS exact_count FROM edge;");
|
|
for (auto &r : row) {
|
|
return std::stoi(r[0]);
|
|
}
|
|
} |