Add params for max user count, max number of {priv,pub} {in,out} messages per user per epoch

Also preliminary support for roles

App/appconfig.cpp

@@ -72,8 +72,18 @@ bool config_parse(Config &config, const std::string configstr,
     for (auto & entry : conftree) {
         if (!entry.first.compare("params")) {
             for (auto & pentry : entry.second) {
-                if (!pentry.first.compare("msgsize")) {
-                    config.msgsize = pentry.second.get_value<uint16_t>();
+                if (!pentry.first.compare("msg_size")) {
+                    config.msg_size = pentry.second.get_value<uint16_t>();
+                } else if (!pentry.first.compare("user_count")) {
+                    config.user_count = pentry.second.get_value<uint32_t>();
+                } else if (!pentry.first.compare("priv_out")) {
+                    config.m_priv_out = pentry.second.get_value<uint8_t>();
+                } else if (!pentry.first.compare("priv_in")) {
+                    config.m_priv_in = pentry.second.get_value<uint8_t>();
+                } else if (!pentry.first.compare("pub_out")) {
+                    config.m_pub_out = pentry.second.get_value<uint8_t>();
+                } else if (!pentry.first.compare("pub_in")) {
+                    config.m_pub_in = pentry.second.get_value<uint8_t>();
                 } else {
                     std::cerr << "Unknown field in params: " <<
                         pentry.first << "\n";
@@ -84,7 +94,9 @@ bool config_parse(Config &config, const std::string configstr,
         } else if (!entry.first.compare("nodes")) {
             for (auto & node : entry.second) {
                 NodeConfig nc;
-                nc.weight = 1;  // default
+                // defaults
+                nc.weight = 1;
+                nc.roles = ROLE_INGESTION | ROLE_ROUTING | ROLE_STORAGE;
                 for (auto & nentry : node.second) {
                     if (!nentry.first.compare("name")) {
                         nc.name = nentry.second.get_value<std::string>();
@@ -133,7 +145,12 @@ bool config_parse(Config &config, const std::string configstr,
 
     // Now load the config into the enclave
     EnclaveAPIParams apiparams;
-    apiparams.msgsize = config.msgsize;
+    apiparams.user_count = config.user_count;
+    apiparams.msg_size = config.msg_size;
+    apiparams.m_priv_out = config.m_priv_out;
+    apiparams.m_priv_in = config.m_priv_in;
+    apiparams.m_pub_out = config.m_pub_out;
+    apiparams.m_pub_in = config.m_pub_in;
     nodenum_t num_nodes = (nodenum_t)(config.nodes.size());
     std::vector<EnclaveAPINodeConfig> apinodeconfigs;
     apinodeconfigs.resize(num_nodes);
@@ -141,6 +158,7 @@ bool config_parse(Config &config, const std::string configstr,
         memmove(&apinodeconfigs[i].pubkey,
             &config.nodes[i].pubkey, sizeof(apinodeconfigs[i].pubkey));
         apinodeconfigs[i].weight = config.nodes[i].weight;
+        apinodeconfigs[i].roles = config.nodes[i].roles;
     }
     ret &= ecall_config_load(&apiparams, apinodeconfigs.data(),
         num_nodes, config.my_node_num);

App/appconfig.hpp

@@ -14,12 +14,18 @@ struct NodeConfig {
     std::string listenhost, listenport;
     std::string clistenhost, clistenport;
     uint8_t weight;
+    uint8_t roles;
 };
 
 // The global config
 struct Config {
     // global params
-    uint16_t msgsize;
+    uint32_t user_count;
+    uint16_t msg_size;
+    uint8_t m_priv_out;
+    uint8_t m_priv_in;
+    uint8_t m_pub_out;
+    uint8_t m_pub_in;
     // config for each node
     std::vector<NodeConfig> nodes;
     // Which node is this one?

App/launch

@@ -45,6 +45,18 @@ if __name__ == "__main__":
         help='manifest.yaml file')
     aparse.add_argument('-p', default=PUBKEYS,
         help='pubkeys.yaml file')
+    aparse.add_argument('-z', default=None,
+        help='override message size')
+    aparse.add_argument('-u', default=None,
+        help='override max number of users')
+    aparse.add_argument('-B', default=None,
+        help='override max number of outgoing private messages per user per epoch')
+    aparse.add_argument('-b', default=None,
+        help='override max number of incoming private messages per user per epoch')
+    aparse.add_argument('-C', default=None,
+        help='override max number of outgoing public messages per user per epoch')
+    aparse.add_argument('-c', default=None,
+        help='override max number of incoming public messages per user per epoch')
     aparse.add_argument('-n', nargs='*', help='nodes to include')
     aparse.add_argument('cmd', nargs='*', help='experiment to run')
     args = aparse.parse_args()
@@ -52,7 +64,16 @@ if __name__ == "__main__":
     with open(args.m) as mf:
         manifest = yaml.safe_load(mf)
 
-    config = mkconfig.create_json(args.m, args.p, args.n)
+    params_overrides = {
+        'msg_size': args.z,
+        'user_count': args.u,
+        'priv_out': args.B,
+        'priv_in': args.b,
+        'pub_out': args.C,
+        'pub_in': args.c,
+    }
+
+    config = mkconfig.create_json(args.m, args.p, args.n, params_overrides)
     # There must not be any newlines in the config json string
     if "\n" in config:
         print("Error: config.json must not contain embedded newlines")

App/mkconfig.py

@@ -18,11 +18,12 @@ MANIFEST = "manifest.yaml"
 # The default pubkeys file
 PUBKEYS = "pubkeys.yaml"
 
-def create_json(manifestfile, pubkeysfile, nodelist):
+def create_json(manifestfile, pubkeysfile, nodelist, params_override):
     """Given a manifest.yaml file, a pubkeys.yaml file, and (optionally) a
-    list of nodes to include, generate the config.json file to give to
-    each of the nodes.  If the list of nodes is missing, include all nodes
-    in the manifest.yaml file.  It is an error to include a node that is
+    list of nodes to include and a dictionary of parameter settings to
+    override, generate the config.json file to give to each of the
+    nodes.  If the list of nodes is missing, include all nodes in the
+    manifest.yaml file.  It is an error to include a node that is
     missing from either the manifest.yaml or the pubkeys.yaml files."""
     with open(manifestfile) as mf:
         manifest = yaml.safe_load(mf)
@@ -33,6 +34,9 @@ def create_json(manifestfile, pubkeysfile, nodelist):
     config = {}
     if "params" in manifest:
         config['params'] = manifest['params']
+        for ov in params_override:
+            if params_override[ov] is not None:
+                config['params'][ov] = params_override[ov]
     config['nodes'] = []
     for node in nodelist:
         if node == "params":
@@ -57,10 +61,30 @@ if __name__ == "__main__":
         help='manifest.yaml file')
     aparse.add_argument('-p', default=PUBKEYS,
         help='pubkeys.yaml file')
+    aparse.add_argument('-z', default=None,
+        help='override message size')
+    aparse.add_argument('-u', default=None,
+        help='override max number of users')
+    aparse.add_argument('-B', default=None,
+        help='override max number of outgoing private messages per user per epoch')
+    aparse.add_argument('-b', default=None,
+        help='override max number of incoming private messages per user per epoch')
+    aparse.add_argument('-C', default=None,
+        help='override max number of outgoing public messages per user per epoch')
+    aparse.add_argument('-c', default=None,
+        help='override max number of incoming public messages per user per epoch')
     aparse.add_argument('-n', nargs='*', help='nodes to include')
     args = aparse.parse_args()
 
-    json = create_json(args.m, args.p, args.n)
+    params_overrides = {
+        'msg_size': args.z,
+        'user_count': args.u,
+        'priv_out': args.B,
+        'priv_in': args.b,
+        'pub_out': args.C,
+        'pub_in': args.c,
+    }
+    json = create_json(args.m, args.p, args.n, params_overrides)
 
     if json is not None:
         print(json)

Enclave/config.cpp

@@ -9,16 +9,73 @@ bool ecall_config_load(EnclaveAPIParams *apiparams,
     nodenum_t num_nodes, nodenum_t my_node_num)
 {
     g_teems_config.num_nodes = num_nodes;
+    g_teems_config.num_ingestion_nodes = 0;
+    g_teems_config.num_routing_nodes = 0;
+    g_teems_config.num_storage_nodes = 0;
     g_teems_config.my_node_num = my_node_num;
-    g_teems_config.msgsize = apiparams->msgsize;
+    g_teems_config.user_count = apiparams->user_count;
+    g_teems_config.msg_size = apiparams->msg_size;
+    g_teems_config.m_priv_out = apiparams->m_priv_out;
+    g_teems_config.m_priv_in = apiparams->m_priv_in;
+    g_teems_config.m_pub_out = apiparams->m_pub_out;
+    g_teems_config.m_pub_in = apiparams->m_pub_in;
+    // Temporary vectors to store node numbers for nodes of different
+    // types, where the node numbers are smaller than our own node
+    // number
+    std::vector<nodenum_t> ing_smaller, rte_smaller, str_smaller;
     uint16_t cumul_weight = 0;
     g_teems_config.weights.clear();
+    g_teems_config.ingestion_nodes.clear();
+    g_teems_config.routing_nodes.clear();
+    g_teems_config.storage_nodes.clear();
     for (nodenum_t i=0; i<num_nodes; ++i) {
         NodeWeight nw;
         nw.startweight = cumul_weight;
         nw.weight = apinodeconfigs[i].weight;
+        if (apinodeconfigs[i].roles & ROLE_INGESTION) {
+            g_teems_config.num_ingestion_nodes += 1;
+            if (i < my_node_num) {
+                ing_smaller.push_back(i);
+            } else {
+                g_teems_config.ingestion_nodes.push_back(i);
+            }
+        }
+        if (apinodeconfigs[i].roles & ROLE_ROUTING) {
+            g_teems_config.num_routing_nodes += 1;
+            if (i < my_node_num) {
+                rte_smaller.push_back(i);
+            } else {
+                g_teems_config.routing_nodes.push_back(i);
+            }
+        }
+        if (apinodeconfigs[i].roles & ROLE_STORAGE) {
+            g_teems_config.num_storage_nodes += 1;
+            if (i < my_node_num) {
+                str_smaller.push_back(i);
+            } else {
+                g_teems_config.storage_nodes.push_back(i);
+            }
+        }
         cumul_weight += nw.weight;
         g_teems_config.weights.push_back(nw);
     }
+    // Concatenate the *_smaller vectors to the ends of the
+    // g_teems_config.*_nodes vectors.  This way, each node has a list
+    // of nodes of each role starting with itself and "looping around".
+    // This should make the communication pattern have less of a
+    // bottleneck.
+    g_teems_config.ingestion_nodes.insert(
+        g_teems_config.ingestion_nodes.end(),
+        ing_smaller.begin(),
+        ing_smaller.end());
+    g_teems_config.routing_nodes.insert(
+        g_teems_config.routing_nodes.end(),
+        rte_smaller.begin(),
+        rte_smaller.end());
+    g_teems_config.storage_nodes.insert(
+        g_teems_config.storage_nodes.end(),
+        str_smaller.begin(),
+        str_smaller.end());
+
     return comms_init_nodestate(apinodeconfigs, num_nodes, my_node_num);
 }

Enclave/config.hpp

@@ -16,9 +16,20 @@ struct NodeWeight {
 
 struct Config {
     nodenum_t num_nodes;
+    nodenum_t num_ingestion_nodes;
+    nodenum_t num_routing_nodes;
+    nodenum_t num_storage_nodes;
     nodenum_t my_node_num;
-    uint16_t msgsize;
+    uint32_t user_count;
+    uint16_t msg_size;
+    uint8_t m_priv_out;
+    uint8_t m_priv_in;
+    uint8_t m_pub_out;
+    uint8_t m_pub_in;
     std::vector<NodeWeight> weights;
+    std::vector<nodenum_t> ingestion_nodes;
+    std::vector<nodenum_t> routing_nodes;
+    std::vector<nodenum_t> storage_nodes;
 };
 
 extern Config g_teems_config;

Enclave/enclave_api.h

@@ -6,12 +6,22 @@
 typedef uint16_t nodenum_t;
 
 struct EnclaveAPIParams {
-    uint16_t msgsize;
+    uint32_t user_count;
+    uint16_t msg_size;
+    uint8_t m_priv_out;
+    uint8_t m_priv_in;
+    uint8_t m_pub_out;
+    uint8_t m_pub_in;
 };
 
+#define ROLE_INGESTION 0x01
+#define ROLE_ROUTING 0x02
+#define ROLE_STORAGE 0x04
+
 struct EnclaveAPINodeConfig {
     sgx_ec256_public_t pubkey;
     uint8_t weight;
+    uint8_t roles;
 };
 
 #define SEALED_PRIVKEY_SIZE 610