Changes to support public routing protocol

App/net.cpp

@@ -221,9 +221,13 @@ void NetIO::ing_receive_msgbundle(tcp::socket* csocket, clientid_t c_simid)
             return;
         }
 
-
+        bool ret;
         //Ingest the message_bundle
-        bool ret = ecall_ingest_msgbundle(c_simid, msgbundle, conf.m_priv_out);
+        if(conf.private_routing) {
+            ret = ecall_ingest_msgbundle(c_simid, msgbundle, conf.m_priv_out);
+        } else {
+            ret = ecall_ingest_msgbundle(c_simid, msgbundle, conf.m_pub_out);
+        }
         free(msgbundle);
 
         // Continue to async receive client message bundles
@@ -486,15 +490,25 @@ NetIO::NetIO(boost::asio::io_context &io_context, const Config &config)
 
 
     auth_size = sizeof(clientid_t) + sizeof(unsigned long) + SGX_AESGCM_KEY_SIZE;
-    msgbundle_size = SGX_AESGCM_IV_SIZE
-        + (conf.m_priv_out * (conf.msg_size + TOKEN_SIZE))
-        + SGX_AESGCM_MAC_SIZE;
-    uint16_t priv_out = config.m_priv_out;
-    token_bundle_size = ((priv_out * TOKEN_SIZE)
-        + SGX_AESGCM_IV_SIZE + SGX_AESGCM_MAC_SIZE);
-    uint16_t priv_in = conf.m_priv_in;
-    mailbox_size = (priv_in * conf.msg_size) + SGX_AESGCM_IV_SIZE
-        + SGX_AESGCM_MAC_SIZE;
+    uint16_t priv_out, priv_in, pub_in;
+    if(config.private_routing) {
+        priv_out = conf.m_priv_out;
+        priv_in = conf.m_priv_in;
+        msgbundle_size = SGX_AESGCM_IV_SIZE
+            + (conf.m_priv_out * (conf.msg_size + TOKEN_SIZE))
+            + SGX_AESGCM_MAC_SIZE;
+        token_bundle_size = ((priv_out * TOKEN_SIZE)
+            + SGX_AESGCM_IV_SIZE + SGX_AESGCM_MAC_SIZE);
+        mailbox_size = (priv_in * conf.msg_size) + SGX_AESGCM_IV_SIZE
+            + SGX_AESGCM_MAC_SIZE;
+    } else {
+        pub_in = conf.m_pub_in;
+        msgbundle_size = SGX_AESGCM_IV_SIZE
+            + (conf.m_pub_out * conf.msg_size)
+            + SGX_AESGCM_MAC_SIZE;
+        mailbox_size = (pub_in * conf.msg_size) + SGX_AESGCM_IV_SIZE
+            + SGX_AESGCM_MAC_SIZE;
+    }
 
 
     if(myconf.roles & ROLE_STORAGE) {

Client/clients.cpp

@@ -23,6 +23,7 @@ std::vector<uint16_t> storage_map;
 std::vector<uint16_t> ingestion_map;
 unsigned long setup_time;
 uint16_t nthreads = 1;
+bool private_routing;
 
 // Split a hostport string like "127.0.0.1:12000" at the rightmost colon
 // into a host part "127.0.0.1" and a port part "12000".
@@ -130,6 +131,15 @@ void displayEncMessageBundle(unsigned char *bundle, uint16_t priv_out,
     printf("\n");
 }
 
+static inline uint32_t encPubMsgBundleSize(uint16_t pub_out, uint16_t msg_size)
+{
+    return(SGX_AESGCM_IV_SIZE + (pub_out * msg_size) + SGX_AESGCM_MAC_SIZE);
+}
+
+static inline uint32_t ptPubMsgBundleSize(uint16_t pub_out, uint16_t msg_size)
+{
+    return(pub_out * msg_size);
+}
 
 static inline uint32_t encMsgBundleSize(uint16_t priv_out, uint16_t msg_size)
 {
@@ -185,7 +195,8 @@ bool config_parse(Config &config, const std::string configstr,
                 } else if (!pentry.first.compare("master_secret")) {
                     std::string hex_key = pentry.second.data();
                     memcpy(config.master_secret, hex_key.c_str(), SGX_AESGCM_KEY_SIZE);
-
+                } else if (!pentry.first.compare("private_routing")) {
+                    config.private_routing = pentry.second.get_value<bool>();
                 } else {
                     std::cerr << "Unknown field in params: " <<
                         pentry.first << "\n";
@@ -519,8 +530,10 @@ void Client::generateMessageBundle(uint8_t priv_out, uint32_t msg_size,
         ptr+=(remaining_message_size);
     }
 
-    // Add the tokens for this msgbundle
-    memcpy(ptr, token_list, config.m_priv_out * TOKEN_SIZE);
+    if(private_routing) {
+        // Add the tokens for this msgbundle
+        memcpy(ptr, token_list, config.m_priv_out * TOKEN_SIZE);
+    }
 }
 
 
@@ -550,16 +563,26 @@ bool Client::encryptMessageBundle(uint32_t enc_bundle_size, unsigned char *pt_ms
 
 void Client::sendMessageBundle()
 {
-
     uint16_t priv_out = config.m_priv_out;
+    uint16_t pub_out = config.m_pub_out;
     uint16_t msg_size = config.msg_size;
-    uint32_t send_pt_msgbundle_size = ptMsgBundleSize(priv_out, msg_size);
-    uint32_t send_enc_msgbundle_size = encMsgBundleSize(priv_out, msg_size);
-    unsigned char *send_pt_msgbundle = (unsigned char*) malloc (send_pt_msgbundle_size);
-    unsigned char *send_enc_msgbundle = (unsigned char*) malloc (send_enc_msgbundle_size);
+    uint32_t send_pt_msgbundle_size, send_enc_msgbundle_size;
 
-    generateMessageBundle(priv_out, msg_size, send_pt_msgbundle);
+    if(private_routing) {
+        send_pt_msgbundle_size = ptMsgBundleSize(priv_out, msg_size);
+        send_enc_msgbundle_size = encMsgBundleSize(priv_out, msg_size);
+    } else {
+        send_pt_msgbundle_size = ptPubMsgBundleSize(pub_out, msg_size);
+        send_enc_msgbundle_size = encPubMsgBundleSize(pub_out, msg_size);
+    }
 
+    unsigned char *send_pt_msgbundle = (unsigned char*) malloc (send_pt_msgbundle_size);
+    unsigned char *send_enc_msgbundle = (unsigned char*) malloc (send_enc_msgbundle_size);
+    if(private_routing) {
+        generateMessageBundle(priv_out, msg_size, send_pt_msgbundle);
+    } else {
+        generateMessageBundle(pub_out, msg_size, send_pt_msgbundle);
+    }
     encryptMessageBundle(send_enc_msgbundle_size, send_pt_msgbundle, send_enc_msgbundle);
 
 #ifdef VERBOSE_CLIENT
@@ -777,64 +800,105 @@ void Client::epoch_process() {
         + SGX_AESGCM_MAC_SIZE;
     unsigned char *enc_tokens = (unsigned char*) malloc (token_bundle_size);
 
-    //Async read the encrypted tokens for this epoch
-    boost::asio::async_read(*storage_sock, boost::asio::buffer(enc_tokens, token_bundle_size),
-        [this, enc_tokens, token_bundle_size, pt_token_size]
-        (boost::system::error_code ec, std::size_t) {
+    if(private_routing) {
+        //Async read the encrypted tokens for this epoch
+        boost::asio::async_read(*storage_sock, boost::asio::buffer(enc_tokens, token_bundle_size),
+            [this, enc_tokens, token_bundle_size, pt_token_size]
+            (boost::system::error_code ec, std::size_t) {
 
-        if (ec) {
-            if(ec == boost::asio::error::eof) {
-                delete(storage_sock);
-            }
-            else {
-                printf("Error %s\n", ec.message().c_str());
-                printf("Client::epoch_process boost async_read_tokens failed\n");
+            if (ec) {
+                if(ec == boost::asio::error::eof) {
+                    delete(storage_sock);
+                }
+                else {
+                    printf("Error %s\n", ec.message().c_str());
+                    printf("Client::epoch_process boost async_read_tokens failed\n");
+                }
+                return;
             }
-            return;
-        }
 
-#ifdef VERBOSE_CLIENT
-        if(sim_id == 0) {
-            printf("TEST: Client 0: Encrypted token bundle received:\n");
-            for(uint32_t i = 0; i < token_bundle_size; i++) {
-                printf("%x", enc_tokens[i]);
+    #ifdef VERBOSE_CLIENT
+            if(sim_id == 0) {
+                printf("TEST: Client 0: Encrypted token bundle received:\n");
+                for(uint32_t i = 0; i < token_bundle_size; i++) {
+                    printf("%x", enc_tokens[i]);
+                }
+                printf("\n");
             }
-            printf("\n");
-        }
-#endif
+    #endif
 
 
-        // Decrypt the token bundle
-        unsigned char *enc_tkn_ptr = enc_tokens + SGX_AESGCM_IV_SIZE;
-        unsigned char *enc_tkn_tag = enc_tokens + SGX_AESGCM_IV_SIZE + pt_token_size;
+            // Decrypt the token bundle
+            unsigned char *enc_tkn_ptr = enc_tokens + SGX_AESGCM_IV_SIZE;
+            unsigned char *enc_tkn_tag = enc_tokens + SGX_AESGCM_IV_SIZE + pt_token_size;
 
 
-        int decrypted_bytes =  gcm_decrypt(enc_tkn_ptr, pt_token_size,
-                NULL, 0, enc_tkn_tag, (unsigned char*) &(this->stg_key),
-                enc_tokens, SGX_AESGCM_IV_SIZE, (unsigned char*) (this->token_list));
+            int decrypted_bytes =  gcm_decrypt(enc_tkn_ptr, pt_token_size,
+                    NULL, 0, enc_tkn_tag, (unsigned char*) &(this->stg_key),
+                    enc_tokens, SGX_AESGCM_IV_SIZE, (unsigned char*) (this->token_list));
 
-        if(decrypted_bytes != pt_token_size) {
-            printf("Client::epoch_process gcm_decrypt tokens failed. decrypted_bytes = %d \n", decrypted_bytes);
-        }
+            if(decrypted_bytes != pt_token_size) {
+                printf("Client::epoch_process gcm_decrypt tokens failed. decrypted_bytes = %d \n", decrypted_bytes);
+            }
 
-        free(enc_tokens);
+            free(enc_tokens);
 
-        /*
-        unsigned char *tkn_ptr = (unsigned char*) this->token_list;
-        if(sim_id==0) {
-            printf("TEST: Client 0: Decrypted client tokens:\n");
-            for(int i = 0; i < 2 * SGX_AESGCM_KEY_SIZE; i++) {
-                printf("%x", tkn_ptr[i]);
+            /*
+            unsigned char *tkn_ptr = (unsigned char*) this->token_list;
+            if(sim_id==0) {
+                printf("TEST: Client 0: Decrypted client tokens:\n");
+                for(int i = 0; i < 2 * SGX_AESGCM_KEY_SIZE; i++) {
+                    printf("%x", tkn_ptr[i]);
+                }
+                printf("\n");
             }
-            printf("\n");
-        }
-        */
+            */
+
+            // Async read the messages recieved in the last epoch
+            uint16_t priv_in = config.m_priv_in;
+            uint16_t msg_size = config.msg_size;
+            uint32_t recv_pt_mailbox_size = ptMailboxSize(priv_in, msg_size);
+            uint32_t recv_enc_mailbox_size = encMailboxSize(priv_in, msg_size);
+            unsigned char *recv_pt_mailbox = (unsigned char*) malloc (recv_pt_mailbox_size);
+            unsigned char *recv_enc_mailbox = (unsigned char*) malloc (recv_enc_mailbox_size);
+
+            boost::asio::async_read(*storage_sock,
+                boost::asio::buffer(recv_enc_mailbox, recv_enc_mailbox_size),
+                [this, recv_pt_mailbox, recv_enc_mailbox]
+                (boost::system::error_code ecc, std::size_t) {
+
+                if (ecc) {
+                    if(ecc == boost::asio::error::eof) {
+                        delete(storage_sock);
+                    }
+                    else {
+                        printf("Error %s\n", ecc.message().c_str());
+                    }
+                    printf("Client: boost async_read failed for recieving msg_bundle\n");
+                    return;
+                }
+
+    #ifdef VERBOSE_CLIENT
+                if(sim_id == 0) {
+                    printf("TEST: Client 0: Encrypted msgbundle received\n");
+                }
+    #endif
+
+                // Do whatever processing with the received messages here
+                free(recv_enc_mailbox);
+                free(recv_pt_mailbox);
 
+                // Send this epoch's message bundle
+                sendMessageBundle();
+                epoch_process();
+            });
+        });
+    } else {
         // Async read the messages recieved in the last epoch
-        uint16_t priv_in = config.m_priv_in;
+        uint16_t pub_in = config.m_pub_in;
         uint16_t msg_size = config.msg_size;
-        uint32_t recv_pt_mailbox_size = ptMailboxSize(priv_in, msg_size);
-        uint32_t recv_enc_mailbox_size = encMailboxSize(priv_in, msg_size);
+        uint32_t recv_pt_mailbox_size = ptMailboxSize(pub_in, msg_size);
+        uint32_t recv_enc_mailbox_size = encMailboxSize(pub_in, msg_size);
         unsigned char *recv_pt_mailbox = (unsigned char*) malloc (recv_pt_mailbox_size);
         unsigned char *recv_enc_mailbox = (unsigned char*) malloc (recv_enc_mailbox_size);
 
@@ -854,12 +918,6 @@ void Client::epoch_process() {
                 return;
             }
 
-#ifdef VERBOSE_CLIENT
-            if(sim_id == 0) {
-                printf("TEST: Client 0: Encrypted msgbundle received\n");
-            }
-#endif
-
             // Do whatever processing with the received messages here
             free(recv_enc_mailbox);
             free(recv_pt_mailbox);
@@ -868,8 +926,7 @@ void Client::epoch_process() {
             sendMessageBundle();
             epoch_process();
         });
-    });
-
+    }
 }
 
 void client_epoch_process(uint32_t cstart, uint32_t cstop)
@@ -955,6 +1012,7 @@ int main(int argc, char **argv)
        exit(1);
     }
 
+    private_routing = config.private_routing;
     clients = new Client[config.user_count];
 #ifdef VERBOSE_CLIENT
     printf("Number of ingestion_nodes = %ld, Number of storage_node = %ld\n",
@@ -974,20 +1032,24 @@ int main(int argc, char **argv)
     // Start background threads; one will perform the work and the other
     // will execute the async_write handlers
     // TODO: Cleanup and distribute this based on nthreads.
-    // Currently assumes 8 threads are available for client simulator.
+    // Currently assumes 4 threads are available for client simulator on chime.
     boost::thread t([&]{io_context.run();});
     boost::thread t2([&]{io_context.run();});
     boost::thread t3([&]{io_context.run();});
+    /*
     boost::thread t4([&]{io_context.run();});
     boost::thread t5([&]{io_context.run();});
     boost::thread t6([&]{io_context.run();});
+    */
     io_context.run();
     t.join();
     t2.join();
     t3.join();
+    /*
     t4.join();
     t5.join();
     t6.join();
+    */
 
     delete [] clients;
 }

Enclave/ingest.cpp

@@ -67,7 +67,11 @@ void Ingestion::initialize(uint32_t num, uint32_t start, sgx_aes_gcm_128bit_key_
 
     generateClientKeys(ESK);
 
-    max_buffer_size = g_teems_config.m_priv_out * cnum;
+    if(g_teems_config.private_routing) {
+        max_buffer_size = g_teems_config.m_priv_out * cnum;
+    } else {
+        max_buffer_size = g_teems_config.m_pub_out * cnum;
+    }
     buffer = &(route_state.ingbuf);
 }
 
@@ -90,7 +94,12 @@ bool Ingestion::processMsgBundle(clientid_t cid, unsigned char *msgbundle,
     msgbundle += SGX_AESGCM_IV_SIZE;
 
     uint16_t msg_size = g_teems_config.msg_size;
-    uint32_t msgbundle_size = num_msgs * (msg_size + TOKEN_SIZE);
+    uint32_t msgbundle_size;
+    if(g_teems_config.private_routing) {
+        msgbundle_size = num_msgs * (msg_size + TOKEN_SIZE);
+    } else {
+        msgbundle_size = num_msgs * msg_size;
+    }
     unsigned char *dec_msgbundle = (unsigned char *) malloc (msgbundle_size);
     sgx_aes_gcm_128bit_tag_t *tag = (sgx_aes_gcm_128bit_tag_t*) (msgbundle + msgbundle_size);
 
@@ -106,46 +115,48 @@ bool Ingestion::processMsgBundle(clientid_t cid, unsigned char *msgbundle,
     // TODO: Verify the tokens from end of the msgbundle
     // before appending them into the MsgBuffer
     bool verified  = true;
-    unsigned char token_body[TOKEN_SIZE];
-    const sgx_aes_gcm_128bit_key_t *pTSK = &(g_teems_config.TSK);
-    unsigned char *dm_ptr = dec_msgbundle;
-    unsigned char *tkn_ptr = dm_ptr + (msg_size * num_msgs);
-    sgx_cmac_state_handle_t cmac_handle;
-    sgx_cmac_128bit_tag_t token;
-
-    for(int k =0; k < num_msgs; k++)
-    {
-        ret = SGX_SUCCESS;
-        memset(token_body, 0, TOKEN_SIZE);
-        memcpy(token_body, dm_ptr, sizeof(clientid_t) * 2);
-        memcpy(token_body + sizeof(clientid_t) * 2, (uint8_t*) &ingestion_epoch, sizeof(ingestion_epoch));
-        ret = sgx_rijndael128_cmac_msg(pTSK, token_body, TOKEN_SIZE,
-            (sgx_cmac_128bit_tag_t*) &token);
-        if(ret!=SGX_SUCCESS) {
-            printf("Ingestion::processMsgBundle: Token recreation FAIL\n");
-            printf("ret = %x", ret);
-            return false;
-        }
-
-        /*
-        printf("Received token:\n");
-        for(int l=0; l<SGX_CMAC_MAC_SIZE; l++) {
-            printf("%x", tkn_ptr[l]);
-        }
-        printf("Verify created token:\n");
-        for(int l=0; l<SGX_CMAC_MAC_SIZE; l++) {
-            printf("%x", token[l]);
+    if(g_teems_config.private_routing) {
+        unsigned char token_body[TOKEN_SIZE];
+        const sgx_aes_gcm_128bit_key_t *pTSK = &(g_teems_config.TSK);
+        unsigned char *dm_ptr = dec_msgbundle;
+        unsigned char *tkn_ptr = dm_ptr + (msg_size * num_msgs);
+        sgx_cmac_state_handle_t cmac_handle;
+        sgx_cmac_128bit_tag_t token;
+
+        for(int k =0; k < num_msgs; k++)
+        {
+            ret = SGX_SUCCESS;
+            memset(token_body, 0, TOKEN_SIZE);
+            memcpy(token_body, dm_ptr, sizeof(clientid_t) * 2);
+            memcpy(token_body + sizeof(clientid_t) * 2, (uint8_t*) &ingestion_epoch, sizeof(ingestion_epoch));
+            ret = sgx_rijndael128_cmac_msg(pTSK, token_body, TOKEN_SIZE,
+                (sgx_cmac_128bit_tag_t*) &token);
+            if(ret!=SGX_SUCCESS) {
+                printf("Ingestion::processMsgBundle: Token recreation FAIL\n");
+                printf("ret = %x", ret);
+                return false;
+            }
+
+            /*
+            printf("Received token:\n");
+            for(int l=0; l<SGX_CMAC_MAC_SIZE; l++) {
+                printf("%x", tkn_ptr[l]);
+            }
+            printf("Verify created token:\n");
+            for(int l=0; l<SGX_CMAC_MAC_SIZE; l++) {
+                printf("%x", token[l]);
+            }
+            */
+
+            int diff = memcmp(token, tkn_ptr, SGX_CMAC_MAC_SIZE);
+            if(diff!=0) {
+                printf("Ingestion::processMsgBundle: Tokens do not match FAIL\n");
+                return false;
+            }
+
+            dm_ptr+= msg_size;
+            tkn_ptr+=SGX_CMAC_MAC_SIZE;
         }
-        */
-
-        int diff = memcmp(token, tkn_ptr, SGX_CMAC_MAC_SIZE);
-        if(diff!=0) {
-            printf("Ingestion::processMsgBundle: Tokens do not match FAIL\n");
-            return false;
-        }
-
-        dm_ptr+= msg_size;
-        tkn_ptr+=SGX_CMAC_MAC_SIZE;
     }
 
     if(verified) {

Enclave/storage.cpp

@@ -228,7 +228,15 @@ static void *processMsgs_launch(void *voidargs) {
     uint32_t user_start = args->start;
     uint32_t user_end = args->start + args->num;
 
-    uint32_t mailbox_size = g_teems_config.m_priv_in * g_teems_config.msg_size;
+    uint32_t mailbox_size, num_expected_msgs;
+    if (g_teems_config.private_routing) {
+        mailbox_size = g_teems_config.m_priv_in * g_teems_config.msg_size;
+        num_expected_msgs = g_teems_config.m_priv_in * storage_state.max_users;
+    } else {
+        mailbox_size = g_teems_config.m_pub_in * g_teems_config.msg_size;
+        num_expected_msgs = g_teems_config.m_pub_in * storage_state.max_users;
+    }
+
     uint32_t enc_mailbox_size = mailbox_size + SGX_AESGCM_IV_SIZE + SGX_AESGCM_MAC_SIZE;
     unsigned char *epoch_buf_ptr = epoch_mailboxes +
         enc_mailbox_size * user_start;
@@ -377,7 +385,7 @@ void storage_received(MsgBuffer &storage_buf)
     printf_with_rtclock_diff(start_sort, "end oblivious sort (%u)\n", storage_buf.inserted);
 #endif
 
-    // For public routing, remove excess per-user messages by making them 
+    // For public routing, remove excess per-user messages by making them
     // padding, and then compact non-padding messages.
     if (!g_teems_config.private_routing) {
         uint8_t *msg = storage_state.stg_buf.buf;
@@ -390,7 +398,7 @@ void storage_received(MsgBuffer &storage_buf)
             num_user_msgs = oselect_uint32_t(1, num_user_msgs+1,
                 uid == prev_uid);
             // Select if messages per user not exceeded and msg is not padding
-            sel = ((uint8_t) ((num_user_msgs <= g_teems_config.m_pub_in))) &   
+            sel = ((uint8_t) ((num_user_msgs <= g_teems_config.m_pub_in))) &
                 ((uint8_t) uid != uid_mask);
             storage_state.pub_selected[i] = (bool) sel;
             // Make padding if not selected
@@ -483,9 +491,10 @@ void storage_received(MsgBuffer &storage_buf)
     storage_buf.reset();
     pthread_mutex_unlock(&storage_buf.mutex);
 
-    bool ret = generate_all_tokens();
+    if (g_teems_config.private_routing) {
+        bool ret = generate_all_tokens();
+    }
 
-    uint32_t num_expected_msgs = g_teems_config.m_priv_in * storage_state.max_users;
     processMsgs();
 
     storage_epoch++;

gen_manifest.py

@@ -32,6 +32,8 @@ def generate_manifest(N, M, T, B):
   pub_out: 2
   # The number of public messages each user can receive per epoch
   pub_in: 2
+  # Private or public routing protocol selection
+  private_routing: False
   # A hardcoded master secret for generating keys to bootstrap
   # client -> server communications
   master_secret: \"AHardCodedAESKey\"\n'''.format(N = str(N), B = str(B))