Fixing and seperating sizes on msgbundles (the messages clients send each epoch, which has tokens alongside the messages) and mailbox (the messages clients receive each epoch, which do NOT have tokens)

App/net.cpp

@@ -198,7 +198,7 @@ void NetIO::ing_receive_msgbundle(tcp::socket* csocket, clientid_t c_simid)
         }
 
         //Ingest the message_bundle
-        bool ret = ecall_ingest_msgbundle(c_simid, msgbundle, apiparams.m_priv_out);
+        bool ret = ecall_ingest_msgbundle(c_simid, msgbundle, conf.m_priv_out);
         free(msgbundle);
 
         // Continue to async receive client message bundles
@@ -344,22 +344,11 @@ void NetIO::send_client_mailbox()
         unsigned char *tkn_ptr = epoch_tokens + lcid * token_bundle_size;
         unsigned char *buf_ptr = epoch_msgbundles + lcid * msgbundle_size;
 
-        /*
-        if(lcid == 0) {
-            printf("In NetiO::send_client_mailbox. Token_bundle for lcid 0:\n");
-            for(int k=0; k<token_bundle_size; k++) {
-                printf("%x", tkn_ptr[k]);
-            }
-            printf("\n");
-        }
-        */
-
         if(client_sockets[lcid]!=nullptr) {
             boost::asio::async_write(*(client_sockets[lcid]),
                 boost::asio::buffer(tkn_ptr, token_bundle_size),
                 [this, lcid, buf_ptr](boost::system::error_code ec, std::size_t){
 
-                //printf("NetIO::send_client_mailbox, Client %d tokens was sent\n", lcid);
                 if (ec) {
                     if(ec == boost::asio::error::eof) {
                         // Client connection terminated so we delete this socket
@@ -372,7 +361,7 @@ void NetIO::send_client_mailbox()
                 }
 
                 boost::asio::async_write(*(client_sockets[lcid]),
-                    boost::asio::buffer(buf_ptr, msgbundle_size),
+                    boost::asio::buffer(buf_ptr, mailbox_size),
                     [this, lcid](boost::system::error_code ecc, std::size_t){
 
                     //printf("NetIO::send_client_mailbox, Client %d messages was sent\n", lcid);
@@ -395,8 +384,6 @@ void NetIO::send_client_mailbox()
         }
         */
     }
-
-
 }
 
 NetIO::NetIO(boost::asio::io_context &io_context, const Config &config)
@@ -472,7 +459,13 @@ NetIO::NetIO(boost::asio::io_context &io_context, const Config &config)
 
     auth_size = sizeof(clientid_t) + sizeof(unsigned long) + SGX_AESGCM_KEY_SIZE;
     msgbundle_size = SGX_AESGCM_IV_SIZE
-        + (apiparams.m_priv_out * (apiparams.msg_size + TOKEN_SIZE))
+        + (conf.m_priv_out * (conf.msg_size + TOKEN_SIZE))
+        + SGX_AESGCM_MAC_SIZE;
+    uint16_t priv_out = config.m_priv_out;
+    token_bundle_size = ((priv_out * SGX_AESGCM_KEY_SIZE)
+        + SGX_AESGCM_IV_SIZE + SGX_AESGCM_MAC_SIZE);
+    uint16_t priv_in = conf.m_priv_in;
+    mailbox_size = (priv_in * conf.msg_size) + SGX_AESGCM_IV_SIZE
         + SGX_AESGCM_MAC_SIZE;
 
 
@@ -494,10 +487,7 @@ NetIO::NetIO(boost::asio::io_context &io_context, const Config &config)
             client_sockets.emplace_back(nullptr);
         }
 
-        uint16_t num_priv_channels = config.m_priv_out;
         uint32_t epoch_msgbundles_size = num_clients_per_stg * msgbundle_size;
-        token_bundle_size = ((num_priv_channels * SGX_AESGCM_KEY_SIZE)
-            + SGX_AESGCM_IV_SIZE + SGX_AESGCM_MAC_SIZE);
         uint32_t epoch_tokens_size = num_clients_per_stg * token_bundle_size;
         epoch_msgbundles = (unsigned char *) malloc(epoch_msgbundles_size);
         epoch_tokens = (unsigned char *) malloc (epoch_tokens_size);

App/net.hpp

@@ -146,6 +146,7 @@ class NetIO {
     unsigned char *epoch_msgbundles;
     uint16_t num_stg_nodes;
     uint32_t token_bundle_size;
+    uint32_t mailbox_size;
     void stg_authenticate_new_client(tcp::socket* socket,
         const boost::system::error_code& error);
     void stg_start_accept();

Client/clients.cpp

@@ -139,6 +139,16 @@ static inline uint32_t ptMsgBundleSize(uint16_t priv_out, uint16_t msg_size)
     return(priv_out * (msg_size + TOKEN_SIZE));
 }
 
+static inline uint32_t encMailboxSize(uint16_t priv_in, uint16_t msg_size)
+{
+    return(SGX_AESGCM_IV_SIZE + (priv_in * msg_size) + SGX_AESGCM_MAC_SIZE);
+}
+
+static inline uint32_t ptMailboxSize(uint16_t priv_in, uint16_t msg_size)
+{
+    return(priv_in * msg_size);
+}
+
 bool config_parse(Config &config, const std::string configstr,
     std::vector<NodeConfig> &ingestion_nodes,
     std::vector<NodeConfig> &storage_nodes,
@@ -476,7 +486,13 @@ void Client::sendMessageBundle()
 
     boost::asio::async_write(*ingestion_sock,
         boost::asio::buffer(send_enc_msgbundle, send_enc_msgbundle_size),
-        [this] (boost::system::error_code ecc, std::size_t) {
+        [this, send_enc_msgbundle] (boost::system::error_code ecc, std::size_t) {
+
+#ifdef VERBOSE_CLIENT
+        if(sim_id==0){
+            printf("TEST: Client 0 send their msgbundle\n");
+        }
+#endif
 
         if (ecc) {
             if(ecc == boost::asio::error::eof) {
@@ -489,7 +505,11 @@ void Client::sendMessageBundle()
             return;
         }
 
+        free(send_enc_msgbundle);
+
     });
+
+    free(send_pt_msgbundle);
 }
 
 int Client::sendIngAuthMessage(unsigned long epoch_no)
@@ -663,7 +683,7 @@ void Client::epoch_process() {
             return;
         }
 
-        /*
+#ifdef VERBOSE_CLIENT
         if(sim_id == 0) {
             printf("TEST: Client 0: Encrypted token bundle received:\n");
             for(uint32_t i = 0; i < token_bundle_size; i++) {
@@ -671,22 +691,26 @@ void Client::epoch_process() {
             }
             printf("\n");
         }
-        */
+#endif
+
 
         // Decrypt the token bundle
         unsigned char *enc_tkn_ptr = enc_tokens + SGX_AESGCM_IV_SIZE;
         unsigned char *enc_tkn_tag = enc_tokens + SGX_AESGCM_IV_SIZE + pt_token_size;
 
+
         int decrypted_bytes =  gcm_decrypt(enc_tkn_ptr, pt_token_size,
                 NULL, 0, enc_tkn_tag, (unsigned char*) &(this->stg_key),
                 enc_tokens, SGX_AESGCM_IV_SIZE, (unsigned char*) (this->token_list));
+
         if(decrypted_bytes != pt_token_size) {
-            printf("Client::epoch_process gcm_decrypt tokens failed \n");
+            printf("Client::epoch_process gcm_decrypt tokens failed. decrypted_bytes = %d \n", decrypted_bytes);
         }
 
-        unsigned char *tkn_ptr = (unsigned char*) this->token_list;
         free(enc_tokens);
+
         /*
+        unsigned char *tkn_ptr = (unsigned char*) this->token_list;
         if(sim_id==0) {
             printf("TEST: Client 0: Decrypted client tokens:\n");
             for(int i = 0; i < 2 * SGX_AESGCM_KEY_SIZE; i++) {
@@ -699,14 +723,14 @@ void Client::epoch_process() {
         // Async read the messages recieved in the last epoch
         uint16_t priv_in = config.m_priv_in;
         uint16_t msg_size = config.msg_size;
-        uint32_t recv_pt_msgbundle_size = ptMsgBundleSize(priv_in, msg_size);
-        uint32_t recv_enc_msgbundle_size = encMsgBundleSize(priv_in, msg_size);
-        unsigned char *recv_pt_msgbundle = (unsigned char*) malloc (recv_pt_msgbundle_size);
-        unsigned char *recv_enc_msgbundle = (unsigned char*) malloc (recv_enc_msgbundle_size);
+        uint32_t recv_pt_mailbox_size = ptMailboxSize(priv_in, msg_size);
+        uint32_t recv_enc_mailbox_size = encMailboxSize(priv_in, msg_size);
+        unsigned char *recv_pt_mailbox = (unsigned char*) malloc (recv_pt_mailbox_size);
+        unsigned char *recv_enc_mailbox = (unsigned char*) malloc (recv_enc_mailbox_size);
 
         boost::asio::async_read(*storage_sock,
-            boost::asio::buffer(recv_enc_msgbundle, recv_enc_msgbundle_size),
-            [this, recv_pt_msgbundle, recv_enc_msgbundle]
+            boost::asio::buffer(recv_enc_mailbox, recv_enc_mailbox_size),
+            [this, recv_pt_mailbox, recv_enc_mailbox]
             (boost::system::error_code ecc, std::size_t) {
 
             if (ecc) {
@@ -720,15 +744,20 @@ void Client::epoch_process() {
                 return;
             }
 
+#ifdef VERBOSE_CLIENT
+            if(sim_id == 0) {
+                printf("TEST: Client 0: Encrypted msgbundle received\n");
+            }
+#endif
+
             // Do whatever processing with the received messages here
-            free(recv_enc_msgbundle);
-            free(recv_pt_msgbundle);
+            free(recv_enc_mailbox);
+            free(recv_pt_mailbox);
 
             // Send this epoch's message bundle
             sendMessageBundle();
+            epoch_process();
         });
-
-        epoch_process();
     });
 
 }

Enclave/storage.cpp

@@ -99,14 +99,16 @@ bool generate_all_tokens()
         memset(token_body, 0, pt_tokens_size);
         memset(tkn_iv_ptr, 0, SGX_AESGCM_IV_SIZE);
         // IV = epoch_no, for encrypting the token bundle
-        memcpy(tkn_iv_ptr, &storage_epoch, sizeof(storage_epoch));
+        // epoch_no used is for the next epoch
+        unsigned long epoch_val = storage_epoch + 1;
+        memcpy(tkn_iv_ptr, &epoch_val, sizeof(epoch_val));
 
         unsigned char *ptr = tkn_ptr;
         for(int i = 0; i<g_teems_config.m_priv_out; i++)
         {
             memcpy(ptr, (&(clients[lcid].my_id)), sizeof(clientid_t));
             memcpy(ptr + sizeof(clientid_t), (&(clients[lcid].priv_friends[i])), sizeof(clientid_t));
-            memcpy(ptr + 2 * sizeof(clientid_t), &storage_epoch, sizeof(storage_epoch));
+            memcpy(ptr + 2 * sizeof(clientid_t), &epoch_val, sizeof(epoch_val));
             ptr+=SGX_AESGCM_KEY_SIZE;
         }
 
@@ -185,10 +187,12 @@ bool processMsgs() {
         uint64_t *iv_ctr = (uint64_t*) clients[lcid].iv;
         (*iv_ctr)+=1;
 
+        /*
         if(lcid==0) {
             printf("\n\nMessage for lcid 0, S, R = %d, %d\n\n\n", *((uint32_t*) stg_buf_ptr),
                 *((uint32_t*) (stg_buf_ptr + 4)));
         }
+        */
 
         stg_buf_ptr+=msg_bundle_size;
         epoch_buf_ptr+=enc_msg_bundle_size;
@@ -199,39 +203,6 @@ bool processMsgs() {
     return true;
 }
 
-bool generateDummies() {
-    unsigned char *epoch_buf_ptr = epoch_msgbundles;
-    uint32_t msg_bundle_size = g_teems_config.m_priv_in * g_teems_config.msg_size;
-    uint32_t enc_msg_bundle_size = msg_bundle_size + SGX_AESGCM_IV_SIZE + SGX_AESGCM_MAC_SIZE;
-    sgx_status_t ret = SGX_SUCCESS;
-    unsigned char *epoch_buf_ct_ptr = epoch_buf_ptr + SGX_AESGCM_IV_SIZE;
-    unsigned char *epoch_buf_tag_ptr = epoch_buf_ct_ptr + msg_bundle_size;
-    unsigned char *zeroes = new unsigned char[msg_bundle_size];
-    memset(zeroes, 0, msg_bundle_size);
-
-    for(uint32_t lcid = 0; lcid <storage_state.max_users; lcid++) {
-        memcpy(epoch_buf_ptr, clients[lcid].iv, SGX_AESGCM_IV_SIZE);
-        ret = sgx_rijndael128GCM_encrypt(&(clients[lcid].key), zeroes, msg_bundle_size,
-            (uint8_t*) epoch_buf_ct_ptr, epoch_buf_ptr, SGX_AESGCM_IV_SIZE, NULL, 0,
-            (sgx_aes_gcm_128bit_tag_t*) epoch_buf_tag_ptr);
-        if(ret!=SGX_SUCCESS) {
-            printf("processMsgs: Encrypting msgs FAIL\n");
-            return false;
-        }
-
-        // Update IV
-        uint64_t *iv_ctr = (uint64_t*) clients[lcid].iv;
-        (*iv_ctr)+=1;
-
-        epoch_buf_ptr+=enc_msg_bundle_size;
-        epoch_buf_ct_ptr+=enc_msg_bundle_size;
-        epoch_buf_tag_ptr+=enc_msg_bundle_size;
-    }
-
-    delete zeroes;
-    return true;
-}
-
 // route_init will call this function; no one else should call it
 // explicitly.  The parameter is the number of messages that can fit in
 // the storage-side MsgBuffer.  Returns true on success, false on
@@ -407,11 +378,7 @@ void storage_received(MsgBuffer &storage_buf)
     generate_all_tokens();
 
     uint32_t num_expected_msgs = g_teems_config.m_priv_in * storage_state.max_users;
-    if(num_msgs!= 0) {
-        processMsgs();
-    } else {
-        generateDummies();
-    }
+    processMsgs();
 
     storage_epoch++;