Ingestion processes msgbundle, and queues messages into a msgBuffer

App/net.cpp

@@ -10,6 +10,7 @@
 #define COMMAND_MESSAGE 0x01
 #define COMMAND_CHUNK 0x02
 
+#define VERBOSE_NET
 NetIO *g_netio = NULL;
 
 NodeIO::NodeIO(tcp::socket &&socket, nodenum_t nodenum) :
@@ -170,40 +171,6 @@ void NodeIO::recv_commands(
         });
 }
 
-#ifdef VERBOSE_NET
-void displayMessage(unsigned char *msg, uint16_t msg_size) {
-    clientid_t sid, rid;
-    unsigned char *ptr = msg;
-    sid = *((clientid_t*) ptr);
-    ptr+=sizeof(sid);
-    rid = *((clientid_t*) ptr);
-    uint16_t srpair_size = sizeof(sid)*2;
-    printf("Sender ID: %d, Receiver ID: %d, Token: N/A\n", sid, rid );
-    printf("Message: ");
-    for(int j = 0; j<msg_size - srpair_size; j++) {
-        printf("%x", (*ptr));
-        ptr++;
-    }
-    printf("\n");
-}
-
-void displayMessageBundle(unsigned char *bundle, uint16_t priv_out, uint16_t msg_size) {
-    unsigned char *ptr = bundle;
-    /*
-    // Header is already parsed on this end
-    uint64_t header = *((uint64_t*) ptr);
-    ptr+=sizeof(uint64_t);
-    */
-
-    for(int i=0; i<priv_out; i++) {
-        displayMessage(ptr, msg_size);
-        printf("\n");
-        ptr+=msg_size;
-    }
-
-}
-#endif
-
 /*
   Handler for received client messages.
 
@@ -235,25 +202,11 @@ void NetIO::handle_async_clients(std::shared_ptr<tcp::socket> csocket,
             boost::asio::read(*csocket,
                boost::asio::buffer(msgbundle, msgbundle_size));
 
-#ifdef VERBOSE_NET
-            displayMessageBundle(msgbundle, apiparams.m_priv_out, apiparams.msg_size);
-#endif
-
             //Ingest the message_bundle
             bool ret = ecall_ingest_msgbundle(cid, msgbundle, apiparams.m_priv_out);
             free(msgbundle);
         }
 
-
-        /*
-            This should read a MESSAGES_DROP_OFF packet of fixed length
-            from a client.
-            Send this packet over to ingestion processing:
-                - Decrypt the packet with the correct key for that client id
-                - Verify private channel token
-                - Buffer the message for route
-        */
-
         start_accept(auth_size, msgbundle_size);
     } else {
         printf("Accept handler failed\n");
@@ -353,7 +306,8 @@ NetIO::NetIO(boost::asio::io_context &io_context, const Config &config)
 
         size_t auth_size, msgbundle_size;
         auth_size = SGX_AESGCM_MAC_SIZE;
-        msgbundle_size = (apiparams.m_priv_out * apiparams.msg_size) + SGX_AESGCM_MAC_SIZE;
+        msgbundle_size = SGX_AESGCM_IV_SIZE +
+            (apiparams.m_priv_out * apiparams.msg_size) + SGX_AESGCM_MAC_SIZE;
         start_accept(auth_size, msgbundle_size);
     }
 

Client/clients.cpp

@@ -78,7 +78,7 @@ void displayMessage(unsigned char *msg, uint16_t msg_size) {
     printf("\n");
 }
 
-void displayMessageBundle(unsigned char *bundle, uint16_t priv_out, uint16_t msg_size) {
+void displayPtMessageBundle(unsigned char *bundle, uint16_t priv_out, uint16_t msg_size) {
     unsigned char *ptr = bundle;
     uint64_t header = *((uint64_t*) ptr);
     ptr+=sizeof(uint64_t);
@@ -88,12 +88,40 @@ void displayMessageBundle(unsigned char *bundle, uint16_t priv_out, uint16_t msg
         printf("\n");
         ptr+=msg_size;
     }
+    printf("\n");
+}
+
+void displayEncMessageBundle(unsigned char *bundle, uint16_t priv_out, uint16_t msg_size) {
+    unsigned char *ptr = bundle;
+    uint64_t header = *((uint64_t*) ptr);
+    ptr+=sizeof(uint64_t);
+
+    printf("IV: ");
+    for(int i=0; i<SGX_AESGCM_IV_SIZE; i++) {
+        printf("%x", ptr[i]);
+    }
+    printf("\n");
+    ptr+= SGX_AESGCM_IV_SIZE;
+
+    for(int i=0; i<priv_out; i++) {
+        displayMessage(ptr, msg_size);
+        ptr+=msg_size;
+    }
 
+    printf("MAC: ");
+    for(int i=0; i<SGX_AESGCM_MAC_SIZE; i++) {
+        printf("%x", ptr[i]);
+    }
+    printf("\n");
 }
 
+
 #define HEADER_SIZE 8
-static inline uint32_t messageBundleSize(uint16_t priv_out, uint16_t msg_size) {
-    return(HEADER_SIZE + (priv_out * msg_size) + SGX_AESGCM_MAC_SIZE);
+static inline uint32_t encMsgBundleSize(uint16_t priv_out, uint16_t msg_size) {
+    return(HEADER_SIZE + SGX_AESGCM_IV_SIZE + (priv_out * msg_size) + SGX_AESGCM_MAC_SIZE);
+}
+static inline uint32_t ptMsgBundleSize(uint16_t priv_out, uint16_t msg_size) {
+    return(HEADER_SIZE + (priv_out * msg_size));
 }
 
 bool config_parse(Config &config, const std::string configstr,
@@ -249,12 +277,13 @@ int generateClientEncryptionKey(clientid_t client_number, aes_key &EMK, aes_key
     memset(tag, 0, SGX_AESGCM_KEY_SIZE);
     memcpy(iv, &client_number, sizeof(client_number));
 
+    /*
     printf("Client Key: (before Gen) ");
     for(int i=0;i<SGX_AESGCM_KEY_SIZE;i++) {
         printf("%x", client_key[i]);
     }
     printf("\n");
-
+    */
 
     if (sizeof(zeroes) != gcm_encrypt(zeroes, SGX_AESGCM_KEY_SIZE, NULL, 0, EMK,
             iv, SGX_AESGCM_IV_SIZE, client_key, tag)) {
@@ -262,12 +291,13 @@ int generateClientEncryptionKey(clientid_t client_number, aes_key &EMK, aes_key
         return -1;
     }
 
-
+    /*
     printf("Client Key: (after Gen) ");
     for(int i=0;i<SGX_AESGCM_KEY_SIZE;i++) {
         printf("%x", client_key[i]);
     }
     printf("\n");
+    */
 
     return 1;
 }
@@ -305,23 +335,23 @@ void Client::initializeSocket(boost::asio::io_context &ioc,
 
 /*
 
-    Populates the buffer payload with a valid message payload.
-    Assumes that it is supplied with a payload buffer of the correct length
+    Populates the buffer pt_msgbundle with a valid message pt_msgbundle.
+    Assumes that it is supplied with a pt_msgbundle buffer of the correct length
 
-    Correct length for payload  =  8 + (priv_out)*(msg_size) + 16 bytes
+    Correct length for pt_msgbundle  =  8 + (priv_out)*(msg_size) + 16 bytes
 
 */
 void Client::generateMessageBundle(uint8_t priv_out, uint32_t msg_size,
-    unsigned char *payload)
+    unsigned char *pt_msgbundle)
 {
-    unsigned char *ptr = payload;
+    unsigned char *ptr = pt_msgbundle;
     uint64_t header = (id << 8) + CLIENT_MESSAGE_BUNDLE;
 
     // Setup header
     memcpy(ptr, (uint8_t*) &header, sizeof(header));
     ptr+=sizeof(header);
 
-    // Setup message payload
+    // Setup message pt_msgbundle
     for(uint32_t i = 0; i < priv_out; i++) {
         memcpy(ptr, &id, sizeof(id));
         ptr+=(sizeof(id));
@@ -332,36 +362,56 @@ void Client::generateMessageBundle(uint8_t priv_out, uint32_t msg_size,
         memset(ptr, 0, remaining_message_size);
         ptr+=(remaining_message_size);
     }
-
-    memset(ptr, 0, SGX_AESGCM_MAC_SIZE);
 }
 
 
-void Client::encryptMessageBundle(uint32_t bundle_size, unsigned char *payload)
+bool Client::encryptMessageBundle(uint32_t enc_bundle_size, unsigned char *pt_msgbundle,
+    unsigned char *enc_msgbundle)
 {
+    // Copy the header
+    memcpy(enc_msgbundle, pt_msgbundle, HEADER_SIZE);
+
+    // Encrypt the rest of the pt_msgbundle
+    unsigned char *pt_msgbundle_start = pt_msgbundle + HEADER_SIZE;
+    unsigned char *enc_msgbundle_start = enc_msgbundle + HEADER_SIZE + SGX_AESGCM_IV_SIZE;
+    unsigned char *enc_tag = enc_msgbundle + enc_bundle_size - SGX_AESGCM_MAC_SIZE;
+    size_t bytes_to_encrypt = enc_bundle_size - SGX_AESGCM_MAC_SIZE - HEADER_SIZE - SGX_AESGCM_IV_SIZE;
+    if (bytes_to_encrypt != gcm_encrypt(pt_msgbundle_start, bytes_to_encrypt,
+        NULL, 0, key, iv, SGX_AESGCM_IV_SIZE, enc_msgbundle_start, enc_tag)) {
+            printf("Client: encryptMessageBundle FAIL\n");
+            return 0;
+    }
 
+    // Copy the IV into the bundle
+    unsigned char *enc_msgbundle_iv = enc_msgbundle + HEADER_SIZE;
+    memcpy(enc_msgbundle_iv, iv, SGX_AESGCM_IV_SIZE);
+
+    // Update IV
+    uint64_t *iv_ctr = (uint64_t*) iv;
+    (*iv_ctr)+=1;
+    return 1;
 }
 
 /*
 
-      Assumes payload is a buffer of size messageBundleSize(priv_out, msg_size)
+      Assumes pt_msgbundle is a buffer of size messageBundleSize(priv_out, msg_size)
 */
 
 void Client::sendMessageBundle(uint16_t priv_out, uint16_t msg_size,
-    unsigned char *payload)
+    unsigned char *pt_msgbundle, unsigned char *enc_msgbundle)
 {
-    uint32_t bundle_size = messageBundleSize(priv_out, msg_size);
+    uint32_t enc_bundle_size = encMsgBundleSize(priv_out, msg_size);
 
-    generateMessageBundle(priv_out, msg_size, payload);
+    generateMessageBundle(priv_out, msg_size, pt_msgbundle);
 
-    //encryptMessageBundle(bundle_size, payload);
+    encryptMessageBundle(enc_bundle_size, pt_msgbundle, enc_msgbundle);
 
-    displayMessageBundle(payload, priv_out, msg_size);
-    //Send over the ingestion_sock
+    //displayPtMessageBundle(pt_msgbundle, priv_out, msg_size);
 
-    boost::asio::write(*ingestion_sock,
-        boost::asio::buffer(payload, bundle_size));
+    //displayEncMessageBundle(enc_msgbundle, priv_out, msg_size);
 
+    boost::asio::write(*ingestion_sock,
+        boost::asio::buffer(enc_msgbundle, enc_bundle_size));
 }
 
 
@@ -425,12 +475,13 @@ int main(int argc, char **argv)
     uint16_t priv_out = config.m_priv_out;
     uint16_t msg_size = config.msg_size;
 
-    uint32_t bundle_size = messageBundleSize(priv_out, msg_size);
-    unsigned char *payload = (unsigned char*) malloc (bundle_size);
-
+    uint32_t pt_bundle_size = ptMsgBundleSize(priv_out, msg_size);
+    uint32_t enc_bundle_size = encMsgBundleSize(priv_out, msg_size);
+    unsigned char *pt_msgbundle = (unsigned char*) malloc (pt_bundle_size);
+    unsigned char *enc_msgbundle = (unsigned char*) malloc (enc_bundle_size);
 
     uint64_t epoch = 1;
-    while(epoch<3) {
+    while(epoch<2) {
 
         for(uint32_t i=0; i<num_clients_total; i++) {
             if(epoch==1) {
@@ -446,7 +497,6 @@ int main(int argc, char **argv)
                 clients[i].initializeSocket(io_context, ingestion_nodes[ing_node_this_client]);
                 //clients[i].sendAuthMessage();
 
-
                 /*
                 // Test that the keys generated match those generated within
                 // enclave config
@@ -460,12 +510,12 @@ int main(int argc, char **argv)
                 */
             }
 
-            clients[i].sendMessageBundle(priv_out, msg_size, payload);
+            clients[i].sendMessageBundle(priv_out, msg_size, pt_msgbundle, enc_msgbundle);
         }
         epoch++;
         sleep(1);
     }
 
-    free(payload);
+    free(pt_msgbundle);
     delete [] clients;
 }

Enclave/config.cpp

@@ -162,10 +162,11 @@ bool ecall_config_load(threadid_t nthreads, bool private_routing,
         /*
         // Check that the keys generated in Enclave match the ones generated
         // in the client application
-        for(uint32_t i=0; i<g_ing.clients.num; i++) {
-            printf("Client Key %d: ", i + g_ing.clients.start);
+        for(uint32_t i=0; i<g_ing.getClientNum(); i++) {
+            printf("Client Key %d: ", i + g_ing.getClientStart());
             sgx_aes_gcm_128bit_key_t key;
-            memcpy(key, (g_ing.clients).keys[i], SGX_AESGCM_KEY_SIZE);
+            unsigned char* ckey = (g_ing.getClientKeys())[i];
+            memcpy(key, ckey, SGX_AESGCM_KEY_SIZE);
             for(int j = 0; j<SGX_AESGCM_KEY_SIZE; j++) {
                 printf("%x", key[j]);
             }

Enclave/ingest.cpp

@@ -7,6 +7,43 @@
 
 Ingestion g_ing;
 
+void displayMessage(unsigned char *msg, uint16_t msg_size) {
+    clientid_t sid, rid;
+    unsigned char *ptr = msg;
+    sid = *((clientid_t*) ptr);
+    ptr+=sizeof(sid);
+    rid = *((clientid_t*) ptr);
+    printf("Sender ID: %d, Receiver ID: %d, Token: N/A\n", sid, rid );
+    printf("Message: ");
+    for(int j = 0; j<msg_size - sizeof(sid)*2; j++) {
+        printf("%x", (*ptr));
+        ptr++;
+    }
+    printf("\n");
+}
+
+void displayEncMessageBundle(unsigned char *bundle, uint16_t priv_out, uint16_t msg_size) {
+    unsigned char *ptr = bundle;
+
+    printf("IV: ");
+    for(int i=0; i<SGX_AESGCM_IV_SIZE; i++) {
+        printf("%x", ptr[i]);
+    }
+    printf("\n");
+    ptr+= SGX_AESGCM_IV_SIZE;
+
+    for(int i=0; i<priv_out; i++) {
+        displayMessage(ptr, msg_size);
+        ptr+=msg_size;
+    }
+
+    printf("MAC: ");
+    for(int i=0; i<SGX_AESGCM_MAC_SIZE; i++) {
+        printf("%x", ptr[i]);
+    }
+    printf("\n");
+}
+
 bool ecall_ingest_msgbundle(clientid_t cid, unsigned char *msgbundle,
     uint32_t num_msgs) {
 
@@ -20,7 +57,7 @@ void Ingestion::initialize(uint32_t cnum, uint32_t cstart, sgx_aes_gcm_128bit_ke
     clients.num = cnum;
     clients.start = cstart;
     clients.end = cnum + cstart;
-    clients.keys = new sgx_aes_gcm_128bit_key_t[num];
+    clients.keys = new sgx_aes_gcm_128bit_key_t[cnum];
     generateClientKeys(ESK);
 
     // Initialize the MsgBuffer to correct size
@@ -30,12 +67,29 @@ void Ingestion::initialize(uint32_t cnum, uint32_t cstart, sgx_aes_gcm_128bit_ke
 
 bool Ingestion::processMsgBundle(clientid_t cid, unsigned char *msgbundle,
     uint32_t num_msgs) {
+
     // Fetch corresponding client key
-    sgx_aes_gcm_128bit_key_t &ckey = g_ing.clients.keys[cid];
-    // Decrypt and verify tag for the message bundle
+    clientid_t lcid = cid - g_ing.clients.start;
+    sgx_aes_gcm_128bit_key_t ckey;
+    memcpy(ckey, (g_ing.clients).keys[lcid], SGX_AESGCM_KEY_SIZE);
+    unsigned char *iv = msgbundle;
+    msgbundle += SGX_AESGCM_IV_SIZE;
 
-    // Append msgbundle to g_ing.buffer;
     uint16_t msg_size = g_teems_config.msg_size;
+    uint32_t msgbundle_size = num_msgs * msg_size;
+    unsigned char *dec_msgbundle = (unsigned char *) malloc (msgbundle_size);
+    sgx_aes_gcm_128bit_tag_t tag;
+    memcpy(tag, msgbundle + msgbundle_size, SGX_AESGCM_MAC_SIZE);
+    //sgx_aes_gcm_128bit_tag_t *tag = (sgx_aes_gcm_128bit_tag_t*) (msgbundle + msgbundle_size);
+
+    sgx_status_t ret = sgx_rijndael128GCM_decrypt(&ckey, msgbundle, msgbundle_size,
+        dec_msgbundle, iv, SGX_AESGCM_IV_SIZE, NULL, 0, &tag);
+    if(ret!=SGX_SUCCESS) {
+        printf("Ingestion::processMsgBundle FAIL\n");
+        printf("Error code: %d", (uint32_t) ret);
+    }
+
+    // Append msgbundle to g_ing.buffer;
     MsgBuffer &msg_queue = g_ing.buffer;
 
     pthread_mutex_lock(&msg_queue.mutex);
@@ -50,12 +104,13 @@ bool Ingestion::processMsgBundle(clientid_t cid, unsigned char *msgbundle,
     pthread_mutex_unlock(&msg_queue.mutex);
 
     memmove(msg_queue.buf + head * msg_size,
-        msgbundle, num_msgs * msg_size);
+        dec_msgbundle, num_msgs * msg_size);
 
     pthread_mutex_lock(&msg_queue.mutex);
     msg_queue.inserted += num_msgs;
     pthread_mutex_unlock(&msg_queue.mutex);
 
+    free(dec_msgbundle);
     return true;
 }
 
@@ -67,7 +122,7 @@ void Ingestion::generateClientKeys(sgx_aes_gcm_128bit_key_t &ESK)
     {
         unsigned char zeroes[SGX_AESGCM_KEY_SIZE];
         unsigned char iv[SGX_AESGCM_IV_SIZE];
-        sgx_aes_gcm_128bit_tag_t mac;
+        sgx_aes_gcm_128bit_tag_t tag;
         memset(zeroes, 0, SGX_AESGCM_KEY_SIZE);
         memset(iv, 0, SGX_AESGCM_IV_SIZE);
 
@@ -77,7 +132,7 @@ void Ingestion::generateClientKeys(sgx_aes_gcm_128bit_key_t &ESK)
         sgx_status_t ret = SGX_SUCCESS;
         ret = sgx_rijndael128GCM_encrypt((const sgx_aes_gcm_128bit_key_t *) (ESK),
             zeroes, SGX_AESGCM_KEY_SIZE, (uint8_t*) (clients.keys[i]), iv,
-            SGX_AESGCM_IV_SIZE, NULL, 0, &mac);
+            SGX_AESGCM_IV_SIZE, NULL, 0, &tag);
         if(ret!=SGX_SUCCESS) {
             printf("Ingestion::GCK FAIL\n");
         }

Enclave/ingest.hpp

@@ -8,14 +8,12 @@ struct ClientList {
     sgx_aes_gcm_128bit_key_t *keys;
 };
 
-
-
-class Ingestion : public ClientList {
+class Ingestion{
 private:
 
     ClientList clients;
     MsgBuffer buffer;
-    size_t max_buffer_size;
+    uint32_t max_buffer_size;
 
 public:
 
@@ -26,6 +24,18 @@ public:
 
     void initialize(uint32_t cnum, uint32_t cstart, sgx_aes_gcm_128bit_key_t &ESK);
 
+    uint32_t getClientNum() {
+        return (clients.num);
+    }
+
+    uint32_t getClientStart() {
+        return (clients.start);
+    }
+
+    sgx_aes_gcm_128bit_key_t* getClientKeys() {
+        return (clients.keys);
+    }
+
     bool processMsgBundle(clientid_t cid, unsigned char *msgbundle,
         uint32_t num_msgs);
 };