Clients and Storage servers generate a shared secret key

Client/clients.cpp

@@ -64,7 +64,8 @@ static bool hextobuf(unsigned char *buf, const char *str, size_t len)
     return true;
 }
 
-void displayMessage(unsigned char *msg, uint16_t msg_size) {
+void displayMessage(unsigned char *msg, uint16_t msg_size)
+{
     clientid_t sid, rid;
     unsigned char *ptr = msg;
     sid = *((clientid_t*) ptr);
@@ -80,7 +81,9 @@ void displayMessage(unsigned char *msg, uint16_t msg_size) {
     printf("\n");
 }
 
-void displayPtMessageBundle(unsigned char *bundle, uint16_t priv_out, uint16_t msg_size) {
+void displayPtMessageBundle(unsigned char *bundle, uint16_t priv_out,
+    uint16_t msg_size)
+{
     unsigned char *ptr = bundle;
 
     for(int i=0; i<priv_out; i++) {
@@ -91,7 +94,9 @@ void displayPtMessageBundle(unsigned char *bundle, uint16_t priv_out, uint16_t m
     printf("\n");
 }
 
-void displayEncMessageBundle(unsigned char *bundle, uint16_t priv_out, uint16_t msg_size) {
+void displayEncMessageBundle(unsigned char *bundle, uint16_t priv_out,
+    uint16_t msg_size)
+{
     unsigned char *ptr = bundle;
     uint64_t header = *((uint64_t*) ptr);
     ptr+=sizeof(uint64_t);
@@ -116,10 +121,13 @@ void displayEncMessageBundle(unsigned char *bundle, uint16_t priv_out, uint16_t
 }
 
 
-static inline uint32_t encMsgBundleSize(uint16_t priv_out, uint16_t msg_size) {
+static inline uint32_t encMsgBundleSize(uint16_t priv_out, uint16_t msg_size)
+{
     return(SGX_AESGCM_IV_SIZE + (priv_out * msg_size) + SGX_AESGCM_MAC_SIZE);
 }
-static inline uint32_t ptMsgBundleSize(uint16_t priv_out, uint16_t msg_size) {
+
+static inline uint32_t ptMsgBundleSize(uint16_t priv_out, uint16_t msg_size)
+{
     return((priv_out * msg_size));
 }
 
@@ -152,7 +160,8 @@ bool config_parse(Config &config, const std::string configstr,
                     config.m_pub_out = pentry.second.get_value<uint8_t>();
                 } else if (!pentry.first.compare("pub_in")) {
                     config.m_pub_in = pentry.second.get_value<uint8_t>();
-                // Currently hardcoding an AES key for client -> server communication
+                // A hardcoded shared secret to derive various
+                // keys for client -> server communications and tokens
                 } else if (!pentry.first.compare("master_secret")) {
                     std::string hex_key = pentry.second.data();
                     memcpy(config.master_secret, hex_key.c_str(), SGX_AESGCM_KEY_SIZE);
@@ -269,10 +278,12 @@ int generateMasterKeys(sgx_aes_gcm_128bit_key_t master_secret,
 }
 
 /*
-    Takes the client_number, the master aes_key for generating client encryption keys,
-    and the client aes_key to be generated.
+    Takes the client_number, the master aes_key for generating client keys
+    for encrypted communication with ingestion (client_ing_key) and
+    storage servers (client_stg_key)
 */
-int generateClientEncryptionKey(clientid_t client_number, aes_key &EMK, aes_key &client_key)
+int generateClientKeys(clientid_t client_number, aes_key &EMK,
+    aes_key &client_ing_key, aes_key &client_stg_key)
 {
     unsigned char zeroes[SGX_AESGCM_KEY_SIZE];
     unsigned char iv[SGX_AESGCM_IV_SIZE];
@@ -285,31 +296,56 @@ int generateClientEncryptionKey(clientid_t client_number, aes_key &EMK, aes_key
     /*
     printf("Client Key: (before Gen) ");
     for(int i=0;i<SGX_AESGCM_KEY_SIZE;i++) {
-        printf("%x", client_key[i]);
+        printf("%x", client_ing_key[i]);
     }
     printf("\n");
     */
 
     if (sizeof(zeroes) != gcm_encrypt(zeroes, SGX_AESGCM_KEY_SIZE, NULL, 0, EMK,
-            iv, SGX_AESGCM_IV_SIZE, client_key, tag)) {
-        printf("generateClientEncryptionKey failed\n");
+            iv, SGX_AESGCM_IV_SIZE, client_ing_key, tag)) {
+        printf("generateClientKeys failed\n");
+        return -1;
+    }
+
+    memset(iv, 0, SGX_AESGCM_IV_SIZE);
+    memcpy(iv, &client_number, sizeof(client_number));
+    memcpy(iv +sizeof(client_number), "STG", sizeof("STG"));
+    if (sizeof(zeroes) != gcm_encrypt(zeroes, SGX_AESGCM_KEY_SIZE, NULL, 0, EMK,
+            iv, SGX_AESGCM_IV_SIZE, client_stg_key, tag)) {
+        printf("generateClientKeys failed\n");
         return -1;
     }
 
     /*
-    printf("Client Key: (after Gen) ");
-    for(int i=0;i<SGX_AESGCM_KEY_SIZE;i++) {
-        printf("%x", client_key[i]);
+    if(client_number % 10 == 0) {
+        printf("Client %d Storage Key: (after Gen) ", client_number);
+        for(int i=0;i<SGX_AESGCM_KEY_SIZE;i++) {
+            printf("%x", client_stg_key[i]);
+        }
+        printf("\n");
     }
-    printf("\n");
     */
 
     return 1;
 }
 
 
+void Client::initClient(clientid_t cid, aes_key ikey, aes_key skey,
+    uint16_t num_storage_nodes, std::vector<uint16_t> &storage_map)
+{
+    sim_id = cid;
+    uint16_t stg_no = cid % num_storage_nodes;
+    uint16_t stg_id = storage_map[stg_no];
+    id = stg_id << DEST_UID_BITS;
+    id += (cid/num_storage_nodes);
+
+    memcpy(ing_key, ikey, SGX_AESGCM_KEY_SIZE);
+    memcpy(stg_key, skey, SGX_AESGCM_KEY_SIZE);
+}
+
 void Client::initializeSocket(boost::asio::io_context &ioc,
-    NodeConfig &ing_server) {
+    NodeConfig &ing_server)
+{
 
     boost::system::error_code err;
     boost::asio::ip::tcp::resolver resolver(ioc);
@@ -368,16 +404,16 @@ bool Client::encryptMessageBundle(uint32_t enc_bundle_size, unsigned char *pt_ms
     unsigned char *enc_tag = enc_msgbundle + enc_bundle_size - SGX_AESGCM_MAC_SIZE;
     size_t bytes_to_encrypt = enc_bundle_size - SGX_AESGCM_MAC_SIZE - SGX_AESGCM_IV_SIZE;
     if (bytes_to_encrypt != gcm_encrypt(pt_msgbundle_start, bytes_to_encrypt,
-        NULL, 0, key, iv, SGX_AESGCM_IV_SIZE, enc_msgbundle_start, enc_tag)) {
+        NULL, 0, ing_key, ing_iv, SGX_AESGCM_IV_SIZE, enc_msgbundle_start, enc_tag)) {
             printf("Client: encryptMessageBundle FAIL\n");
             return 0;
     }
 
     // Copy IV into the bundle
-    memcpy(enc_msgbundle, iv, SGX_AESGCM_IV_SIZE);
+    memcpy(enc_msgbundle, ing_iv, SGX_AESGCM_IV_SIZE);
 
     // Update IV
-    uint64_t *iv_ctr = (uint64_t*) iv;
+    uint64_t *iv_ctr = (uint64_t*) ing_iv;
     (*iv_ctr)+=1;
     return 1;
 }
@@ -421,16 +457,12 @@ int Client::sendAuthMessage(unsigned long epoch_no)
     unsigned char epoch_iv[SGX_AESGCM_IV_SIZE] = {0};
     memcpy(epoch_iv, &epoch_no, sizeof(epoch_no));
 
-    if (sizeof(zeroes) != gcm_encrypt(zeroes, SGX_AESGCM_KEY_SIZE, NULL, 0, key,
+    if (sizeof(zeroes) != gcm_encrypt(zeroes, SGX_AESGCM_KEY_SIZE, NULL, 0, ing_key,
             epoch_iv, SGX_AESGCM_IV_SIZE, am_ptr, tag)) {
-        printf("generateClientEncryptionKey failed\n");
+        printf("generateClientKeys failed\n");
         return -1;
     }
 
-    // Update IV
-    uint64_t *iv_ctr = (uint64_t*) iv;
-    (*iv_ctr)+=1;
-
 #ifdef VERBOSE_CLIENT
     printf("Client %d auth_message: \n", id);
     for(int i=0; i<auth_size; i++) {
@@ -452,7 +484,8 @@ void generateClients(boost::asio::io_context &io_context,
     uint32_t num_clients_total, uint32_t clients_per_ing,
     uint32_t ing_with_additional)
 {
-    aes_key client_key;
+    aes_key client_ing_key;
+    aes_key client_stg_key;
     uint16_t num_stg_nodes = storage_nodes.size();
     uint16_t *stg_map = new uint16_t[num_stg_nodes];
 
@@ -463,8 +496,9 @@ void generateClients(boost::asio::io_context &io_context,
             ing_node_this_client = ing_with_additional + (leftover / (clients_per_ing-1));
         }
 
-        int ret = generateClientEncryptionKey(i, EMK, client_key);
-        clients[i].initClient(i, client_key, num_stg_nodes, storage_map);
+        int ret = generateClientKeys(i, EMK, client_ing_key, client_stg_key);
+
+        clients[i].initClient(i, client_ing_key, client_stg_key, num_stg_nodes, storage_map);
 
         clients[i].initializeSocket(io_context, ingestion_nodes[ing_node_this_client]);
 
@@ -574,7 +608,6 @@ int main(int argc, char **argv)
     std::thread threads[nthreads];
 
     // Generate all the clients for the experiment
-
     for(int i=0; i<nthreads; i++) {
         uint32_t cstart, cstop;
         cstart = i * clients_per_thread;

Client/clients.hpp

@@ -38,8 +38,12 @@ private:
     // Format: the first DEST_STORAGE_NODE_BITS bits store the storage node
     // number and the userid at that storage node in the last DEST_UID_BITS
     clientid_t id;
-    aes_key key;
-    unsigned char iv[SGX_AESGCM_IV_SIZE] = {0};
+
+    aes_key ing_key;
+    aes_key stg_key;
+    // Clients send encrypted messages to ingestion
+    // so they set and increment the IV
+    unsigned char ing_iv[SGX_AESGCM_IV_SIZE] = {0};
 
     boost::asio::ip::tcp::socket *ingestion_sock = NULL;
 
@@ -53,34 +57,15 @@ private:
 
 public:
 
-    Client () {
-        memset(key, 0, SGX_AESGCM_KEY_SIZE);
-        memset(iv, 0, SGX_AESGCM_IV_SIZE);
-    }
+    Client () {}
 
-    void initClient(clientid_t cid, aes_key ckey, uint16_t num_storage_nodes,
-        std::vector<uint16_t> &storage_map) {
-        sim_id = cid;
-        uint16_t stg_no = cid % num_storage_nodes;
-        uint16_t stg_id = storage_map[stg_no];
-        id = stg_id << DEST_UID_BITS;
-        id += (cid/num_storage_nodes);
-        //printf("Client sim_id = %d, stg_id = %d, cid = %d\n", sim_id, stg_id, id);
-        memcpy(key, ckey, SGX_AESGCM_KEY_SIZE);
-    }
+    void initClient(clientid_t cid, aes_key ikey, aes_key skey,
+        uint16_t num_storage_nodes, std::vector<uint16_t> &storage_map);
 
     bool socketReady(){
         return(ingestion_sock!=NULL);
     }
 
-    clientid_t getid(){
-        return id;
-    }
-
-    unsigned char* getKey(){
-        return ((unsigned char*) key);
-    }
-
     void initializeSocket(boost::asio::io_context &ioc, NodeConfig &ing_server);
 
     int sendAuthMessage(unsigned long epoch_no);

Enclave/config.cpp

@@ -142,32 +142,37 @@ bool ecall_config_load(threadid_t nthreads, bool private_routing,
     // Initialize the threadpool and the pseudorandom bytes pools
     threadpool_init(nthreads);
 
-    if(apinodeconfigs[my_node_num].roles & ROLE_INGESTION) {
-        sgx_aes_gcm_128bit_key_t ESK, TSK;
-        generateMasterKeys(g_teems_config.master_secret, ESK, TSK);
+    uint8_t my_role = apinodeconfigs[my_node_num].roles;
+    if( (my_role & ROLE_INGESTION) || (my_role & ROLE_STORAGE) ) {
+        generateMasterKeys(g_teems_config.master_secret,
+            g_teems_config.ESK, g_teems_config.TSK);
 
         uint32_t num_clients_total = g_teems_config.user_count;
-        uint32_t num_ing_nodes = g_teems_config.num_ingestion_nodes;
-        uint32_t clients_per_server = CEILDIV(num_clients_total, num_ing_nodes);
-        uint32_t ing_with_additional = num_clients_total % num_ing_nodes;
-        uint32_t num_smaller_ing = (uint32_t) ing_smaller.size();
-        uint32_t num_clients_this_ing = clients_per_server;
-        num_clients_this_ing += (num_smaller_ing < ing_with_additional)? 1: 0;
-        uint32_t client_start = num_smaller_ing * clients_per_server;
-        if (ing_with_additional > 0) {
-            if(ing_with_additional > num_smaller_ing) {
-                client_start+= num_smaller_ing;
-            } else {
-                client_start+= ing_with_additional;
+
+        if(my_role & ROLE_INGESTION) {
+            uint32_t num_ing_nodes = g_teems_config.num_ingestion_nodes;
+            uint32_t clients_per_server = CEILDIV(num_clients_total, num_ing_nodes);
+            uint32_t ing_with_additional = num_clients_total % num_ing_nodes;
+            uint32_t num_smaller_ing = (uint32_t) ing_smaller.size();
+            uint32_t num_clients_this_ing = clients_per_server;
+            num_clients_this_ing += (num_smaller_ing < ing_with_additional)? 1: 0;
+            uint32_t client_start = num_smaller_ing * clients_per_server;
+            if (ing_with_additional > 0) {
+                if(ing_with_additional > num_smaller_ing) {
+                    client_start+= num_smaller_ing;
+                } else {
+                    client_start+= ing_with_additional;
+                }
             }
+            g_ing.initialize(num_clients_this_ing, client_start, g_teems_config.ESK);
         }
-        g_ing.initialize(num_clients_this_ing, client_start, ESK);
-
     }
 
+
     if (!route_init()) {
         return false;
     }
+
     return comms_init_nodestate(apinodeconfigs, num_nodes, my_node_num);
 }
 

Enclave/config.hpp

@@ -39,8 +39,11 @@ struct Config {
     // storage_map[i] is the node number of the storage node responsible
     // for the destination adddresses with storage node field i.
     std::vector<nodenum_t> storage_map;
+    sgx_aes_gcm_128bit_key_t ESK;
+    sgx_aes_gcm_128bit_key_t TSK;
 };
 
 extern Config g_teems_config;
 
+
 #endif

Enclave/storage.cpp

@@ -3,9 +3,12 @@
 #include "ORExpand.hpp"
 #include "sort.hpp"
 #include "storage.hpp"
+#include "client.hpp"
 
 #define PROFILE_STORAGE
 
+StgClient *clients;
+
 static struct {
     uint32_t max_users;
     uint32_t my_storage_node_id;
@@ -26,15 +29,20 @@ bool storage_init(uint32_t max_users, uint32_t msg_buf_size)
     storage_state.stg_buf.alloc(msg_buf_size);
     storage_state.dest.resize(msg_buf_size);
     uint32_t my_storage_node_id = 0;
+    uint32_t my_stg_pos = 0;
     for (nodenum_t i=0; i<g_teems_config.num_nodes; ++i) {
         if (g_teems_config.roles[i] & ROLE_STORAGE) {
             if (i == g_teems_config.my_node_num) {
                 storage_state.my_storage_node_id = my_storage_node_id << DEST_UID_BITS;
+                my_stg_pos = my_storage_node_id;
             } else {
                 ++my_storage_node_id;
             }
         }
     }
+
+    storage_generateClientKeys(max_users, my_stg_pos);
+
     return true;
 }
 
@@ -187,3 +195,50 @@ void storage_received(MsgBuffer &storage_buf)
     printf_with_rtclock_diff(start_received, "end storage_received (%u)\n", storage_buf.inserted);
 #endif
 }
+
+bool storage_generateClientKeys(uint32_t num_clients, uint32_t my_stg_no) {
+
+    clients = new StgClient[num_clients];
+    uint32_t num_stg_nodes = g_teems_config.num_storage_nodes;
+    uint32_t c_simid = my_stg_no;
+
+    //printf("In Ingestion::genCK, num_clients = %d\n", num_clients);
+    for (uint32_t i=0; i<num_clients; i++) {
+        const sgx_aes_gcm_128bit_key_t *pESK = &(g_teems_config.ESK);
+        unsigned char zeroes[SGX_AESGCM_KEY_SIZE];
+        unsigned char iv[SGX_AESGCM_IV_SIZE];
+        sgx_aes_gcm_128bit_tag_t tag;
+        memset(zeroes, 0, SGX_AESGCM_KEY_SIZE);
+        memset(iv, 0, SGX_AESGCM_IV_SIZE);
+
+        memcpy(iv, (uint8_t*) (&c_simid), sizeof(c_simid));
+        memcpy(iv + sizeof(c_simid), "STG", sizeof("STG"));
+
+        sgx_status_t ret = SGX_SUCCESS;
+        ret = sgx_rijndael128GCM_encrypt(pESK, zeroes, SGX_AESGCM_KEY_SIZE,
+            (uint8_t*) (clients[i].key), iv, SGX_AESGCM_IV_SIZE, NULL, 0, &tag);
+        if(ret!=SGX_SUCCESS) {
+            printf("stg_generateClientKeys FAIL\n");
+            return false;
+        }
+
+        /*
+        if(c_simid % 10 == 0) {
+            printf("Storage: c_simid = %d, Key:", c_simid);
+            for (int k = 0; k<SGX_AESGCM_KEY_SIZE; k++) {
+                printf("%x", (clients[i].key)[k]);
+            }
+            printf("\n");
+        }
+        */
+        c_simid+=num_stg_nodes;
+
+    }
+
+    return true;
+}
+
+bool authenticateClient()
+{
+
+}

Enclave/storage.hpp

@@ -16,4 +16,7 @@ bool storage_init(uint32_t max_users, uint32_t msg_buf_size);
 // done with it.
 void storage_received(MsgBuffer &storage_buf);
 
+
+bool storage_generateClientKeys(uint32_t num_clients, uint32_t my_stg_no);
+
 #endif