networking has been debugged. It now works, no issues.

prsona/cfg/commands.cfg

@@ -0,0 +1,3 @@
+V 127.0.0.1:8082
+E
+R 127.0.0.1:8083 127.0.0.1:8082

prsona/inc/networkServer.hpp

@@ -43,7 +43,7 @@ std::string make_epoch_update_string(
     const Twistpoint& nextGenerator,
     bool doUserTallies);
 
-void distribute_epoch_updates(
+struct mg_connection *distribute_epoch_updates(
     const std::string& recipient,
     int recipientPort,
     const std::string& data,

prsona/inc/networking.hpp

@@ -16,7 +16,7 @@
 #define EPOCH_READY_URI "/ready"
 #define TRIGGER_EPOCH_URI "/epoch"
 #define TRIGGER_VOTE_URI "/vote"
-#define TRIGGER_REP_URI "/rep?"
+#define TRIGGER_REP_URI "/rep"
 #define WHICH_EPOCH_URI "/which"
 #define UPDATE_LOCK_URI "/lock"
 #define UPDATE_UNLOCK_URI "/unlock"

prsona/inc/server.hpp

@@ -53,6 +53,7 @@ class PrsonaServer : public PrsonaBase {
         Proof get_user_tally_commitment(const Twistpoint& request) const;
         Proof get_server_tally_commitment(const Twistpoint& request) const;
         Proof get_pseudonyms_commitment() const;
+        void print_current_commitments() const;
 
         // CLIENT INTERACTIONS
         void add_new_client(

prsona/src/clientMain.cpp

@@ -331,33 +331,61 @@ string make_vote_string(
     return buffer.str();
 }
 
-void send_item(
-    const string& target, int targetPort, const string& data, const char* whichUri)
+char *send_item(
+    std::default_random_engine& rng,
+    const string& target,
+    int targetPort,
+    const string& data,
+    const char* whichUri,
+    bool responseExpected)
 {
     struct synchronization_tool sync;
-    bool flag = false;
-    while (!flag)
-    {
-        struct mg_connection *conn = mg_connect_websocket_client(
-            target.c_str(),
-            targetPort,
-            USE_SSL,
-            NULL,
-            0,
-            whichUri,
-            "null",
-            synchro_websocket_data_handler,
-            empty_websocket_close_handler,
-            (void *) &sync);
+    char *retval = NULL;
 
-        if (!conn)
+    unique_lock<mutex> lck(sync.mtx);
+    sync.val = 0;
+    sync.val2 = 0;
+
+    while (!sync.val)
+    {
+        struct mg_connection *conn = NULL;
+        while (!conn)
         {
-            cerr << "Couldn't obtain BGN details" << endl;
-            continue;
+            if (responseExpected)
+            {
+                conn = mg_connect_websocket_client(
+                    target.c_str(),
+                    targetPort,
+                    USE_SSL,
+                    NULL,
+                    0,
+                    whichUri,
+                    "null",
+                    file_websocket_data_handler,
+                    file_websocket_close_handler,
+                    (void *) &sync);
+            }
+            else
+            {
+                conn = mg_connect_websocket_client(
+                    target.c_str(),
+                    targetPort,
+                    USE_SSL,
+                    NULL,
+                    0,
+                    whichUri,
+                    "null",
+                    synchro_websocket_data_handler,
+                    synchro_websocket_close_handler,
+                    (void *) &sync);    
+            }
+
+            if (!conn)
+                cerr << "Couldn't connect to server for purposes of sending item." << endl;
         }
 
-        unique_lock<mutex> lck(sync.mtx);
-        sync.val = 0;
+        if (responseExpected)
+            retval = set_temp_filename(rng, conn);
 
         mg_websocket_client_write(
             conn,
@@ -371,13 +399,13 @@ void send_item(
             "",
             0);
 
-        while (!sync.val)
+        while (!sync.val2)
             sync.cv.wait(lck);
 
         mg_close_connection(conn);
-
-        flag = true;
     }
+
+    return retval;
 }
 
 void make_vote(
@@ -430,7 +458,7 @@ void make_vote(
             shortTermPublicKey,
             VOTES_BY_URI,
             VOTES_BY_COMMIT_URI);
-    
+
     vector<Proof> voteProof;
     encryptedVotes =
         prsonaClient->make_votes(
@@ -443,7 +471,7 @@ void make_vote(
     string data =
         make_vote_string(voteProof, encryptedVotes, shortTermPublicKey);
 
-    send_item(target, targetPort, data, GIVE_NEW_VOTE_URI);
+    send_item(generator, target, targetPort, data, GIVE_NEW_VOTE_URI, false);
 }
 
 string make_vote_string(
@@ -475,7 +503,7 @@ string make_rep_proof_string(
 
     sizeOfVector.set(pi.size());
     
-    buffer >> sizeOfVector;
+    buffer << sizeOfVector;
     for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << pi[i];
 
@@ -485,7 +513,7 @@ string make_rep_proof_string(
     return buffer.str();
 }
 
-void make_reputation_proof(
+bool make_reputation_proof(
     default_random_engine& generator,
     const string& target,
     int targetPort,
@@ -533,7 +561,16 @@ void make_reputation_proof(
     string data =
         make_rep_proof_string(repProof, shortTermPublicKey, threshold);
 
-    send_item(target, targetPort, data, REP_PROOF_URI);
+    char *responseFile = send_item(generator, target, targetPort, data, REP_PROOF_URI, true);
+
+    ifstream response(responseFile);
+
+    char passed = response.get();
+
+    remove(responseFile);
+    delete responseFile;
+
+    return passed == '\x01';
 }
 
 int main(int argc, char *argv[])
@@ -691,7 +728,7 @@ int main(int argc, char *argv[])
     AltRemoteControlHandler triggerVoteHandler(CLIENT_MAKE_VOTE, &exitSync, "Client will make new votes!");
     server.addHandler(TRIGGER_VOTE_URI, triggerVoteHandler);
 
-    AltRemoteControlHandler triggerRepHandler(CLIENT_MAKE_REP_PROOF, &exitSync, "Client will make new votes!");
+    AltRemoteControlHandler triggerRepHandler(CLIENT_MAKE_REP_PROOF, &exitSync, "Client will make reputation proof!");
     server.addHandler(TRIGGER_REP_URI, triggerRepHandler);
 
     cout << "[" << seedStr << "] Entering main ready loop." << endl;
@@ -709,6 +746,7 @@ int main(int argc, char *argv[])
         switch (exitSync.val2)
         {
             case CLIENT_MAKE_VOTE:
+                cout << "[" << seedStr << "] Making new vote row." << endl;
                 make_vote(
                     generator,
                     serverIPs[whichServer],
@@ -717,6 +755,7 @@ int main(int argc, char *argv[])
                     serverIPs,
                     serverPorts,
                     numClients);
+                cout << "[" << seedStr << "] New vote row complete." << endl;
                 break;
 
             case CLIENT_MAKE_REP_PROOF:
@@ -725,6 +764,8 @@ int main(int argc, char *argv[])
 
                 target = fullQuery.substr(0, colonLocation);
                 targetPort = stoi(fullQuery.substr(colonLocation + 1));
+
+                cout << "[" << seedStr << "] Making new reputation proof." << endl;
                 make_reputation_proof(
                     generator,
                     target,
@@ -733,6 +774,7 @@ int main(int argc, char *argv[])
                     serverIPs,
                     serverPorts,
                     numClients);
+                cout << "[" << seedStr << "] New reputation proof complete." << endl;
                 break;
 
             default:

prsona/src/networkClient.cpp

@@ -218,6 +218,12 @@ void get_additional_commitment(
     std::vector<char *> commitmentFilenames;
     std::vector<struct synchronization_tool *> commitmentSyncs;
 
+    std::stringstream buffer;
+    std::string data;
+
+    buffer << shortTermPublicKey;
+    data = buffer.str();
+
     for (size_t i = 0; i < serverIPs.size(); i++)
     {
         if (serverIPs[i] == skipIP && serverPorts[i] == skipPort)
@@ -252,6 +258,12 @@ void get_additional_commitment(
             currSync->val = 0;
             commitmentFilenames.push_back(set_temp_filename(*rng, conn));
 
+            mg_websocket_client_write(
+                conn,
+                MG_WEBSOCKET_OPCODE_BINARY,
+                data.c_str(),
+                data.length());
+
             mg_websocket_client_write(
                 conn,
                 MG_WEBSOCKET_OPCODE_DATACOMPLETE,

prsona/src/networkServer.cpp

@@ -127,6 +127,7 @@ std::string make_epoch_initiator_string(
     
     BinarySizeT sizeOfVector(generatorProof.size());
     buffer << sizeOfVector;
+
     for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << generatorProof[i];
     
@@ -439,17 +440,16 @@ bool read_epoch_update_string(
     return doUserTallies.val();
 }
 
-void distribute_epoch_updates(
+struct mg_connection *distribute_epoch_updates(
     const std::string& recipient,
     int recipientPort,
     const std::string& data,
     const struct synchronization_tool* sync)
 {
-    bool flag = false;
-    while (!flag)
+    struct mg_connection *conn = NULL;
+    while (!conn)
     {
-        struct mg_connection *conn =
-            mg_connect_websocket_client(
+        conn = mg_connect_websocket_client(
                 recipient.c_str(),
                 recipientPort,
                 USE_SSL,
@@ -458,31 +458,26 @@ void distribute_epoch_updates(
                 ACCEPT_EPOCH_UPDATES_URI,
                 "null",
                 synchro_websocket_data_handler,
-                empty_websocket_close_handler,
+                synchro_websocket_close_handler,
                 (void *) sync);
 
         if (!conn)
-        {
             std::cerr << "Trouble giving epoch updates to server at " << recipient << std::endl;
-            continue;
-        }
-        
-        mg_websocket_client_write(
-            conn,
-            MG_WEBSOCKET_OPCODE_BINARY,
-            data.c_str(),
-            data.length());
+    }
 
-        mg_websocket_client_write(
-            conn,
-            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
-            "",
-            0);
+    mg_websocket_client_write(
+        conn,
+        MG_WEBSOCKET_OPCODE_BINARY,
+        data.c_str(),
+        data.length());
 
-        mg_close_connection(conn);
+    mg_websocket_client_write(
+        conn,
+        MG_WEBSOCKET_OPCODE_DATACOMPLETE,
+        "",
+        0);
 
-        flag = true;
-    }   
+    return conn;
 }
 
 /*
@@ -536,6 +531,8 @@ void PrsonaServerWebSocketHandler::handleReadyState(
         case PRSONA_SET_FRESH_GENERATOR:
         case PRSONA_ADD_RAND_SEED_TO_GENERATOR:
         case PRSONA_SET_EG_BLIND_GENERATOR:
+        case PRSONA_EPOCH_BUILD_UP:
+        case PRSONA_EPOCH_BREAK_DOWN:
         case PRSONA_EPOCH_UPDATE:
         case PRSONA_NEW_USER_UPDATE:
         case PRSONA_RECEIVE_PARTIAL_DECRYPTION:
@@ -1105,7 +1102,10 @@ void PrsonaServerWebSocketHandler::distribute_new_vote(
     while (i < serverIPs.size())
     {
         if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
+        {
+            i++;
             continue;
+        }
 
         struct mg_connection *conn =
             mg_connect_websocket_client(
@@ -1135,6 +1135,12 @@ void PrsonaServerWebSocketHandler::distribute_new_vote(
             data.c_str(),
             data.length());
 
+        mg_websocket_client_write(
+            conn,
+            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
+            "",
+            0);
+
         while (!distributeSync.val2)
             distributeSync.cv.wait(syncLock);
 
@@ -1248,6 +1254,9 @@ void PrsonaServerWebSocketHandler::receive_vote(
     Twistpoint shortTermPublicKey;
     file >> shortTermPublicKey;
 
+    std::stringstream helper;
+    helper << shortTermPublicKey;
+
     BinaryBool shouldDeal;
     file >> shouldDeal;
 
@@ -1271,6 +1280,7 @@ void PrsonaServerWebSocketHandler::receive_vote(
     if (shouldDeal.val())
     {
         distribute_new_vote(pi, newVotes, shortTermPublicKey);
+
         release_update_locks(
             lck,
             serverIPs,
@@ -1414,22 +1424,29 @@ void PrsonaServerWebSocketHandler::build_up_midway_pseudonyms(
             false);
 
     struct synchronization_tool epochSync;
+    std::vector<struct mg_connection *> conns;
+    std::unique_lock<std::mutex> lck(epochSync.mtx);
     epochSync.val = 1;
     for (size_t i = 0; i < serverIPs.size(); i++)
     {
         if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
             continue;
 
-        distribute_epoch_updates(
-            serverIPs[i],
-            serverPorts[i],
-            data,
-            &epochSync);
+        struct mg_connection *currConn = 
+            distribute_epoch_updates(
+                serverIPs[i],
+                serverPorts[i],
+                data,
+                &epochSync);
+
+        conns.push_back(currConn);
     }
 
-    std::unique_lock<std::mutex> lck(epochSync.mtx);
     while (epochSync.val < serverIPs.size())
         epochSync.cv.wait(lck);
+
+    for (size_t i = 0; i < conns.size(); i++)
+        mg_close_connection(conns[i]);
     
     data = make_epoch_initiator_string(
         pi[0][0],
@@ -1491,23 +1508,30 @@ void PrsonaServerWebSocketHandler::break_down_midway_pseudonyms(
             true);
 
     struct synchronization_tool epochSync;
+    std::vector<struct mg_connection *> conns;
     epochSync.val = 1;
     for (size_t i = 0; i < serverIPs.size(); i++)
     {
         if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
             continue;
 
-        distribute_epoch_updates(
-            serverIPs[i],
-            serverPorts[i],
-            data,
-            &epochSync);
+        struct mg_connection *currConn = 
+            distribute_epoch_updates(
+                serverIPs[i],
+                serverPorts[i],
+                data,
+                &epochSync);
+
+        conns.push_back(currConn);
     }
 
     std::unique_lock<std::mutex> lck(epochSync.mtx);
     while (epochSync.val < serverIPs.size())
         epochSync.cv.wait(lck);
 
+    for (size_t i = 0; i < conns.size(); i++)
+        mg_close_connection(conns[i]);
+
     epochNum->fetch_add(1);
 
     mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

prsona/src/networking.cpp

@@ -110,10 +110,10 @@ int synchro_websocket_data_handler(
 
     if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_DATACOMPLETE)
     {
-        struct synchronization_tool *synch = (struct synchronization_tool *) user_data;
+        struct synchronization_tool *sync = (struct synchronization_tool *) user_data;
 
-        std::unique_lock<std::mutex> lck(synch->mtx);
-        synch->val++;
+        std::unique_lock<std::mutex> lck(sync->mtx);
+        sync->val++;
 
         return false;
     }
@@ -168,5 +168,6 @@ void file_websocket_close_handler(
 
     std::unique_lock<std::mutex> lck(sync->mtx);
     sync->val = 1;
+    sync->val2 = 1;
     sync->cv.notify_all();
 }

prsona/src/orchestratorMain.cpp

@@ -149,7 +149,7 @@ void wait_for_clients_ready(string dealer, int dealerPort, size_t numClients)
 
 void trigger_epoch(string dealer, int dealerPort)
 {
-    this_thread::sleep_for(ONE_SECOND);
+    wait_for_servers_ready(dealer, dealerPort);
 
     bool flag = false;
     while (!flag)
@@ -230,7 +230,7 @@ void trigger_reputation_proof(string target, int targetPort, string verifier, in
         if (!conn)
             continue;
 
-        sysString << "GET " << TRIGGER_REP_URI << verifier << ":" << verifierPort << " HTTP/1.1\r\n";
+        sysString << "GET " << TRIGGER_REP_URI << "?" << verifier << ":" << verifierPort << " HTTP/1.1\r\n";
         sysString << "Host: " << target << "\r\n\r\n";
         data = sysString.str();
 
@@ -239,6 +239,8 @@ void trigger_reputation_proof(string target, int targetPort, string verifier, in
 
         flag = true;
     }
+
+    this_thread::sleep_for(ONE_SECOND);
 }
 
 void execute_experiment(string dealer, int dealerPort)
@@ -271,10 +273,11 @@ void execute_experiment(string dealer, int dealerPort)
 
             case 'R':
                 char *target, *targetPortStr, *verifier, *verifierPortStr;
-                target = strtok(buffer, " :");
+                target = strtok(buffer + 1, " :");
                 targetPortStr = strtok(NULL, " :");
                 verifier = strtok(NULL, " :");
                 verifierPortStr = strtok(NULL, " :");
+
                 trigger_reputation_proof(
                     string(target),
                     atoi(targetPortStr),
@@ -412,7 +415,10 @@ int main(int argc, char* argv[])
     cout << "[ORC] Starting clients." << endl;
 
     for (size_t i = 0; i < numClients; i++)
+    {
         clientStartup.push_back(thread(start_remote_actor, clientIPs[i], false, "c" + to_string(i), experimentOutput));
+        this_thread::sleep_for(ONE_SECOND);
+    }
 
     cout << "[ORC] Waiting for confirmation that servers have all clients logged." << endl;    
 

prsona/src/server.cpp

@@ -170,6 +170,27 @@ Proof PrsonaServer::get_pseudonyms_commitment() const
     return generate_valid_pseudonyms_proof(currentPseudonyms);
 }
 
+void PrsonaServer::print_current_commitments() const
+{    
+    std::stringstream helper; 
+    std::cout << "Commitments:\n";
+    std::cout << "User tallies:\n";
+    for (size_t i = 0; i < currentUserEncryptedTallies.size(); i++)
+    {
+        helper.str("");
+        helper << currentUserEncryptedTallies[i];
+        std::cout << "Tally " << i + 1 << " of " << currentUserEncryptedTallies.size() << ": " << std::hex << oracle(helper.str()) << std::dec << "\n";
+    }
+
+    std::cout << "Server tallies:\n";
+    for (size_t i = 0; i < previousVoteTallies.size(); i++)
+    {
+        helper.str("");
+        helper << previousVoteTallies[i];
+        std::cout << "Tally " << i + 1 << " of " << previousVoteTallies.size() << ": " << std::hex << oracle(helper.str()) << std::dec << "\n";
+    }
+}
+
 /*
  * CLIENT INTERACTIONS
  */

prsona/src/serverMain.cpp

@@ -121,37 +121,6 @@ static void epoch_websocket_close_handler(
     sync->cv.notify_all();
 }
 
-static int tally_websocket_data_handler(
-    struct mg_connection *conn,
-    int bits,
-    char *data,
-    size_t data_len,
-    void *user_data)
-{
-    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE)
-        return false;
-
-    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_DATACOMPLETE)
-    {
-        struct synchronization_tool *sync = (struct synchronization_tool *) user_data;
-
-        unique_lock<mutex> lck(sync->mtx);
-        sync->val++;
-
-        sync->cv.notify_all();
-
-        return false;
-    }
-
-    if ((bits & 0xf) != MG_WEBSOCKET_OPCODE_BINARY && (bits & 0xf) != MG_WEBSOCKET_OPCODE_CONTINUATION)
-    {
-        std::cerr << "Unknown opcode: failing." << std::endl;
-        return false;
-    }
-
-    return true;
-}
-
 Twistpoint get_generator(
     default_random_engine& rng,
     vector<Proof>& pi,
@@ -454,22 +423,30 @@ vector<Proof> epoch_build_up(
                 false);
 
             struct synchronization_tool epochSync;
+            vector<struct mg_connection *> conns;
+            unique_lock<mutex> lck(epochSync.mtx);
             epochSync.val = 1;
             for (size_t j = 0; j < serverIPs.size(); j++)
             {
                 if (i == j)
                     continue;
 
-                distribute_epoch_updates(
-                    serverIPs[j],
-                    serverPorts[j],
-                    data,
-                    &epochSync);
+                struct mg_connection *currConn = 
+                    distribute_epoch_updates(
+                        serverIPs[j],
+                        serverPorts[j],
+                        data,
+                        &epochSync);
+
+                conns.push_back(currConn);
             }
-            unique_lock<mutex> lck(epochSync.mtx);
+
             while (epochSync.val < serverIPs.size())
                 epochSync.cv.wait(lck);
 
+            for (size_t j = 0; j < conns.size(); j++)
+                mg_close_connection(conns[j]);
+
             generatorProofHolder = pi[0];
         }
         else
@@ -555,21 +532,29 @@ void epoch_break_down(
                 true);
 
             struct synchronization_tool epochSync;
+            vector<struct mg_connection *> conns;
             epochSync.val = 1;
             for (size_t j = 0; j < serverIPs.size(); j++)
             {
                 if (i == j)
                     continue;
 
-                distribute_epoch_updates(
-                    serverIPs[j],
-                    serverPorts[j],
-                    data,
-                    &epochSync);
+                struct mg_connection *currConn = 
+                    distribute_epoch_updates(
+                        serverIPs[j],
+                        serverPorts[j],
+                        data,
+                        &epochSync);
+
+                conns.push_back(currConn);
             }
+
             unique_lock<mutex> lck(epochSync.mtx);
             while (epochSync.val < serverIPs.size())
                 epochSync.cv.wait(lck);
+
+            for (size_t j = 0; j < conns.size(); j++)
+                mg_close_connection(conns[j]);
         }
         else
         {
@@ -599,56 +584,49 @@ void tally_scores(
     std::vector<CurveBipoint>& serverTallyScores)
 {
     struct synchronization_tool tallySync;
-    tallySync.val = 0;
+    vector<struct mg_connection *> conns;
+
+    unique_lock<mutex> lck(tallySync.mtx);
+    tallySync.val = 1;
     for (size_t i = 0; i < serverIPs.size(); i++)
     {
         if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
-        {
-            unique_lock<mutex> lck(tallySync.mtx);
-            tallySync.val++;
             continue;
-        }
-        else
+        
+        struct mg_connection *currConn = NULL;
+        while (!currConn)
         {
-            bool flag = false;
-            while (!flag)
-            {
-                struct mg_connection *conn =
-                    mg_connect_websocket_client(
-                        serverIPs[i].c_str(),
-                        serverPorts[i],
-                        USE_SSL,
-                        NULL,
-                        0,
-                        GET_DECRYPTION_URI,
-                        "null",
-                        tally_websocket_data_handler,
-                        empty_websocket_close_handler,
-                        &tallySync);
-
-                if (!conn)
-                {
-                    std::cerr << "Trouble initiating epoch update with server at " << serverIPs[i] << ":" << serverPorts[i] << std::endl;
-                    continue;
-                }
+            currConn = mg_connect_websocket_client(
+                serverIPs[i].c_str(),
+                serverPorts[i],
+                USE_SSL,
+                NULL,
+                0,
+                GET_DECRYPTION_URI,
+                "null",
+                synchro_websocket_data_handler,
+                synchro_websocket_close_handler,
+                &tallySync);
 
-                mg_websocket_client_write(
-                    conn,
-                    MG_WEBSOCKET_OPCODE_DATACOMPLETE,
-                    "",
-                    0);
+            if (!currConn)
+                cerr << "Trouble getting partial decryption from server at " << serverIPs[i] << ":" << serverPorts[i] << endl;
+        }
 
-                mg_close_connection(conn);
+        mg_websocket_client_write(
+            currConn,
+            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
+            "",
+            0);
 
-                flag = true;
-            }
-        }
+        conns.push_back(currConn);
     }
 
-    unique_lock<mutex> lck(tallySync.mtx);
     while (tallySync.val < serverIPs.size())
         tallySync.cv.wait(lck);
 
+    for (size_t i = 0; i < conns.size(); i++)
+        mg_close_connection(conns[i]);
+
     std::vector<EGCiphertext> retval;
     std::vector<Twistpoint> currentPseudonyms = prsonaServer->get_current_pseudonyms();
     std::vector<Scalar> decryptedTalliedScores = prsonaServer->tally_scores();
@@ -710,62 +688,55 @@ void distribute_tallied_scores(
     data = buffer.str();
 
     struct synchronization_tool tallySync;
-    tallySync.val = 0;
+    vector<struct mg_connection *> conns;
+
+    unique_lock<mutex> lck(tallySync.mtx);
+    prsonaServer->receive_tallied_scores(userTallyScores, serverTallyScores);
+    tallySync.val = 1;
     for (size_t i = 0; i < serverIPs.size(); i++)
     {
         if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
-        {
-            prsonaServer->receive_tallied_scores(userTallyScores, serverTallyScores);
-            unique_lock<mutex> lck(tallySync.mtx);
-            tallySync.val++;
             continue;
-        }
-        else
+        
+        struct mg_connection *currConn = NULL;
+        while (!currConn)
         {
-            bool flag = false;
-            while (!flag)
-            {
-                struct mg_connection *conn =
-                    mg_connect_websocket_client(
-                        serverIPs[i].c_str(),
-                        serverPorts[i],
-                        USE_SSL,
-                        NULL,
-                        0,
-                        GIVE_DECRYPTION_URI,
-                        "null",
-                        tally_websocket_data_handler,
-                        empty_websocket_close_handler,
-                        &tallySync);
-
-                if (!conn)
-                {
-                    std::cerr << "Trouble initiating epoch update with server at " << serverIPs[i] << ":" << serverPorts[i] << std::endl;
-                    continue;
-                }
+            currConn = mg_connect_websocket_client(
+                serverIPs[i].c_str(),
+                serverPorts[i],
+                USE_SSL,
+                NULL,
+                0,
+                GIVE_DECRYPTION_URI,
+                "null",
+                synchro_websocket_data_handler,
+                synchro_websocket_close_handler,
+                &tallySync);
 
-                mg_websocket_client_write(
-                    conn,
-                    MG_WEBSOCKET_OPCODE_BINARY,
-                    data.c_str(),
-                    data.length());
+            if (!currConn)
+                cerr << "Trouble giving full re-encryption to server at " << serverIPs[i] << ":" << serverPorts[i] << endl;
+        }
 
-                mg_websocket_client_write(
-                    conn,
-                    MG_WEBSOCKET_OPCODE_DATACOMPLETE,
-                    "",
-                    0);
+        mg_websocket_client_write(
+            currConn,
+            MG_WEBSOCKET_OPCODE_BINARY,
+            data.c_str(),
+            data.length());
 
-                mg_close_connection(conn);
+        mg_websocket_client_write(
+            currConn,
+            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
+            "",
+            0);
 
-                flag = true;
-            }
-        }
+        conns.push_back(currConn);
     }
 
-    unique_lock<mutex> lck(tallySync.mtx);
     while (tallySync.val < serverIPs.size())
         tallySync.cv.wait(lck);
+
+    for (size_t i = 0; i < conns.size(); i++)
+        mg_close_connection(conns[i]);
 }
 
 void epoch(
@@ -781,7 +752,7 @@ void epoch(
     Twistpoint nextGenerator = PrsonaServer::EL_GAMAL_GENERATOR;
 
     struct synchronization_tool updateSync;
-    
+
     unique_lock<mutex> lck(*updateMtx, defer_lock);
     obtain_update_locks(
         lck,
@@ -848,13 +819,14 @@ void epoch(
 class EpochReadyHandler : public CivetHandler
 {
     public:
-        EpochReadyHandler(struct synchronization_tool *exitSync, struct synchronization_tool *readySync, size_t numServers)
-            : exitSync(exitSync), readySync(readySync), numServers(numServers)
+        EpochReadyHandler(struct synchronization_tool *exitSync, struct synchronization_tool *readySync, mutex *updateMtx, size_t numServers)
+            : exitSync(exitSync), readySync(readySync), updateMtx(updateMtx), numServers(numServers)
         { /* */ }
 
         bool handleGet(CivetServer *server, struct mg_connection *conn)
         {
             unique_lock<mutex> exitLock(exitSync->mtx, defer_lock);
+            unique_lock<mutex> updateLock(*updateMtx, defer_lock);
             unique_lock<mutex> readyLock(readySync->mtx);
 
             if (readySync->val < numServers)
@@ -864,19 +836,26 @@ class EpochReadyHandler : public CivetHandler
                           "text/plain\r\nConnection: close\r\n\r\n");
                 mg_printf(conn, "Server is waiting for other servers to begin.\n");
             }
-            else if (exitLock.try_lock())
+            else if (!exitLock.try_lock())
             {
                 mg_printf(conn,
-                          "HTTP/1.1 200 OK\r\nContent-Type: "
+                          "HTTP/1.1 503 Service Unavailable\r\nContent-Type: "
                           "text/plain\r\nConnection: close\r\n\r\n");
-                mg_printf(conn, "Server is ready for epoch.\n");
+                mg_printf(conn, "Server is still in a previous epoch.\n");   
             }
-            else
+            else if (!updateLock.try_lock())
             {
                 mg_printf(conn,
                           "HTTP/1.1 503 Service Unavailable\r\nContent-Type: "
                           "text/plain\r\nConnection: close\r\n\r\n");
-                mg_printf(conn, "Server is still in a previous epoch.\n");   
+                mg_printf(conn, "Server is handling other updates.\n");   
+            }
+            else
+            {
+                mg_printf(conn,
+                          "HTTP/1.1 200 OK\r\nContent-Type: "
+                          "text/plain\r\nConnection: close\r\n\r\n");
+                mg_printf(conn, "Server is ready for epoch.\n");
             }
 
             return true;
@@ -884,6 +863,7 @@ class EpochReadyHandler : public CivetHandler
 
     private:
         struct synchronization_tool *exitSync, *readySync;
+        mutex *updateMtx;
         const size_t numServers;
 
 };
@@ -1243,7 +1223,7 @@ int main(int argc, char *argv[])
         AltRemoteControlHandler triggerEpochHandler(1, &exitSync, "Server will initiate epoch!");
         server.addHandler(TRIGGER_EPOCH_URI, triggerEpochHandler);
 
-        EpochReadyHandler epochReadyHandler(&exitSync, &readySync, numServers);
+        EpochReadyHandler epochReadyHandler(&exitSync, &readySync, &updateMtx, numServers);
         server.addHandler(EPOCH_READY_URI, epochReadyHandler);
 
         EpochNumHandler epochNumHandler(&epochNum);
@@ -1256,10 +1236,14 @@ int main(int argc, char *argv[])
 
             if (exitSync.val2)
             {
-                cout << "[" << seedStr << "] Executing epoch." << endl;
+                size_t currEpoch = epochNum.load();
+                cout << "[" << seedStr << "] Executing epoch calculations (going from t = " << currEpoch << " to " << currEpoch + 1 << ")." << endl;
 
                 epoch(&updateMtx, &epochNum, prsonaServer, rng, serverIPs, serverPorts, selfIP, selfPort);
 
+                currEpoch = epochNum.load();
+                cout << "[" << seedStr << "] Epoch calculations complete (now in t = " << currEpoch << ")." << endl;
+
                 exitSync.val2 = 0;
             }
         }