updates to networking code

prsona/inc/networkClient.hpp

@@ -2,67 +2,24 @@
 #define __PRSONA_NETWORK_CLIENT_HPP
 
 #include <string>
-#include <sstream>
-#include <cstring>
-#include <cstdio>
+#include <vector>
+#include <random>
+
 #include "client.hpp"
-#include "CivetServer.h"
-
-#define MG_WEBSOCKET_OPCODE_DATACOMPLETE 0xb
-#define PRSONA_PORT 8080
-#define USE_SSL 0
-
-#define USER_TALLY_URI "/ws?h"
-#define USER_TALLY_COMMIT_URI "/ws?m"
-
-enum RequestType {
-    PRSONA_ADD_CLIENT = 'a',
-    PRSONA_RECEIVE_VOTE,
-    PRSONA_GET_BGN_PUBKEY,
-    PRSONA_GET_NUM_CLIENTS,
-    PRSONA_GET_NUM_SERVERS,
-    PRSONA_GET_VOTES_BY,
-    PRSONA_GET_ALL_VOTES,
-    PRSONA_GET_USER_TALLY,
-    PRSONA_GET_SERVER_TALLY,
-    PRSONA_GET_PSEUDONYMS,
-    PRSONA_GET_VOTE_ROW_COMMITMENT,
-    PRSONA_GET_VOTE_MATRIX_COMMITMENT,
-    PRSONA_GET_USER_TALLY_COMMITMENT,
-    PRSONA_GET_SERVER_TALLY_COMMITMENT,
-    PRSONA_GET_PSEUDONYMS_COMMITMENT,
-    PRSONA_GET_BGN_DETAILS,
-    PRSONA_ADD_CURR_SEED_TO_GENERATOR,
-    PRSONA_SET_FRESH_GENERATOR,
-    PRSONA_ADD_RAND_SEED_TO_GENERATOR,
-    PRSONA_SET_EG_BLIND_GENERATOR,
-    PRSONA_EPOCH_BUILD_UP,
-    PRSONA_EPOCH_BREAK_DOWN,
-    PRSONA_EPOCH_UPDATE,
-    PRSONA_NEW_USER_UPDATE,
-    PRSONA_GET_PARTIAL_DECRYPTION,
-    PRSONA_RECEIVE_PARTIAL_DECRYPTION,
-    PRSONA_GET_FRESH_GENERATOR = '-',
-    PRSONA_GET_EG_BLIND_GENERATOR ='_',
-    PRSONA_RECEIVE_FRESH_GENERATOR = 'a',
-    PRSONA_RECEIVE_VOTE_TALLY,
-    PRSONA_RECEIVE_NEW_USER_DATA,
-    PRSONA_VERIFY_REPUTATION_PROOF
-};
-
-struct synchronization_tool {
-    std::mutex mtx;
-    std::condition_variable cv;
-    size_t val, val2;
-};
+#include "networking.hpp"
+
+// #define BGN_TMP_FILE (TMP_DIR "bgn")
+// #define GEN_TMP_FILE (TMP_DIR "generator")
+// #define EPOCH_GEN_TMP_FILE (TMP_DIR "epoch")
 
 class PrsonaClientWebSocketHandler : public CivetWebSocketHandler {
     public:
         // CONSTRUCTORS
         PrsonaClientWebSocketHandler(
             const PrsonaClient *prsonaClient, 
-            const std::vector<std::string> serverIPs,
-            const std::string selfIP);
+            const std::vector<std::string>& serverIPs,
+            const std::string& selfIP,
+            const std::default_random_engine *generator);
 
         virtual bool handleConnection(
             CivetServer *server,
@@ -87,8 +44,42 @@ class PrsonaClientWebSocketHandler : public CivetWebSocketHandler {
         const PrsonaClient *prsonaClient;
         const std::vector<std::string> serverIPs;
         const std::string selfIP;
+        const std::default_random_engine *generator;
 
         void verify_reputation_proof(struct mg_connection *c) const;
 }
 
+template <typename T>
+T get_committed_val_from_file(
+    struct synchronization_tool *sync,
+    char *filename,
+    Proof& pi);
+
+template <typename T>
+T get_first_committed_val(
+    const std::string& server,
+    Proof& pi,
+    const Twistpoint& shortTermPublicKey,
+    const char *firstUri);
+
+Proof get_commitment_from_file(
+    struct synchronization_tool *sync,
+    char *filename);
+
+void get_additional_commitment(
+    const std::vector<std::string>& servers,
+    const std::string& skip,
+    std::vector<Proof>& pi,
+    const Twistpoint& shortTermPublicKey,
+    char *which);
+
+template <typename T>
+T get_server_committed_val(
+    std::default_random_engine *generator,
+    const std::vector<std::string>& serverIPs,
+    std::vector<Proof>& pi,
+    const Twistpoint& shortTermPublicKey,
+    const char *firstUri
+    const char *commitUri);
+
 #endif

prsona/inc/networkServer.hpp

@@ -2,85 +2,10 @@
 #define __PRSONA_NETWORK_SERVER_HPP
 
 #include <string>
-#include <sstream>
-#include <cstring>
-#include <cstdio>
+#include <vector>
+
 #include "server.hpp"
-#include "CivetServer.h"
-
-#define MG_WEBSOCKET_OPCODE_DATACOMPLETE 0xb
-#define UPDATE_LOCK_URI "/lock"
-#define UPDATE_UNLOCK_URI "/unlock"
-#define GIVE_NEW_VOTE_URI "/ws?b"
-#define ACCEPT_EPOCH_UPDATES_URI "/ws?w"
-#define GIVE_NEW_USER_URI "/ws?x"
-#define PRSONA_PORT 8080
-#define USE_SSL 0
-
-enum RequestType {
-    PRSONA_ADD_CLIENT = 'a',
-    PRSONA_RECEIVE_VOTE,
-    PRSONA_GET_BGN_PUBKEY,
-    PRSONA_GET_NUM_CLIENTS,
-    PRSONA_GET_NUM_SERVERS,
-    PRSONA_GET_VOTES_BY,
-    PRSONA_GET_ALL_VOTES,
-    PRSONA_GET_USER_TALLY,
-    PRSONA_GET_SERVER_TALLY,
-    PRSONA_GET_PSEUDONYMS,
-    PRSONA_GET_VOTE_ROW_COMMITMENT,
-    PRSONA_GET_VOTE_MATRIX_COMMITMENT,
-    PRSONA_GET_USER_TALLY_COMMITMENT,
-    PRSONA_GET_SERVER_TALLY_COMMITMENT,
-    PRSONA_GET_PSEUDONYMS_COMMITMENT,
-    PRSONA_GET_BGN_DETAILS,
-    PRSONA_ADD_CURR_SEED_TO_GENERATOR,
-    PRSONA_SET_FRESH_GENERATOR,
-    PRSONA_ADD_RAND_SEED_TO_GENERATOR,
-    PRSONA_SET_EG_BLIND_GENERATOR,
-    PRSONA_EPOCH_BUILD_UP,
-    PRSONA_EPOCH_BREAK_DOWN,
-    PRSONA_EPOCH_UPDATE,
-    PRSONA_NEW_USER_UPDATE,
-    PRSONA_GET_PARTIAL_DECRYPTION,
-    PRSONA_RECEIVE_PARTIAL_DECRYPTION,
-    PRSONA_GET_FRESH_GENERATOR = '-',
-    PRSONA_GET_EG_BLIND_GENERATOR ='_',
-    PRSONA_RECEIVE_FRESH_GENERATOR = 'a',
-    PRSONA_RECEIVE_VOTE_TALLY,
-    PRSONA_RECEIVE_NEW_USER_DATA,
-    PRSONA_VERIFY_REPUTATION_PROOF
-};
-
-struct synchronization_tool {
-    std::mutex mtx;
-    std::condition_variable cv;
-    size_t val, val2;
-};
-
-static int synchro_websocket_data_handler(
-    struct mg_connection *conn,
-    int bits,
-    char *data,
-    size_t data_len,
-    void *user_data);
-
-static void synchro_websocket_close_handler(
-    const struct mg_connection *conn,
-    void *user_data);
-
-static int empty_websocket_data_handler(
-    struct mg_connection *conn,
-    int bits,
-    char *data,
-    size_t data_len,
-    void *user_data)
-{ return false; }
-
-static void empty_websocket_close_handler(
-    const struct mg_connection *conn,
-    void *user_data)
-{ /* */ }
+#include "networking.hpp"
 
 void obtain_update_locks(
     std::unique_lock<std::mutex> &lck,
@@ -118,6 +43,7 @@ class PrsonaServerWebSocketHandler : public CivetWebSocketHandler  {
         PrsonaServerWebSocketHandler(
             const PrsonaServer *prsonaServer, 
             const std::mutex *updateMtx,
+            const std::atomic<size_t> *epochNum,
             const std::vector<std::string> serverIPs,
             const std::string selfIP);
 
@@ -143,7 +69,7 @@ class PrsonaServerWebSocketHandler : public CivetWebSocketHandler  {
     private:
         const PrsonaServer *prsonaServer;
         const std::mutex *updateMtx;
-        const size_t *epochNum;
+        const std::atomic<size_t> *epochNum;
         const std::vector<std::string> serverIPs;
         const std::string selfIP;
 

prsona/inc/networking.hpp

@@ -0,0 +1,170 @@
+#ifndef __PRSONA_NETWORKING_HPP
+#define __PRSONA_NETWORKING_HPP
+
+#include <mutex>
+#include <condition_variable>
+
+#include "CivetServer.h"
+
+#define MG_WEBSOCKET_OPCODE_DATACOMPLETE 0xb
+#define PRSONA_PORT 8080
+#define USE_SSL 0
+
+#define EXIT_URI "/exit"
+#define SERVER_READY_URI "/init"
+#define EPOCH_READY_URI "/ready"
+#define TRIGGER_EPOCH_URI "/epoch"
+#define TRIGGER_VOTE_URI "/vote"
+#define TRIGGER_REP_URI "/rep?"
+#define WHICH_EPOCH_URI "/which"
+#define UPDATE_LOCK_URI "/lock"
+#define UPDATE_UNLOCK_URI "/unlock"
+
+#define GIVE_NEW_VOTE_URI "/ws?b"
+#define PUBLIC_BGN_URI "/ws?c"
+#define NUM_CLIENTS_URI "/ws?d"
+#define VOTES_BY_URI "/ws?f"
+#define USER_TALLY_URI "/ws?h"
+#define VOTES_BY_COMMIT_URI "/ws?k"
+#define USER_TALLY_COMMIT_URI "/ws?m"
+#define PRIVATE_BGN_URI "/ws?p"
+#define GET_FRESH_GEN_URI "/ws?q"
+#define GIVE_FRESH_GEN_URI "/ws?r"
+#define GET_BLIND_GEN_URI "/ws?s"
+#define GIVE_BLIND_GEN_URI "/ws?t"
+#define EPOCH_BUILD_UP_URI "/ws?u"
+#define EPOCH_BREAK_DOWN_URI "/ws?v"
+#define ACCEPT_EPOCH_UPDATES_URI "/ws?w"
+#define GIVE_NEW_USER_URI "/ws?x"
+#define GET_DECRYPTION_URI "/ws?y"
+#define GIVE_DECRYPTION_URI "/ws?z"
+#define FRESH_GEN_URI "/ws?-"
+#define BLIND_GEN_URI "/ws?_"
+#define REP_PROOF_URI "/ws?d"
+
+#define TMP_FILE_SIZE 12
+#define TMP_DIR "~/tmp/"
+#define TMP_DIR_SIZE 6
+
+enum RequestType {
+    PRSONA_ADD_CLIENT = 'a',
+    PRSONA_RECEIVE_VOTE,
+    PRSONA_GET_BGN_PUBKEY,
+    PRSONA_GET_NUM_CLIENTS,
+    PRSONA_GET_NUM_SERVERS,
+    PRSONA_GET_VOTES_BY,
+    PRSONA_GET_ALL_VOTES,
+    PRSONA_GET_USER_TALLY,
+    PRSONA_GET_SERVER_TALLY,
+    PRSONA_GET_PSEUDONYMS,
+    PRSONA_GET_VOTE_ROW_COMMITMENT,
+    PRSONA_GET_VOTE_MATRIX_COMMITMENT,
+    PRSONA_GET_USER_TALLY_COMMITMENT,
+    PRSONA_GET_SERVER_TALLY_COMMITMENT,
+    PRSONA_GET_PSEUDONYMS_COMMITMENT,
+    PRSONA_GET_BGN_DETAILS,
+    PRSONA_ADD_CURR_SEED_TO_GENERATOR,
+    PRSONA_SET_FRESH_GENERATOR,
+    PRSONA_ADD_RAND_SEED_TO_GENERATOR,
+    PRSONA_SET_EG_BLIND_GENERATOR,
+    PRSONA_EPOCH_BUILD_UP,
+    PRSONA_EPOCH_BREAK_DOWN,
+    PRSONA_EPOCH_UPDATE,
+    PRSONA_NEW_USER_UPDATE,
+    PRSONA_GET_PARTIAL_DECRYPTION,
+    PRSONA_RECEIVE_PARTIAL_DECRYPTION,
+    PRSONA_GET_FRESH_GENERATOR = '-',
+    PRSONA_GET_EG_BLIND_GENERATOR ='_',
+    PRSONA_RECEIVE_FRESH_GENERATOR = 'a',
+    PRSONA_RECEIVE_VOTE_TALLY,
+    PRSONA_RECEIVE_NEW_USER_DATA,
+    PRSONA_VERIFY_REPUTATION_PROOF
+};
+
+struct synchronization_tool {
+    std::mutex mtx;
+    std::condition_variable cv;
+    size_t val, val2;
+};
+
+std::string random_string(size_t length);
+void set_temp_filename(struct mg_connection *conn) const;
+
+class RemoteControlHandler : public CivetHandler
+{
+    public:
+        ExitHandler(struct synchronization_tool *sync)
+            : sync(sync)
+        { /* */ }
+
+        ExitHandler(struct synchronization_tool *sync,
+            const std::string& message)
+            : sync(sync), message(message)
+        { /* */ }
+
+        bool handleGet(CivetServer *server, struct mg_connection *conn);
+
+    private:
+        const struct synchronization_tool *sync;
+        const std::string message;
+};
+
+class AltRemoteControlHandler : public CivetHandler
+{
+    public:
+        ExitHandler(size_t value, struct synchronization_tool *sync)
+            : value(value), sync(sync)
+        { /* */ }
+
+        ExitHandler(size_t value,
+            struct synchronization_tool *sync,
+            const std::string& message)
+            : value(value), sync(sync), message(message)
+        { /* */ }
+
+        bool handleGet(CivetServer *server, struct mg_connection *conn);
+        std::string query() const;
+
+    private:
+        const size_t value;
+        const struct synchronization_tool *sync;
+        const std::string message;
+        std::string query;
+};
+
+static int empty_websocket_data_handler(
+    struct mg_connection *conn,
+    int bits,
+    char *data,
+    size_t data_len,
+    void *user_data)
+{ return false; }
+
+static void empty_websocket_close_handler(
+    const struct mg_connection *conn,
+    void *user_data)
+{ /* */ }
+
+static int synchro_websocket_data_handler(
+    struct mg_connection *conn,
+    int bits,
+    char *data,
+    size_t data_len,
+    void *user_data);
+
+static void synchro_websocket_close_handler(
+    const struct mg_connection *conn,
+    void *user_data);
+
+static int file_websocket_data_handler(
+    struct mg_connection *conn,
+    int bits,
+    char *data,
+    size_t data_len,
+    void *user_data);
+
+static void file_websocket_close_handler(
+    const struct mg_connection *conn,
+    void *user_data);
+
+#endif

prsona/src/clientMain.cpp

@@ -5,24 +5,17 @@
 #include <fstream>
 #include <cstring>
 #include <cstdlib>
-#include <vector>
-#include <string>
 
 #include "networkClient.hpp"
 
-#define EPOCH_URI "/epoch"
-#define EXIT_URI "/exit"
-#define BGN_URI "/ws?c"
-#define BLIND_GEN_URI "/ws?_"
-
-#define BGN_TMP_FILE "~/tmp/bgn"
-#define GEN_TMP_FILE "~/tmp/generator"
-
 using namespace std;
 
-struct synchronization_tool exitSync, bgnSync, generatorSync;
+enum EventType {
+    CLIENT_MAKE_VOTE = 1,
+    CLIENT_MAKE_REP_PROOF
+};
 
-size_t epochNum = 0;
+struct synchronization_tool exitSync;
 
 // Initialize the classes we use
 void initialize_prsona_classes()
@@ -32,99 +25,295 @@ void initialize_prsona_classes()
     PrsonaBase::set_client_malicious();
 }
 
-PrsonaClient *create_client_from_files(size_t numServers)
+BGNPublicKey get_bgn_public_key_from_file(
+    struct synchronization_tool *sync,
+    const char *filename)
 {
-    unique_lock<mutex> lck1(bgnSync.mtx);
-    ifstream bgnFile(BGN_TMP_FILE);
+    unique_lock<mutex> lck(sync->mtx);
+    ifstream bgnFile(filename);
+
     BGNPublicKey publicKey;
     bgnFile >> publicKey;
-    lck1.unlock();
 
-    unique_lock<mutex> lck2(generatorSync.mtx);
-    ifstream genFile(GEN_TMP_FILE);
-    vector<Proof> pi;
-    Twistpoint generator;
+    return publicKey;
+}
 
-    size_t sizeOfPi;
-    genFile >> sizeOfPi;
-    for (size_t i = 0; i < sizeOfPi; i++)
+BGNPublicKey get_bgn_public_key(
+    default_random_engine *generator,
+    const vector<string>& serverIPs)
+{
+    struct synchronization_tool sync;
+
+    uniform_int_distribution<size_t> distribution(0, serverIPs.size() - 1);
+    size_t whichServer = distribution(*randomGenerator);
+
+    bool flag = false;
+    while (!flag)
     {
-        Proof currProof;
-        genFile >> currProof;
-        pi.push_back(currProof);
+        struct mg_connection *conn = mg_connect_websocket_client(
+            serverIPs[whichServer].c_str(),
+            PRSONA_PORT,
+            USE_SSL,
+            NULL,
+            0,
+            PUBLIC_BGN_URI,
+            "null",
+            file_websocket_data_handler,
+            file_websocket_close_handler,
+            (void *) &sync);
+
+        if (!conn)
+        {
+            cerr << "Couldn't obtain BGN details" << endl;
+            return 1;
+        }
+
+        unique_lock<mutex> lck(sync.mtx);
+        char *filename = set_temp_filename(conn);
+        sync.val = 0;
+
+        mg_websocket_client_write(
+            conn,
+            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
+            "",
+            0);
+
+        while (!sync.val)
+            sync.cv.wait(lck);
+
+        mg_close_connection(conn);
+
+        flag = true;
     }
-    genFile >> generator;
 
-    return new PrsonaClient(pi, generator, publicKey, numServers);
+    BGNPublicKey retval =
+        get_bgn_public_key_from_file(&sync, filename);
+
+    remove(filename);
+    delete filename;
+
+    return retval;
+}
+
+PrsonaClient *create_client(
+    default_random_engine *generator,
+    const vector<string> serverIPs,
+    size_t numServers)
+{
+    BGNPublicKey publicKey =
+        get_bgn_public_key(
+            generator,
+            serverIPs);
+
+    vector<Proof> generatorProof;
+    Twistpoint blindGenerator =
+        get_generator(
+            generator,
+            serverIPs,
+            generatorProof,
+            false);
+
+    return new PrsonaClient(generatorProof, blindGenerator, publicKey, numServers);
+}
+
+string make_vote_string(
+    const vector<Proof>& pi,
+    const vector<TwistBipoint>& newVotes,
+    const Twistpoint& shortTermPublicKey)
+{
+    stringstream buffer;
+    BinarySizeT sizeOfVector;
+
+    sizeOfVector.set(pi.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
+        buffer << pi[i];
+
+    sizeOfVector.set(newVotes.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
+        buffer << newVotes[i];
+
+    buffer << shortTermPublicKey;
+
+    BinaryBool shouldDeal(true);
+    buffer << shouldDeal;
+
+    return buffer.str();
 }
 
-PrsonaClient *create_client(size_t numServers)
+void send_item(
+    const string& target, const string& data, const char* whichUri)
 {
-    struct mg_connection *conn = mg_connect_websocket_client(
-        serverIPs[0].c_str(),
-        PRSONA_PORT,
-        USE_SSL,
-        NULL,
-        0,
-        BGN_URI,
-        "null",
-        bgn_websocket_data_handler,
-        bgn_websocket_close_handler,
-        NULL);
-
-    if (!conn)
+    struct synchronization_tool sync;
+    bool flag = false;
+    while (!flag)
     {
-        cerr << "Couldn't obtain BGN details" << endl;
-        return 1;
+        struct mg_connection *conn = mg_connect_websocket_client(
+            target.c_str(),
+            PRSONA_PORT,
+            USE_SSL,
+            NULL,
+            0,
+            whichUri,
+            "null",
+            synchro_websocket_data_handler,
+            empty_websocket_close_handler,
+            (void *) &sync);
+
+        if (!conn)
+        {
+            cerr << "Couldn't obtain BGN details" << endl;
+            continue;
+        }
+
+        unique_lock<mutex> lck(sync.mtx);
+        sync.val = 0;
+
+        mg_websocket_client_write(
+            conn,
+            MG_WEBSOCKET_OPCODE_BINARY,
+            data.c_str(),
+            data.length());
+
+        mg_websocket_client_write(
+            conn,
+            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
+            "",
+            0);
+
+        while (!sync.val)
+            sync.cv.wait(lck)
+
+        mg_close_connection(conn);
+
+        flag = true;
     }
+}
+
+void make_vote(
+    default_random_engine& generator,
+    const string& target,
+    PrsonaClient* prsonaClient,
+    const vector<string>& serverIPs,
+    size_t numClients)
+{
+    uniform_int_distribution<size_t> voteDistribution(
+        0, PrsonaBase::get_max_allowed_vote());
 
-    unique_lock<mutex> lck1(bgnSync.mtx);
-    remove(BGN_TMP_FILE);
-    bgnSync.val = 0;
-    mg_websocket_client_write(
-        conn,
-        MG_WEBSOCKET_OPCODE_DATACOMPLETE,
-        "",
-        0);
-
-    while (!bgnSync.val)
-        bgnSync.cv.wait(lck1);
-
-    lck1.unlock();
-
-    struct mg_connection *conn = mg_connect_websocket_client(
-        serverIPs[0].c_str(),
-        PRSONA_PORT,
-        USE_SSL,
-        NULL,
-        0,
-        BLIND_GEN_URI,
-        "null",
-        blind_gen_websocket_data_handler,
-        blind_gen_websocket_close_handler,
-        NULL);
-
-    if (!conn)
+    // Make the correct number of new votes, but shuffle where they go
+    vector<Scalar> votes;
+    vector<bool> replaces;
+    for (size_t j = 0; j < numClients; j++)
     {
-        cerr << "Couldn't obtain BGN details" << endl;
-        return 1;
+        votes.push_back(Scalar(voteDistribution(generator)));
+        replaces.push_back(j < numVotes);
     }
+    shuffle(replaces.begin(), replaces.end(), generator);
+
+    vector<Proof> generatorProof;
+    Twistpoint freshGenerator =
+        get_generator(
+            generator,
+            serverIPs,
+            generatorProof,
+            true);
+
+    prsonaClient->receive_fresh_generator(
+        generatorProof, freshGenerator);
+    
+    Twistpoint shortTermPublicKey =
+        prsonaClient->get_short_term_public_key();
+
+    vector<Proof> fullProof;
+    vector<TwistBipoint> encryptedVotes =
+        get_server_committed_val<vector<TwistBipoint>>(
+            &generator,
+            serverIPs,
+            fullProof,
+            shortTermPublicKey,
+            VOTES_BY_URI,
+            VOTES_BY_COMMIT_URI);
+    
+    vector<Proof> voteProof;
+    encryptedVotes =
+        prsonaClient->make_votes(
+            currVoteProof,
+            fullProof,
+            encryptedVotes,
+            votes,
+            replaces);
+
+    string data =
+        make_vote_string(voteProof, encryptedVotes, shortTermPublicKey);
+
+    send_item(target, data, GIVE_NEW_VOTE_URI);
+}
+
+string make_vote_string(
+    const vector<Proof>& pi,
+    const Twistpoint& shortTermPublicKey,
+    const Scalar& threshold)
+{
+    stringstream buffer;
+    BinarySizeT sizeOfVector;
 
-    unique_lock<mutex> lck2(generatorSync.mtx);
-    remove(GEN_TMP_FILE);
-    generatorSync.val = 0;
-    mg_websocket_client_write(
-        conn,
-        MG_WEBSOCKET_OPCODE_DATACOMPLETE,
-        "",
-        0);
+    sizeOfVector.set(pi.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
+        buffer << pi[i];
 
-    while (!generatorSync.val)
-        generatorSync.cv.wait(lck2);
+    buffer << shortTermPublicKey;
+    buffer << threshold;
 
-    lck2.unlock();
+    return buffer.str();
+}
 
-    return create_client_from_files(numServers);
+void make_reputation_proof(
+    default_random_engine& generator,
+    const string& target,
+    PrsonaClient* prsonaClient,
+    const vector<string>& serverIPs,
+    size_t numClients)
+{
+    vector<Proof> generatorProof;
+    Twistpoint freshGenerator =
+        get_generator(
+            generator,
+            serverIPs,
+            generatorProof,
+            true);
+
+    prsonaClient->receive_fresh_generator(
+        generatorProof, freshGenerator);
+
+    Twistpoint shortTermPublicKey =
+        prsonaClient->get_short_term_public_key();
+
+    vector<Proof> encryptedScoreProof;
+    EGCiphertext encryptedScore =
+        get_server_committed_val<EGCiphertext>(
+            &generator,
+            serverIPs,
+            encryptedScoreProof,
+            shortTermPublicKey,
+            USER_TALLY_URI,
+            USER_TALLY_COMMIT_URI);
+
+    prsonaClient->receive_vote_tally(
+        encryptedScoreProof,
+        encryptedScore);
+
+    Scalar threshold(0);
+
+    vector<Proof> repProof =
+        prsonaClient->generate_reputation_proof(
+            threshold, numClients);
+
+    string data =
+        make_rep_proof_string(repProof, shortTermPublicKey, threshold);
+
+    send_item(target, data, REP_PROOF_URI);
 }
 
 int main(int argc, char *argv[])
@@ -142,6 +331,8 @@ int main(int argc, char *argv[])
     vector<string> serverIPs, clientIPs;
     string selfIP;
 
+    string seedStr = "seed";
+
     char buffer[40];
     ifstream serverConfig("serverIPs.cfg");
     while (!serverConfig.eof())
@@ -171,6 +362,7 @@ int main(int argc, char *argv[])
     size_t numServers = serverIPs.size();
     size_t numClients = clientIPs.size();
     bool maliciousServers = true;
+    uniform_int_distribution<size_t> distribution(0, numServers - 1);
 
     if (argc > 1)
     {
@@ -178,6 +370,9 @@ int main(int argc, char *argv[])
         maliciousServers = setting;
     }
 
+    seed_seq seed(seedStr.begin(), seedStr.end());
+    default_random_engine generator(seed);
+
     cout << "Establishing PRSONA client with the following parameters: " << endl;
     cout << numServers << " PRSONA servers" << endl;
     cout << numClients << " PRSONA clients" << endl;
@@ -193,15 +388,48 @@ int main(int argc, char *argv[])
     PrsonaClient *prsonaClient = create_client(numServers);
     CivetServer server(options);
 
-    PrsonaClientWebSocketHandler wsHandler(prsonaClient, &updateMtx, &epochNum, serverIPs, selfIP);
+    PrsonaClientWebSocketHandler wsHandler(prsonaClient, serverIPs, selfIP, &generator);
     server.addWebSocketHandler("/ws", wsHandler);
 
-    ExitHandler exitHandler;
+    unique_lock<mutex> exitLock(exitSync.mtx);
+    exitSync.val = 0;
+    exitSync.val2 = 0;
+    RemoteControlHandler exitHandler(&exitSync, "Client coming down!");
     server.addHandler(EXIT_URI, exitHandler);
 
-    unique_lock<mutex> lck(exitSync.mtx);
+    AltRemoteControlHandler triggerVoteHandler(CLIENT_MAKE_VOTE, &exitSync, "Client will make new votes!");
+    server.addHandler(TRIGGER_VOTE_URI, triggerVoteHandler);
+
+    AltRemoteControlHandler triggerRepHandler(CLIENT_MAKE_REP_PROOF, &exitSync, "Client will make new votes!");
+    server.addHandler(TRIGGER_REP_URI, triggerRepHandler);
+
     while (!exitSync.val)
-        exitSync.cv.wait(lck);
+    {
+        while (!exitSync.val && !exitSync.val2)
+            exitSync.cv.wait(lck);
+
+        size_t whichServer = distribution(generator);
+
+        switch (exitSync.val2)
+        {
+            case CLIENT_MAKE_VOTE:
+                make_vote(
+                    serverIPs[whichServer],
+                    prsonaClient);
+                break;
+
+            case CLIENT_MAKE_REP_PROOF:
+                make_reputation_proof(
+                    triggerRepHandler.query(),
+                    prsonaClient);
+                break;
+
+            default:
+                break;
+        }
+
+        exitSync.val2 = 0;
+    }
 
     mg_exit_library();
 

prsona/src/networkClient.cpp

@@ -1,26 +1,331 @@
+#include <cstring>
+#include <cstdio>
+
 #include "networkClient.hpp"
 
-std::string random_string(size_t length)
+template <>
+EGCiphertext get_committed_val_from_file<EGCiphertext>(
+    struct synchronization_tool *sync,
+    char *filename,
+    Proof& pi)
+{
+    std::unique_lock<std::mutex> lck(sync->mtx);
+    std::ifstream valFile(filename);
+
+    EGCiphertext retval;
+    valFile >> pi;
+    valFile >> retval;
+
+    return retval;
+}
+
+template <>
+std::vector<TwistBipoint> get_committed_val_from_file<std::vector<TwistBipoint>>(
+    struct synchronization_tool *sync,
+    char *filename,
+    Proof& pi)
+{
+    std::unique_lock<std::mutex> lck(sync->mtx);
+    std::ifstream valFile(filename);
+
+    valFile >> pi;
+
+    std::vector<TwistBipoint> retval;
+
+    BinarySizeT sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
+    {
+        TwistBipoint currVote;
+        valFile >> currVote;
+
+        retval.push_back(currVote);
+    }
+
+    return retval;
+}
+
+template <typename T>
+T get_first_committed_val(
+    const std::string& server,
+    Proof& pi,
+    const Twistpoint& shortTermPublicKey,
+    const char *firstUri)
+{
+    struct synchronization_tool sync;
+
+    std::stringstream buffer;
+    std::string data;
+
+    buffer << shortTermPublicKey;
+    data = buffer.str();
+
+    bool flag = false;
+    while (!flag)
+    {
+        struct mg_connection *conn =
+            mg_connect_websocket_client(
+                server.c_str(),
+                PRSONA_PORT,
+                USE_SSL,
+                NULL,
+                0,
+                firstUri,
+                "null",
+                file_websocket_data_handler,
+                file_websocket_close_handler,
+                (void *) &sync);
+
+        if (!conn)
+        {
+            std::cerr << "Trouble getting encrypted score from server at " << server << std::endl;
+            continue;
+        }
+
+        std::unique_lock<std::mutex> lck(sync.mtx);
+        char *filename = set_temp_filename(conn);
+        sync.val = 0;
+
+        mg_websocket_client_write(
+            conn,
+            MG_WEBSOCKET_OPCODE_BINARY,
+            data.c_str(),
+            data.length());
+
+        mg_websocket_client_write(
+            conn,
+            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
+            "",
+            0);
+
+        while (!sync.val)
+            sync.cv.wait(lck)
+
+        mg_close_connection(conn);
+
+        flag = true;
+    }
+
+    T retval = get_committed_val_from_file<T>(filename, pi);
+
+    remove(filename);
+    delete filename;
+
+    return retval;
+}
+
+Proof get_commitment_from_file(
+    struct synchronization_tool *sync,
+    char *filename)
+{
+    std::unique_lock<std::mutex> lck(sync->mtx);
+    std::ifstream scoreFile(filename);
+
+    Proof retval;
+    scoreFile >> retval;
+
+    return retval;
+}
+
+void get_additional_commitment(
+    const std::vector<std::string>& servers,
+    const std::string& skip,
+    std::vector<Proof>& pi,
+    const Twistpoint& shortTermPublicKey,
+    char *commitUri)
+{
+    std::vector<char *> commitmentFilenames;
+    std::vector<struct synchronization_tool *> commitmentSyncs;
+
+    for (size_t i = 0; i < servers.size(); i++)
+    {
+        if (servers[i] == skip)
+            continue;
+
+        struct synchronization_tool *currSync = new struct synchronization_tool;
+        commitmentSyncs.push_back(currSync);
+
+        bool flag = false;
+        while (!flag)
+        {
+            struct mg_connection *conn =
+                mg_connect_websocket_client(
+                    servers[i].c_str(),
+                    PRSONA_PORT,
+                    USE_SSL,
+                    NULL,
+                    0,
+                    commitUri,
+                    "null",
+                    file_websocket_data_handler,
+                    file_websocket_close_handler,
+                    (void *) currSync);
+
+            if (!conn)
+            {
+                std::cerr << "Trouble getting commitment from server at " << servers[i] << std::endl;
+                continue;
+            }
+
+            std::unique_lock<std::mutex> lck(currSync->mtx);
+            currSync->val = 0;
+            commitmentFilenames.push_back(set_temp_filename(conn));
+
+            mg_websocket_client_write(
+                conn,
+                MG_WEBSOCKET_OPCODE_DATACOMPLETE,
+                "",
+                0);
+
+            while (!currSync->val)
+                currSync->cv.wait(lck)
+
+            mg_close_connection(conn);
+
+            flag = true;
+        }  
+    }
+    
+    for (size_t i = 0; i < commitmentFilenames.size(); i++)
+    {
+        pi.push_back(
+            get_commitment_from_file(
+                commitmentSyncs[i], commitmentFilenames[i]));
+
+        delete commitmentSyncs[i];
+
+        remove(commitmentFilenames[i]);
+        delete commitmentFilenames[i];
+    }
+}
+
+template <typename T>
+T get_server_committed_val(
+    std::default_random_engine *generator,
+    const std::vector<std::string>& serverIPs,
+    std::vector<Proof>& pi,
+    const Twistpoint& shortTermPublicKey,
+    const char *firstUri
+    const char *commitUri)
+{
+    std::uniform_int_distribution<size_t> distribution(0, serverIPs.size() - 1);
+    size_t whichServer = distribution(*generator);
+
+    pi.clear();
+
+    Proof firstProof;
+    T retval =
+        get_first_committed_val<T>(
+            serverIPs[whichServer],
+            firstProof,
+            shortTermPublicKey
+            firstUri);
+
+    pi.push_back(firstProof);
+
+    get_additional_commitment(
+        serverIPs,
+        serverIPs[whichServer],
+        pi,
+        shortTermPublicKey,
+        commitUri);
+
+    return retval;
+}
+
+Twistpoint get_generator_from_file(
+    struct synchronization_tool *sync,
+    const char *filename,
+    std::vector<Proof>& pi)
+{
+    std::ifstream genFile(filename);
+
+    BinarySizeT sizeOfVector;
+    genFile >> sizeOfVector;
+
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
+    {
+        Proof currProof;
+        genFile >> currProof;
+
+        pi.push_back(currProof);
+    }
+
+    Twistpoint retval;
+    genFile >> retval;
+
+    return retval;
+}
+
+Twistpoint get_generator(
+    std::default_random_engine *randomGenerator,
+    const std::vector<std::string>& serverIPs,
+    std::vector<Proof>& pi,
+    bool fresh)
 {
-    auto randchar = []() -> char
+    struct synchronization_tool sync;
+
+    std::uniform_int_distribution<size_t> distribution(0, serverIPs.size() - 1);
+    size_t whichServer = distribution(*randomGenerator);
+
+    pi.clear();
+
+    const char* whichUri = (fresh ? FRESH_GEN_URI : BLIND_GEN_URI);
+
+    bool flag = false;
+    while (!flag)
     {
-        const char charset[] =
-        "0123456789_-"
-        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-        "abcdefghijklmnopqrstuvwxyz";
-        const size_t max_index = (sizeof(charset) - 1);
-        return charset[ rand() % max_index ];
-    };
-    std::string str(length,0);
-    std::generate_n(str.begin(), length, randchar);
-    return str;
+        struct mg_connection *conn = mg_connect_websocket_client(
+            serverIPs[whichServer].c_str(),
+            PRSONA_PORT,
+            USE_SSL,
+            NULL,
+            0,
+            whichUri,
+            "null",
+            file_websocket_data_handler,
+            file_websocket_close_handler,
+            (void *) &sync);
+
+        if (!conn)
+        {
+            cerr << "Couldn't obtain BGN details" << endl;
+            return 1;
+        }
+
+        unique_lock<mutex> lck(sync.mtx);
+        char *filename = set_temp_filename(conn);
+        sync.val = 0;
+
+        mg_websocket_client_write(
+            conn,
+            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
+            "",
+            0);
+
+        while (!sync.val)
+            sync.cv.wait(lck);
+
+        mg_close_connection(conn);
+
+        flag = true;
+    }
+
+    Twistpoint retval =
+        get_generator_from_file(&sync, filename, pi);
+
+    remove(filename);
+    delete filename;
+
+    return retval;
 }
 
 PrsonaClientWebSocketHandler::PrsonaClientWebSocketHandler(
     const PrsonaClient *prsonaClient, 
-    const std::vector<std::string> serverIPs,
-    const std::string selfIP)
-: prsonaClient(prsonaClient), serverIPs(serverIPs), selfIP(selfIP)
+    const std::vector<std::string>& serverIPs,
+    const std::string& selfIP,
+    const std::default_random_engine *generator)
+: prsonaClient(prsonaClient), serverIPs(serverIPs),
+    selfIP(selfIP), generator(generator)
 { /* */ }
 
 virtual bool PrsonaClientWebSocketHandler::handleConnection(
@@ -34,20 +339,6 @@ virtual bool PrsonaClientWebSocketHandler::handleConnection(
     return flag;
 }
 
-void PrsonaServerWebSocketHandler::set_temp_filename(
-    struct mg_connection *conn) const
-{
-    std::string filename = random_string(TMP_FILE_SIZE);
-    
-    char *c_filename = new char[TMP_FILE_SIZE+TMP_DIR_SIZE+1];
-    strncpy(c_filename, TMP_DIR, TMP_DIR_SIZE);
-    for (size_t i = 0; i < TMP_FILE_SIZE; i++)
-        c_filename[i + TMP_DIR_SIZE] = filename[i];
-    c_filename[TMP_DIR_SIZE + TMP_FILE_SIZE] = 0;
-
-    mg_set_user_connection_data(conn, c_filename);
-}
-
 virtual void PrsonaClientWebSocketHandler::handleReadyState(
     CivetServer *server,
     struct mg_connection *conn)
@@ -123,110 +414,6 @@ virtual void PrsonaClientWebSocketHandler::handleClose(
     delete filename;
 }
 
-EGCiphertext PrsonaClientWebSocketHandler::get_first_encrypted_score(
-    const std::string& server,
-    Proof& pi,
-    const Twistpoint& shortTermPublicKey) const
-{
-    struct synchronization_tool synch;
-    std::unique_lock<std::mutex> lck(synch.mtx, std::defer_lock);
-
-    bool flag = false;
-    while (!flag)
-    {
-        struct mg_connection *conn =
-            mg_connect_websocket_client(
-                server.c_str(),
-                PRSONA_PORT,
-                USE_SSL,
-                NULL,
-                0,
-                USER_TALLY_URI,
-                "null",
-                score_websocket_data_handler,
-                score_websocket_close_handler,
-                (void *) &synch);
-
-        if (!conn)
-        {
-            std::cerr << "Trouble getting encrypted score from server at " << recipient << std::endl;
-            continue;
-        }
-
-        lck.lock();
-        synch.val = 0;
-        mg_websocket_client_write(
-            conn,
-            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
-            "",
-            0);
-
-        flag = true;
-    }
-
-    while (!synch.val)
-        synch.cv.wait(lck)
-
-    // TODO: Finish retrieving first score
-}
-
-void PrsonaClientWebSocketHandler::get_additional_commitment(
-    const std::vector<std::string>& server,
-    std::vector<Proof>& pi,
-    const Twistpoint& shortTermPublicKey) const
-{
-    bool flag = false;
-    while (!flag)
-    {
-        struct mg_connection *conn =
-            mg_connect_websocket_client(
-                server.c_str(),
-                PRSONA_PORT,
-                USE_SSL,
-                NULL,
-                0,
-                USER_TALLY_COMMIT_URI,
-                "null",
-                synchro_websocket_data_handler,
-                empty_websocket_close_handler,
-                (void *) synch);
-
-        if (!conn)
-        {
-            std::cerr << "Trouble getting encrypted score commitment from server at " << recipient << std::endl;
-            continue;
-        }
-
-        mg_websocket_client_write(
-            conn,
-            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
-            "",
-            0);
-
-        flag = true;
-    }  
-
-    // TODO: Finish retrieving rest of score commitments
-}
-
-EGCiphertext PrsonaClientWebSocketHandler::get_encrypted_score(
-    std::vector<Proof>& pi,
-    const Twistpoint& shortTermPublicKey) const
-{
-    pi.clear();
-
-    Proof firstProof;
-    EGCiphertext retval =
-        get_first_encrypted_score(
-            serverIPs[0], firstProof, shortTermPublicKey);
-
-    pi.push_back(firstProof);
-
-    get_additional_commitment(serverIPs, pi, shortTermPublicKey);
-
-    return retval;
-}
-
 void PrsonaClientWebSocketHandler::verify_reputation_proof(
     struct mg_connection *conn, const char *filename) const
 {
@@ -236,27 +423,35 @@ void PrsonaClientWebSocketHandler::verify_reputation_proof(
     Twistpoint shortTermPublicKey;
     Scalar threshold;
 
-    size_t sizeOfVector;
+    BinarySizeT sizeOfVector;
     file >> sizeOfVector;
-    for (size_t i = 0; i < sizeOfVector; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         Proof currProof;
         file >> currProof;
+
         pi.push_back(currProof);
     }
     file >> shortTermPublicKey;
     file >> threshold;
 
-    for (size_t i = 0; i < sizeOfVector; i++)
-    {
-        CurveBipoint currScore;
-        file >> currScore;
-        serverScores.push_back(currScore);
-    }
+    std::vector<Proof> generatorProof;
+    Twistpoint freshGenerator =
+        get_generator(
+            &generator,
+            serverIPs,
+            generatorProof,
+            true);
 
     std::vector<Proof> encryptedScoreProof;
     EGCiphertext encryptedScore =
-        get_encrypted_score(encryptedScoreProof);
+        get_server_committed_val<EGCiphertext>(
+            &generator,
+            serverIPs,
+            encryptedScoreProof,
+            shortTermPublicKey,
+            USER_TALLY_URI,
+            USER_TALLY_COMMIT_URI);
 
     bool flag =
         prsonaClient->verify_reputation_proof(

prsona/src/networkServer.cpp

@@ -1,67 +1,15 @@
 #include <iostream>
+#include <sstream>
+#include <cstring>
+#include <cstdio>
 
 #include "networkServer.hpp"
 
-#define TMP_FILE_SIZE 12
-#define TMP_DIR "~/tmp/"
-#define TMP_DIR_SIZE 6
-
-std::string random_string(size_t length)
-{
-    auto randchar = []() -> char
-    {
-        const char charset[] =
-        "0123456789_-"
-        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-        "abcdefghijklmnopqrstuvwxyz";
-        const size_t max_index = (sizeof(charset) - 1);
-        return charset[ rand() % max_index ];
-    };
-    std::string str(length,0);
-    std::generate_n(str.begin(), length, randchar);
-    return str;
-}
-
-static int synchro_websocket_data_handler(
-    struct mg_connection *conn,
-    int bits,
-    char *data,
-    size_t data_len,
-    void *user_data)
-{
-    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE)
-        return false;
-
-    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_DATACOMPLETE)
-    {
-        struct synchronization_tool *synch = (struct synchronization_tool *) user_data;
-
-        unique_lock<mutex> lck(synch->mtx);
-        synch->val++;
-
-        return false;
-    }
-
-    std::cerr << "Unknown response when trying to get update lock." << std::endl;
-    return false;    
-}
-
-static void synchro_websocket_close_handler(
-    const struct mg_connection *conn,
-    void *user_data)
-{
-    struct synchronization_tool *synch = (struct synchronization_tool *) user_data;
-
-    unique_lock<mutex> lck(synch->mtx);
-    synch->val2 = 0;
-    synch->cv.notify_all();
-}
-
 void obtain_update_locks(
     std::unique_lock<std::mutex> &lck,
     const std::vector<std::string>& serverIPs,
     const std::string& selfIP,
-    struct synchronization_tool *synch)
+    struct synchronization_tool *sync)
 {
     size_t i = 0;
     while (i < serverIPs.size())
@@ -84,7 +32,7 @@ void obtain_update_locks(
                 "null",
                 synchro_websocket_data_handler,
                 synchro_websocket_close_handler,
-                (void *) synch);
+                (void *) sync);
 
         if (!conn)
         {
@@ -92,19 +40,21 @@ void obtain_update_locks(
             continue;
         }
 
-        unique_lock<mutex> lck(synch->mtx);
-        synch->val = 0;
-        synch->val2 = 1;
+        unique_lock<mutex> lck(sync->mtx);
+        sync->val = 0;
+        sync->val2 = 0;
         mg_websocket_client_write(
             conn,
             MG_WEBSOCKET_OPCODE_DATACOMPLETE,
             "",
             0);
 
-        while (synch->val2)
-            synch->cv.wait(lck);
+        while (!sync->val2)
+            sync->cv.wait(lck);
+
+        mg_close_connection(conn);
 
-        if (synch->val)
+        if (sync->val)
             i++;
     }
 }
@@ -113,7 +63,7 @@ void release_update_locks(
     std::unique_lock<std::mutex> &lck,
     const std::vector<std::string>& serverIPs,
     const std::string& selfIP,
-    struct synchronization_tool *synch)
+    struct synchronization_tool *sync)
 {
     ssize_t i = serverIPs.size() - 1;
     while (i >= 0)
@@ -136,7 +86,7 @@ void release_update_locks(
                 "null",
                 synchro_websocket_data_handler,
                 synchro_websocket_close_handler,
-                (void *) synch);
+                (void *) sync);
 
         if (!conn)
         {
@@ -144,19 +94,21 @@ void release_update_locks(
             continue;
         }
 
-        unique_lock<mutex> lck(synch->mtx);
-        synch->val = 0;
-        synch->val2 = 1;
+        unique_lock<mutex> lck(sync->mtx);
+        sync->val = 0;
+        sync->val2 = 0;
         mg_websocket_client_write(
             conn,
             MG_WEBSOCKET_OPCODE_DATACOMPLETE,
             "",
             0);
 
-        while (synch->val2)
-            synch->cv.wait(lck);
+        while (!sync->val2)
+            sync->cv.wait(lck);
 
-        if (synch->val)
+        mg_close_connection(conn);
+
+        if (sync->val)
             i--;
     }
 }
@@ -165,10 +117,11 @@ std::string make_epoch_initiator_string(
     const std::vector<Proof>& generatorProof,
     const Twistpoint& nextGenerator)
 {
-    stringstream buffer;
+    std::stringstream buffer;
     
-    buffer << generatorProof.size();
-    for (size_t i = 0; i < generatorProof.size(); i++)
+    BinarySizeT sizeOfVector(generatorProof.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << generatorProof[i];
     
     buffer << nextGenerator;
@@ -182,15 +135,16 @@ void read_epoch_initiator_string(
     Twistpoint& nextGenerator)
 {
     ifstream file(filename);
-    size_t limitI;
+    BinarySizeT sizeOfVector;
 
     generatorProof.clear();
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+    file >> sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         Proof currProof;
         file >> currProof;
+
         generatorProof.push_back;
     }
     
@@ -211,65 +165,81 @@ std::string make_epoch_update_string(
     const Twistpoint& nextGenerator,
     bool doUserTallies)
 {
-    stringstream buffer;
+    std::stringstream buffer;
     
-    buffer << pi.size();
-    for (size_t i = 0; i < pi.size(); i++)
+    BinarySizeT sizeOfVectorI, sizeOfVectorJ;
+
+    sizeOfVectorI.set(pi.size());
+    buffer << sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
-        buffer << pi[i].size();
-        for (size_t j = 0; j < pi[i].size(); j++)
+        sizeOfVectorJ.set(pi[i].size());
+        buffer << sizeOfVectorJ;
+
+        for (size_t j = 0; j < sizeOfVectorJ.val(); j++)
             buffer << pi[i][j];
     }
 
-    buffer << permutationCommits.size();
-    for (size_t i = 0; i < permutationCommits.size(); i++)
-        for (size_t j = 0; j < permutationCommits[i].size(); j++)
+    sizeOfVectorI.set(permutationCommits.size());
+    buffer << sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
             buffer << permutationCommits[i][j];
 
-    buffer << freshPseudonymCommits.size();
-    for (size_t i = 0; i < freshPseudonymCommits.size(); i++)
-        for (size_t j = 0; j < freshPseudonymCommits[i].size(); j++)
+    sizeOfVectorI.set(freshPseudonymCommits.size());
+    buffer << sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
             buffer << freshPseudonymCommits[i][j];
 
-    buffer << freshPseudonymSeedCommits.size();
-    for (size_t i = 0; i < freshPseudonymSeedCommits.size(); i++)
-        for (size_t j = 0; j < freshPseudonymSeedCommits[i].size(); j++)
+    sizeOfVectorI.set(freshPseudonymSeedCommits.size());
+    buffer << sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
             buffer << freshPseudonymSeedCommits[i][j];
 
-    buffer << serverTallyCommits.size();
-    for (size_t i = 0; i < serverTallyCommits.size(); i++)
-        for (size_t j = 0; j < serverTallyCommits[i].size(); j++)
+    sizeOfVectorI.set(serverTallyCommits.size());
+    buffer << sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
             buffer << serverTallyCommits[i][j];
 
-    buffer << partwayVoteMatrixCommits.size();
-    for (size_t i = 0; i < partwayVoteMatrixCommits.size(); i++)
-        for (size_t j = 0; j < partwayVoteMatrixCommits[i].size(); j++)
-            for (size_t k = 0; k < partwayVoteMatrixCommits[i][j].size(); k++)
+    sizeOfVectorI.set(partwayVoteMatrixCommits.size());
+    buffer << sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
+            for (size_t k = 0; k < sizeOfVectorI.val(); k++)
                 buffer << partwayVoteMatrixCommits[i][j][k];
 
-    buffer << finalVoteMatrixCommits.size();
-    for (size_t i = 0; i < finalVoteMatrixCommits.size(); i++)
-        for (size_t j = 0; j < finalVoteMatrixCommits[i].size(); j++)
-            for (size_t k = 0; k < finalVoteMatrixCommits[i][j].size(); k++)
+    sizeOfVectorI.set(finalVoteMatrixCommits.size());
+    buffer << sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
+            for (size_t k = 0; k < sizeOfVectorI.val(); k++)
                 buffer << finalVoteMatrixCommits[i][j][k];
 
-    buffer << userTallyMaskCommits.size();
-    for (size_t i = 0; i < userTallyMaskCommits.size(); i++)
-        for (size_t j = 0; j < userTallyMaskCommits[i].size(); j++)
+    sizeOfVectorI.set(userTallyMaskCommits.size());
+    buffer << sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
             buffer << userTallyMaskCommits[i][j];
 
-    buffer << userTallyMessageCommits.size();
-    for (size_t i = 0; i < userTallyMessageCommits.size(); i++)
-        for (size_t j = 0; j < userTallyMessageCommits[i].size(); j++)
+    sizeOfVectorI.set(userTallyMessageCommits.size());
+    buffer << sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
             buffer << userTallyMessageCommits[i][j];
 
-    buffer << userTallySeedCommits.size();
-    for (size_t i = 0; i < userTallySeedCommits.size(); i++)
-        for (size_t j = 0; j < userTallySeedCommits[i].size(); j++)
+    sizeOfVectorI.set(userTallySeedCommits.size());
+    buffer << sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
             buffer << userTallySeedCommits[i][j];
 
     buffer << nextGenerator;
-    buffer << doUserTallies;
+
+    BinaryBool flag(doUserTallies);
+    buffer << flag;
 
     return buffer.str();
 }
@@ -289,7 +259,7 @@ bool read_epoch_update_string(
     Twistpoint& nextGenerator)
 {
     ifstream file(filename);
-    size_t limitI, limitJ;
+    BinarySizeT sizeOfVectorI, sizeOfVectorJ;
     
     pi.clear();
     permutationCommits.clear();
@@ -302,85 +272,92 @@ bool read_epoch_update_string(
     userTallyMessageCommits.clear();
     userTallySeedCommits.clear();
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+
+    file >> sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
         std::vector<Proof> currRow;
 
-        file >> limitJ;
-        for (size_t j = 0; j < limitJ; j++)
+        file >> sizeOfVectorJ;
+        for (size_t j = 0; j < sizeOfVectorJ.val(); j++)
         {
             Proof currProof;
             file >> currProof;
+
             currRow.push_back(currProof);
         }
 
         pi.push_back(currRow);
     }
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+    file >> sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
         std::vector<Twistpoint> currRow;
-        for (size_t j = 0; j < limitI; j++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
         {
             Twistpoint currCommit;
             file >> currCommit;
+
             currRow.push_back(currCommit);
         }
         permutationCommits.push_back(currRow);
     }
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+    file >> sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
         std::vector<Twistpoint> currRow;
-        for (size_t j = 0; j < limitI; j++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
         {
             Twistpoint currCommit;
             file >> currCommit;
+
             currRow.push_back(currCommit);
         }
         freshPseudonymCommits.push_back(currRow);
     }
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+    file >> sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
         std::vector<Twistpoint> currRow;
-        for (size_t j = 0; j < limitI; j++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
         {
             Twistpoint currCommit;
             file >> currCommit;
+
             currRow.push_back(currCommit);
         }
         freshPseudonymSeedCommits.push_back(currRow);
     }
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+    file >> sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
         std::vector<CurveBipoint> currRow;
-        for (size_t j = 0; j < limitI; j++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
         {
             CurveBipoint currCommit;
             file >> currCommit;
+
             currRow.push_back(currCommit);
         }
         serverTallyCommits.push_back(currRow);
     }
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+    file >> sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
         std::vector<std::vector<TwistBipoint>> currMatrix;
-        for (size_t j = 0; j < limitI; j++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
         {
             std::vector<TwistBipoint> currRow;
-            for (size_t k = 0; k < limitI; k++)
+            for (size_t k = 0; k < sizeOfVectorI.val(); k++)
             {
                 TwistBipoint currCommit;
                 file >> currCommit;
+
                 currRow.push_back(currCommit);    
             }
             currMatrix.push_back(currRow);
@@ -388,17 +365,18 @@ bool read_epoch_update_string(
         partwayVoteMatrixCommits.push_back(currMatrix);
     }
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+    file >> sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
         std::vector<std::vector<TwistBipoint>> currMatrix;
-        for (size_t j = 0; j < limitI; j++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
         {
             std::vector<TwistBipoint> currRow;
-            for (size_t k = 0; k < limitI; k++)
+            for (size_t k = 0; k < sizeOfVectorI.val(); k++)
             {
                 TwistBipoint currCommit;
                 file >> currCommit;
+
                 currRow.push_back(currCommit);    
             }
             currMatrix.push_back(currRow);
@@ -406,40 +384,43 @@ bool read_epoch_update_string(
         finalVoteMatrixCommits.push_back(currMatrix);
     }
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+    file >> sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
         std::vector<Twistpoint> currRow;
-        for (size_t j = 0; j < limitI; j++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
         {
             Twistpoint currCommit;
             file >> currCommit;
+
             currRow.push_back(currCommit);
         }
         userTallyMaskCommits.push_back(currRow);
     }
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+    file >> sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
         std::vector<Twistpoint> currRow;
-        for (size_t j = 0; j < limitI; j++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
         {
             Twistpoint currCommit;
             file >> currCommit;
+
             currRow.push_back(currCommit);
         }
         userTallyMessageCommits.push_back(currRow);
     }
 
-    file >> limitI;
-    for (size_t i = 0; i < limitI; i++)
+    file >> sizeOfVectorI;
+    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
     {
         std::vector<Twistpoint> currRow;
-        for (size_t j = 0; j < limitI; j++)
+        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
         {
             Twistpoint currCommit;
             file >> currCommit;
+
             currRow.push_back(currCommit);
         }
         userTallySeedCommits.push_back(currRow);
@@ -447,16 +428,16 @@ bool read_epoch_update_string(
 
     file >> nextGenerator;
     
-    bool doUserTallies;
+    BinaryBool doUserTallies;
     file >> doUserTallies;
 
-    return doUserTallies;
+    return doUserTallies.val();
 }
 
 void distribute_epoch_updates(
     const std::string& recipient,
     const std::string& data,
-    const struct synchronization_tool* synch)
+    const struct synchronization_tool* sync)
 {
     bool flag = false;
     while (!flag)
@@ -472,7 +453,7 @@ void distribute_epoch_updates(
                 "null",
                 synchro_websocket_data_handler,
                 empty_websocket_close_handler,
-                (void *) synch);
+                (void *) sync);
 
         if (!conn)
         {
@@ -492,6 +473,8 @@ void distribute_epoch_updates(
             "",
             0);
 
+        mg_close_connection(conn);
+
         flag = true;
     }   
 }
@@ -503,7 +486,7 @@ void distribute_epoch_updates(
 PrsonaServerWebSocketHandler::PrsonaServerWebSocketHandler(
     const PrsonaServer *prsonaServer,
     const std::mutex *updateMtx,
-    const size_t *epochNum,
+    const std::atomic<size_t> *epochNum,
     const std::vector<std::string> &serverIPs,
     const std::string &selfIP)
 : prsonaServer(prsonaServer), updateMtx(updateMtx), epochNum(epochNum),
@@ -523,20 +506,6 @@ virtual bool PrsonaServerWebSocketHandler::handleConnection(
     return flag;
 }
 
-void PrsonaServerWebSocketHandler::set_temp_filename(
-    struct mg_connection *conn) const
-{
-    std::string filename = random_string(TMP_FILE_SIZE);
-    
-    char *c_filename = new char[TMP_FILE_SIZE+TMP_DIR_SIZE+1];
-    strncpy(c_filename, TMP_DIR, TMP_DIR_SIZE);
-    for (size_t i = 0; i < TMP_FILE_SIZE; i++)
-        c_filename[i + TMP_DIR_SIZE] = filename[i];
-    c_filename[TMP_DIR_SIZE + TMP_FILE_SIZE] = 0;
-
-    mg_set_user_connection_data(conn, c_filename);
-}
-
 virtual void PrsonaServerWebSocketHandler::handleReadyState(
     CivetServer *server,
     struct mg_connection *conn)
@@ -757,7 +726,7 @@ void PrsonaServerWebSocketHandler::get_num_clients(
     std::stringstream buffer;
     std::string data;
 
-    size_t numClients = prsonaServer->get_num_clients();
+    BinarySizeT numClients(prsonaServer->get_num_clients());
     buffer << numClients;
     data = buffer.str();
 
@@ -770,7 +739,7 @@ void PrsonaServerWebSocketHandler::get_num_servers(
     std::stringstream buffer;
     std::string data;
 
-    size_t numServers = prsonaServer->get_num_servers();
+    BinarySizeT numServers(prsonaServer->get_num_servers());
     buffer << numServers;
     data = buffer.str();
 
@@ -797,8 +766,10 @@ void PrsonaServerWebSocketHandler::get_current_votes_by(
         prsonaServer->get_current_votes_by(pi, shortTermPublicKey);
     
     buffer << pi;
-    buffer << votes.size();
-    for (size_t i = 0; i < votes.size(); i++)
+    
+    BinarySizeT sizeOfVector(votes.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << votes[i];
 
     data = buffer.str();
@@ -817,9 +788,11 @@ void PrsonaServerWebSocketHandler::get_all_current_votes(
         prsonaServer->get_all_current_votes(pi);
     
     buffer << pi;
-    buffer << votes.size();
-    for (size_t i = 0; i < votes.size(); i++)
-        for (size_t j = 0; j < votes[i].size(); j++)
+    
+    BinarySizeT sizeOfVector(votes.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
+        for (size_t j = 0; j < sizeOfVector.val(); j++)
             buffer << votes[i][j];
 
     data = buffer.str();
@@ -884,8 +857,10 @@ void PrsonaServerWebSocketHandler::get_current_pseudonyms(
         prsonaServer->get_current_pseudonyms(pi);
     
     buffer << pi;
-    buffer << pseudonyms.size();
-    for (size_t i = 0; i < pseudonyms.size(); i++)
+
+    BinarySizeT sizeOfVector(pseudonyms.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << pseudonyms[i];
 
     data = buffer.str();
@@ -1001,27 +976,33 @@ void PrsonaServerWebSocketHandler::distribute_new_user_updates(
 {
     std::stringstream buffer;
     std::string data;
+    BinarySizeT sizeOfVector;
 
-    buffer << proofOfValidAddition.size();
-    for (size_t i = 0; i < proofOfValidAddition.size(); i++)
+    sizeOfVector.set(proofOfValidAddition.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << proofOfValidAddition[i];
 
-    buffer << previousVoteTallies.size();
-    for (size_t i = 0; i < previousVoteTallies.size(); i++)
+    sizeOfVector.set(previousVoteTallies.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << previousVoteTallies[i];
 
-    buffer << currentPseudonyms.size();
-    for (size_t i = 0; i < currentPseudonyms.size(); i++)
+    sizeOfVector.set(currentPseudonyms.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << currentPseudonyms[i];
 
-    buffer << currentUserEncryptedTallies.size();
-    for (size_t i = 0; i < currentUserEncryptedTallies.size(); i++)
+    sizeOfVector.set(currentUserEncryptedTallies.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << currentUserEncryptedTallies[i];
 
-    buffer << voteMatrix.size();
-    for (size_t i = 0; i < voteMatrix.size(); i++)
-        for (size_t j = 0; j < voteMatrix[i].size(); j++)
-        buffer << voteMatrix[i][j];
+    sizeOfVector.set(voteMatrix.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
+        for (size_t j = 0; j < sizeOfVector.val(); j++)
+            buffer << voteMatrix[i][j];
 
     data = buffer.str();
 
@@ -1042,7 +1023,7 @@ void PrsonaServerWebSocketHandler::distribute_new_user_updates(
                 "null",
                 synchro_websocket_data_handler,
                 synchro_websocket_close_handler,
-                (void *) distributeSynch);
+                (void *) distributeSync);
 
         if (!conn)
         {
@@ -1050,19 +1031,21 @@ void PrsonaServerWebSocketHandler::distribute_new_user_updates(
             continue;
         }
 
-        unique_lock<mutex> synchLock(distributeSynch->mtx);
-        distributeSynch->isReady = false;
-        distributeSynch->isOngoing = true;
+        unique_lock<mutex> syncLock(distributeSync->mtx);
+        distributeSync->val = 0;
+        distributeSync->val2 = 0;
         mg_websocket_client_write(
             conn,
             MG_WEBSOCKET_OPCODE_BINARY,
             data.c_str(),
             data.length());
 
-        while (distributeSynch->isOngoing)
-            distributeSynch->cv.wait(synchLock);
+        while (!distributeSync->val2)
+            distributeSync->cv.wait(syncLock);
+
+        mg_close_connection(conn);
 
-        if (distributeSynch->isReady)
+        if (distributeSync->val)
             i++;
     }
 }
@@ -1074,17 +1057,22 @@ void PrsonaServerWebSocketHandler::distribute_new_vote(
 {
     std::stringstream buffer;
     std::string data;
+    BinarySizeT sizeOfVector;
 
-    buffer << pi.size();
-    for (size_t i = 0; i < pi.size(); i++)
+    sizeOfVector.set(pi.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << pi[i];
 
-    buffer << newVotes.size();
-    for (size_t i = 0; i < newVotes.size(); i++)
+    sizeOfVector.set(newVotes.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << newVotes[i];
 
     buffer << shortTermPublicKey;
-    buffer << false;
+
+    BinaryBool flag(false);
+    buffer << flag;
 
     data = buffer.str();
 
@@ -1105,7 +1093,7 @@ void PrsonaServerWebSocketHandler::distribute_new_vote(
                 "null",
                 synchro_websocket_data_handler,
                 synchro_websocket_close_handler,
-                (void *) distributeSynch);
+                (void *) distributeSync);
 
         if (!conn)
         {
@@ -1113,19 +1101,21 @@ void PrsonaServerWebSocketHandler::distribute_new_vote(
             continue;
         }
 
-        unique_lock<mutex> synchLock(distributeSynch->mtx);
-        distributeSynch->isReady = false;
-        distributeSynch->isOngoing = true;
+        unique_lock<mutex> syncLock(distributeSync->mtx);
+        distributeSync->val = 0;
+        distributeSync->val2 = 0;
         mg_websocket_client_write(
             conn,
             MG_WEBSOCKET_OPCODE_BINARY,
             data.c_str(),
             data.length());
 
-        while (distributeSynch->isOngoing)
-            distributeSynch->cv.wait(synchLock);
+        while (!distributeSync->val2)
+            distributeSync->cv.wait(syncLock);
+
+        mg_close_connection(conn);
 
-        if (distributeSynch->isReady)
+        if (distributeSync->val)
             i++;
     }
 }
@@ -1150,7 +1140,7 @@ void PrsonaServerWebSocketHandler::add_new_client(
         lck,
         serverIPs,
         selfIP,
-        &updateSynch);
+        &updateSync);
 
     std::vector<Proof> proofOfValidAddition;
     prsonaServer->add_new_client(
@@ -1179,10 +1169,11 @@ void PrsonaServerWebSocketHandler::add_new_client(
         lck,
         serverIPs,
         selfIP,
-        &updateSynch);
+        &updateSync);
 
-    buffer << proofOfValidAddition.size();
-    for (size_t i = 0; i < proofOfValidAddition.size(); i++)
+    BinarySizeT sizeOfVector(proofOfValidAddition.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << proofOfValidAddition[i];
 
     data = buffer.str();
@@ -1195,11 +1186,11 @@ void PrsonaServerWebSocketHandler::receive_vote(
 {
     ifstream file(filename);
 
-    size_t sizeOfPi;
-    file >> sizeOfPi;
+    BinarySizeT sizeOfVector;
+    file >> sizeOfVector;
 
     std::vector<Proof> pi;
-    for (size_t i = 0; i < sizeOfPi; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         Proof currProof;
         file >> currProof;
@@ -1207,11 +1198,10 @@ void PrsonaServerWebSocketHandler::receive_vote(
         pi.push_back(currProof);
     }
 
-    size_t sizeOfVotes;
-    file >> sizeOfVotes;
+    file >> sizeOfVector;
 
     std::vector<TwistBipoint> newVotes;
-    for (size_t i = 0; i < sizeOfVotes; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         TwistBipoint currVote;
         file >> currVote;
@@ -1222,17 +1212,17 @@ void PrsonaServerWebSocketHandler::receive_vote(
     Twistpoint shortTermPublicKey;
     file >> shortTermPublicKey;
 
-    bool shouldDeal;
+    BinaryBool shouldDeal;
     file >> shouldDeal;
 
     std::unique_lock<std::mutex> lck(*updateMtx, std::defer_lock);
-    if (shouldDeal)
+    if (shouldDeal.val())
     {
         obtain_update_locks(
             lck,
             serverIPs,
             selfIP,
-            &distributeSynch);
+            &distributeSync);
     }
 
     prsonaServer->receive_vote(
@@ -1240,14 +1230,14 @@ void PrsonaServerWebSocketHandler::receive_vote(
         newVotes,
         shortTermPublicKey);
 
-    if (shouldDeal)
+    if (shouldDeal.val())
     {
         distribute_new_vote(pi, newVotes, shortTermPublicKey);
         release_update_locks(
             lck,
             serverIPs,
             selfIP,
-            &distributeSynch);
+            &distributeSync);
     }
 
     mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
@@ -1307,14 +1297,15 @@ void PrsonaServerWebSocketHandler::set_generator(
 {
     ifstream file(filename);
 
-    size_t sizeOfPi;
-    file >> sizeOfPi;
+    BinarySizeT sizeOfVector;
+    file >> sizeOfVector;
 
     std::vector<Proof> pi;
-    for (size_t i = 0; i < sizeOfPi; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         Proof currProof;
         file >> currProof;
+
         pi.push_back(currProof);
     }
 
@@ -1475,7 +1466,7 @@ void PrsonaServerWebSocketHandler::break_down_midway_pseudonyms(
     while (epochSync.val < serverIPs.size())
         epochSync.cv.wait(lck);
 
-    (*epochNum)++;
+    epochNum->fetch_add(1);
 
     mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
 }
@@ -1541,53 +1532,53 @@ void PrsonaServerWebSocketHandler::import_new_user_update(
 
     ifstream file(filename);
 
-    buffer << voteMatrix.size();
-    for (size_t i = 0; i < voteMatrix.size(); i++)
-        for (size_t j = 0; j < voteMatrix[i].size(); j++)
-        buffer << voteMatrix[i][j];
-
-    size_t sizeOfVector;
+    BinarySizeT sizeOfVector;
     
     file >> sizeOfVector;
-    for (size_t i = 0; i < sizeOfVector; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         Proof currProof;
         file >> currProof;
+
         proofOfValidAddition.push_back(currProof);
     }
 
     file >> sizeOfVector;
-    for (size_t i = 0; i < sizeOfVector; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         CurveBipoint currTally;
         file >> currTally;
+
         previousVoteTallies.push_back(currTally);
     }
 
     file >> sizeOfVector;
-    for (size_t i = 0; i < sizeOfVector; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         Twistpoint currNym;
         file >> currNym;
+
         currentPseudonyms.push_back(currNym);
     }
 
     file >> sizeOfVector;
-    for (size_t i = 0; i < sizeOfVector; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         EGCiphertext currTally;
         file >> currTally;
+
         currentUserEncryptedTallies.push_back(currTally);
     }
 
     file >> sizeOfVector;
-    for (size_t i = 0; i < sizeOfVector; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         std::vector<TwistBipoint> currRow;
-        for (size_t j = 0; j < sizeOfVector; j++)
+        for (size_t j = 0; j < sizeOfVector.val(); j++)
         {
             TwistBipoint currVote;
             file >> currVote;
+
             currRow.push_back(currVote);
         }
         voteMatrix.push_back(currRow);
@@ -1615,19 +1606,21 @@ void PrsonaServerWebSocketHandler::receive_tallied_scores(
     std::vector<EGCiphertext> userScores;
     std::vector<CurveBipoint> serverScores;
 
-    size_t sizeOfVector;
+    BinarySizeT sizeOfVector;
     file >> sizeOfVector;
-    for (size_t i = 0; i < sizeOfVector; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         EGCiphertext currScore;
         file >> currScore;
+
         userScores.push_back(currScore);
     }
 
-    for (size_t i = 0; i < sizeOfVector; i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         CurveBipoint currScore;
         file >> currScore;
+
         serverScores.push_back(currScore);
     }
 
@@ -1649,8 +1642,9 @@ void PrsonaServerWebSocketHandler::get_generator(
     std::stringstream buffer;
     std::string data;
 
-    buffer << pi.size();
-    for (size_t i = 0; i < pi.size(); i++)
+    BinarySizeT sizeOfVector(pi.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << pi[i];
 
     buffer << generator;

prsona/src/networking.cpp

@@ -0,0 +1,151 @@
+#include <algorithm>
+
+std::string random_string(size_t length)
+{
+    auto randchar = []() -> char
+    {
+        const char charset[] =
+        "0123456789_-"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "abcdefghijklmnopqrstuvwxyz";
+        const size_t max_index = (sizeof(charset) - 1);
+        return charset[ rand() % max_index ];
+    };
+    std::string str(length,0);
+    std::generate_n(str.begin(), length, randchar);
+    return str;
+}
+
+char *set_temp_filename(
+    struct mg_connection *conn) const
+{
+    std::string filename = random_string(TMP_FILE_SIZE);
+    
+    char *c_filename = new char[TMP_FILE_SIZE+TMP_DIR_SIZE+1];
+    strncpy(c_filename, TMP_DIR, TMP_DIR_SIZE);
+    for (size_t i = 0; i < TMP_FILE_SIZE; i++)
+        c_filename[i + TMP_DIR_SIZE] = filename[i];
+    c_filename[TMP_DIR_SIZE + TMP_FILE_SIZE] = 0;
+
+    mg_set_user_connection_data(conn, c_filename);
+
+    return c_filename;
+}
+
+bool RemoteControlHandler::handleGet(CivetServer *server, struct mg_connection *conn)
+{
+    unique_lock<mutex> lck(sync->mtx);
+
+    mg_printf(conn,
+              "HTTP/1.1 200 OK\r\nContent-Type: "
+              "text/plain\r\nConnection: close\r\n\r\n");
+    if (message.empty())
+        mg_printf(conn, "Event triggered.\n");
+    else
+        mg_printf(conn, "%s\n", message.c_str());
+
+    sync->val++;
+    sync->cv.notify_all();
+
+    return true;
+}
+
+bool AltRemoteControlHandler::handleGet(CivetServer *server, struct mg_connection *conn)
+{
+    unique_lock<mutex> lck(sync->mtx);
+
+    const struct mg_request_info *info = mg_get_request_info(conn);
+    if (info->query_string)
+        query = info->query_string;
+
+    mg_printf(conn,
+              "HTTP/1.1 200 OK\r\nContent-Type: "
+              "text/plain\r\nConnection: close\r\n\r\n");
+    if (message.empty())
+        mg_printf(conn, "Event triggered.\n");
+    else
+        mg_printf(conn, "%s\n", message.c_str());
+
+    sync->val2 = value;
+    sync->cv.notify_all();
+
+    return true;
+}
+
+std::string AltRemoteControlHandler::query() const
+{
+    return query;
+}
+
+static int synchro_websocket_data_handler(
+    struct mg_connection *conn,
+    int bits,
+    char *data,
+    size_t data_len,
+    void *user_data)
+{
+    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE)
+        return false;
+
+    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_DATACOMPLETE)
+    {
+        struct synchronization_tool *synch = (struct synchronization_tool *) user_data;
+
+        unique_lock<mutex> lck(synch->mtx);
+        synch->val++;
+
+        return false;
+    }
+
+    std::cerr << "Unknown response when trying to get update lock." << std::endl;
+    return false;    
+}
+
+static void synchro_websocket_close_handler(
+    const struct mg_connection *conn,
+    void *user_data)
+{
+    struct synchronization_tool *synch = (struct synchronization_tool *) user_data;
+
+    unique_lock<mutex> lck(synch->mtx);
+    synch->val2 = 1;
+    synch->cv.notify_all();
+}
+
+static int file_websocket_data_handler(
+    struct mg_connection *conn,
+    int bits,
+    char *data,
+    size_t data_len,
+    void *user_data)
+{
+    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE || (bits & 0xf) == MG_WEBSOCKET_OPCODE_DATACOMPLETE)
+        return false;
+
+    if ((bits & 0xf) != MG_WEBSOCKET_OPCODE_BINARY && (bits & 0xf) != MG_WEBSOCKET_OPCODE_CONTINUATION)
+    {
+        std::cerr << "Unknown opcode: failing." << std::endl;
+        return false;
+    }
+
+    struct synchronization_tool *sync = (struct synchronization_tool *) user_data;
+    char *filename = (char *) mg_get_user_connection_data(conn);
+
+    std::unique_lock<std::mutex> lck(sync->mtx);
+    FILE *currFile = fopen(filename, "ab");
+    fwrite(data, sizeof(char), data_len, currFile);
+    fclose(currFile);
+
+    return true;
+}
+
+static void file_websocket_close_handler(
+    const struct mg_connection *conn,
+    void *user_data)
+{
+    struct synchronization_tool *sync = (struct synchronization_tool *) user_data;
+
+    unique_lock<mutex> lck(sync->mtx);
+    sync->val = 1;
+    sync->cv.notify_all();
+}

prsona/src/serverMain.cpp

@@ -1,5 +1,4 @@
-#include <mutex>
-#include <condition_variable>
+#include <atomic>
 #include <chrono>
 #include <iostream>
 #include <fstream>
@@ -10,30 +9,16 @@
 
 #include "networkServer.hpp"
 
-#define EXIT_URI "/exit"
-#define BEGIN_URI "/begin"
-#define READY_URI "/ready"
-#define EPOCH_URI "/epoch"
-#define BGN_URI "/ws?p"
-#define GET_FRESH_GEN_URI "/ws?q"
-#define GIVE_FRESH_GEN_URI "/ws?r"
-#define GET_BLIND_GEN_URI "/ws?s"
-#define GIVE_BLIND_GEN_URI "/ws?t"
-#define EPOCH_BUILD_UP_URI "/ws?u"
-#define EPOCH_BREAK_DOWN_URI "/ws?v"
-#define GIVE_BLIND_GEN_URI "/ws?t"
-#define GET_DECRYPTION_URI "/ws?y"
-#define GIVE_DECRYPTION_URI "/ws?z"
-#define BGN_TMP_FILE "~/tmp/bgn"
-#define GEN_TMP_FILE "~/tmp/generator"
-#define EPOCH_GEN_TMP_FILE "~/tmp/epoch"
+#define BGN_TMP_FILE (TMP_DIR "bgn")
+#define GEN_TMP_FILE (TMP_DIR "generator")
+#define EPOCH_GEN_TMP_FILE (TMP_DIR "epoch")
 
 using namespace std;
 
 struct synchronization_tool exitSync, bgnSync, generatorSync, readySync, updateSync, epochSync, tallySync;
 mutex updateMtx;
 
-size_t epochNum = 0;
+atomic<size_t> epochNum(0);
 
 // Initialize the classes we use
 void initialize_prsona_classes()
@@ -104,14 +89,15 @@ Twistpoint update_data_from_epoch_gen_file(vector<Proof>& pi)
     ifstream epochFile(EPOCH_GEN_TMP_FILE);
     
     Twistpoint retval;
-    size_t piSize;
+    BinarySizeT sizeOfVector;
     pi.clear();
 
-    epochFile >> piSize;
-    for (size_t i = 0; i < piSize; i++)
+    epochFile >> sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
     {
         Proof currProof;
         epochFile >> currProof;
+        
         pi.push_back(currProof);
     }
     epochFile >> retval;
@@ -293,6 +279,8 @@ Twistpoint get_generator(
         while (!generatorSync.val)
             generatorSync.cv.wait(lck);
 
+        mg_close_connection(conn);
+
         retval = update_generator_from_gen_file(currProof);
         pi.push_back(currProof);
     }
@@ -316,8 +304,9 @@ void handout_generator(
     stringstream buffer;
     string data;
     
-    buffer << pi.size();
-    for (size_t i = 0; i < pi.size(); i++)
+    BinarySizeT sizeOfVector(pi.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << pi[i];
 
     buffer << generator;
@@ -364,6 +353,8 @@ void handout_generator(
             MG_WEBSOCKET_OPCODE_DATACOMPLETE,
             "",
             0);
+
+        mg_close_connection(conn);
     }
 }
 
@@ -426,6 +417,8 @@ Twistpoint initiate_epoch_updates(
 
         if (!epochSync.val)
             flag = true;
+
+        mg_close_connection(conn);
     }
 
     if (isBreakdown)
@@ -438,8 +431,6 @@ Twistpoint initiate_epoch_updates(
 
     generatorProofHolder.push_back(generatorProof);
 
-
-
     return retval;
 }
 
@@ -613,8 +604,6 @@ void epoch_break_down(
             unique_lock<mutex> lck(epochSync.mtx);
             while (epochSync.val < serverIPs.size())
                 epochSync.cv.wait(lck);
-
-            epochNum++;
         }
         else
         {
@@ -677,6 +666,8 @@ void tally_scores(
                     MG_WEBSOCKET_OPCODE_DATACOMPLETE,
                     "",
                     0);
+
+                mg_close_connection(conn);
             }
         }
     }
@@ -733,10 +724,11 @@ void distribute_tallied_scores(
     stringstream buffer;
     string data;
     
-    buffer << userTallyScores.size();
-    for (size_t i = 0; i < userTallyScores.size(); i++)
+    BinarySizeT sizeOfVector(userTallyScores.size());
+    buffer << sizeOfVector;
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << userTallyScores[i];
-    for (size_t i = 0; i < serverTallyScores.size(); i++)
+    for (size_t i = 0; i < sizeOfVector.val(); i++)
         buffer << serverTallyScores[i];
 
     data = buffer.str();
@@ -786,6 +778,8 @@ void distribute_tallied_scores(
                     MG_WEBSOCKET_OPCODE_DATACOMPLETE,
                     "",
                     0);
+
+                mg_close_connection(conn);
             }
         }
     }
@@ -842,6 +836,8 @@ void epoch(
         generatorProof,
         nextGenerator);
 
+    epochNum.fetch_add(1);
+
     release_update_locks(
         lck,
         serverIPs,
@@ -849,64 +845,48 @@ void epoch(
         updateSync);
 }
 
-class ExitHandler : public CivetHandler
+class EpochReadyHandler : public CivetHandler
 {
     public:
-        bool handleGet(CivetServer *server, struct mg_connection *conn)
-        {
-            unique_lock<mutex> lck(exitSync.mtx);
-
-            mg_printf(conn,
-                      "HTTP/1.1 200 OK\r\nContent-Type: "
-                      "text/plain\r\nConnection: close\r\n\r\n");
-            mg_printf(conn, "Server coming down!\n");
+        EpochReadyHandler(size_t numServers)
+            : numServers(numServers) { /* */ }
 
-            exitSync.val = 1;
-            exitSync.cv.notify_all();
-
-            return true;
-        }
-};
-
-class BeginHandler : public CivetHandler
-{
-    public:
         bool handleGet(CivetServer *server, struct mg_connection *conn)
         {
-            unique_lock<mutex> lck(exitSync.mtx);
-
-            mg_printf(conn,
-                      "HTTP/1.1 200 OK\r\nContent-Type: "
-                      "text/plain\r\nConnection: close\r\n\r\n");
-            mg_printf(conn, "Server will begin periodically initiating epochs!\n");
+            unique_lock<mutex> lck(exitSync.mtx, defer_lock);
+            unique_lock<mutex> lck(readySync.mtx);
 
-            exitSync.val2 = 1;
-            exitSync.cv.notify_all();
+            if (readySync.val < numServers)
+            {
+                mg_printf(conn,
+                          "HTTP/1.1 503 Service Unavailable\r\nContent-Type: "
+                          "text/plain\r\nConnection: close\r\n\r\n");
+                mg_printf(conn, "Server is waiting for other servers to begin.\n");
+            }
+            else if (lck.try_lock())
+            {
+                mg_printf(conn,
+                          "HTTP/1.1 200 OK\r\nContent-Type: "
+                          "text/plain\r\nConnection: close\r\n\r\n");
+                mg_printf(conn, "Server is ready for epoch.\n");
+            }
+            else
+            {
+                mg_printf(conn,
+                          "HTTP/1.1 503 Service Unavailable\r\nContent-Type: "
+                          "text/plain\r\nConnection: close\r\n\r\n");
+                mg_printf(conn, "Server is still in a previous epoch.\n");   
+            }
 
             return true;
         }
-};
-
-class ReadyHandler : public CivetHandler
-{
-    public:
-        bool handleGet(CivetServer *server, struct mg_connection *conn)
-        {
-            unique_lock<mutex> lck(readySync.mtx);
 
-            mg_printf(conn,
-                      "HTTP/1.1 200 OK\r\nContent-Type: "
-                      "text/plain\r\nConnection: close\r\n\r\n");
-            mg_printf(conn, "ACK\n");
+    private:
+        const size_t numServers;
 
-            readySync.val++;
-            readySync.cv.notify_all();
-
-            return true;
-        }
 };
 
-class EpochHandler : public CivetHandler
+class EpochNumHandler : public CivetHandler
 {
     public:
         bool handleGet(CivetServer *server, struct mg_connection *conn)
@@ -914,7 +894,7 @@ class EpochHandler : public CivetHandler
             mg_printf(conn,
                       "HTTP/1.1 200 OK\r\nContent-Type: "
                       "text/plain\r\nConnection: close\r\n\r\n");
-            mg_printf(conn, "%lu\n", epochNum);
+            mg_printf(conn, "Epoch num: %lu\n", epochNum.load());
 
             return true;
         }
@@ -963,28 +943,17 @@ int main(int argc, char *argv[])
     // Defaults
     size_t numServers = serverIPs.size();
     bool bgnDealer = selfIP == dealerIP;
-    size_t timeToEpoch = 600;
     bool maliciousServers = true;
 
     if (argc > 1)
-        timeToEpoch = atoi(argv[1]);
-    if (argc > 2)
     {
-        bool setting = argv[2][0] == 't' || argv[2][0] == 'T';
+        bool setting = argv[1][0] == 't' || argv[1][0] == 'T';
         maliciousServers = setting;
     }
 
     cout << "Establishing PRSONA server with the following parameters: " << endl;
     cout << numServers << " PRSONA servers" << endl;
-    if (bgnDealer)
-    {
-        cout << "This server IS the trusted BGN dealer" << endl;
-        cout << "The time between epochs is " << timeToEpoch << " seconds" << endl;
-    }
-    else
-    {
-        cout << "This server is NOT the trusted BGN dealer" << endl;
-    }
+    cout << "This server " << (bgnDealer ? "IS" : "is NOT") << "the trusted BGN dealer" << endl;
     cout << "Servers are set to " << (maliciousServers ? "MALICIOUS" : "HBC") << " security" << endl;
     cout << "This server is at IP address: " << selfIP << endl;
     cout << "The BGN dealer is at IP address: " << dealerIP << endl;
@@ -994,12 +963,16 @@ int main(int argc, char *argv[])
     if (maliciousServers)
         PrsonaBase::set_server_malicious();
 
+    cout << "Creating PrsonaServer entity." << endl;
+
     // Entities we operate with
     PrsonaServer *prsonaServer;
     if (bgnDealer)
         prsonaServer = new PrsonaServer(numServers);
     else
     {
+        cout << "Retrieving BGN details." << endl;
+
         struct mg_connection *conn =
             mg_connect_websocket_client(
                 dealerIP.c_str(),
@@ -1015,7 +988,7 @@ int main(int argc, char *argv[])
 
         if (!conn)
         {
-            cerr << "Couldn't obtain BGN details" << endl;
+            cerr << "Couldn't obtain BGN details." << endl;
             return 1;
         }
 
@@ -1031,6 +1004,8 @@ int main(int argc, char *argv[])
         while (!bgnSync.val)
             bgnSync.cv.wait(lck);
 
+        mg_close_connection(conn);
+
         prsonaServer = create_server_from_bgn_file(numServers);
     }
 
@@ -1039,12 +1014,14 @@ int main(int argc, char *argv[])
     PrsonaServerWebSocketHandler wsHandler(prsonaServer, &updateMtx, &epochNum, serverIPs, selfIP);
     server.addWebSocketHandler("/ws", wsHandler);
 
-    ReadyHandler readyHandler;
-    server.addHandler(READY_URI, readyHandler);
-
     if (bgnDealer)
     {
+        cout << "Waiting for other servers to check in and retrieve BGN details." << endl;
+
         unique_lock<mutex> lck(readySync.mtx);
+        RemoteControlHandler serverReadyHandler(&readySync, "ACK");
+        server.addHandler(SERVER_READY_URI, serverReadyHandler);
+
         readySync.val++;
         while (readySync.val < numServers)
             readySync.cv.wait(lck);
@@ -1062,6 +1039,8 @@ int main(int argc, char *argv[])
     }
     else
     {
+        cout << "Notifying BGN dealer that this server is ready." << endl;
+
         stringstream sysString;
         string data;
 
@@ -1073,7 +1052,7 @@ int main(int argc, char *argv[])
                 NULL,
                 0);
 
-        sysString << "GET " << READY_URI << " HTTP/1.1\r\n";
+        sysString << "GET " << SERVER_READY_URI << " HTTP/1.1\r\n";
         sysString << "Host: " << dealerIP << "\r\n\r\n";
         data = sysString.str();
 
@@ -1081,61 +1060,50 @@ int main(int argc, char *argv[])
         mg_close_connection(conn);
     }
 
-    ExitHandler exitHandler;
+    unique_lock<mutex> exitLock(exitSync.mtx);
+    exitSync.val = 0;
+    exitSync.val2 = 0;
+    RemoteControlHandler exitHandler(&exitSync, "Server coming down!");
     server.addHandler(EXIT_URI, exitHandler);
 
+    cout << "Entering main ready loop." << endl;
+
     if (bgnDealer)
     {
-        BeginHandler beginHandler;
-        server.addHandler(BEGIN_URI, beginHandler);
+        AltRemoteControlHandler triggerEpochHandler(1, &exitSync, "Server will initiate epoch!");
+        server.addHandler(TRIGGER_EPOCH_URI, triggerEpochHandler);
 
-        unique_lock<mutex> lck(exitSync.mtx);
-        while (!exitSync.val && !exitSync.val2)
-            exitSync.cv.wait(lck);
+        EpochReadyHandler epochReadyHandler(numServers);
+        server.addHandler(EPOCH_READY_URI, epochReadyHandler);
+
+        EpochNumHandler epochNumHandler;
+        server.addHandler(WHICH_EPOCH_URI, epochNumHandler);
 
         while (!exitSync.val)
         {
-            epoch(prsonaServer);
-            exitSync.cv.waitFor(lck, chrono::seconds(timeToEpoch));
+            while (!exitSync.val && !exitSync.val2)
+                exitSync.cv.wait(exitLock);
+
+            if (exitSync.val2)
+            {
+                cout << "Executing epoch." << endl;
+
+                epoch(prsonaServer);
+
+                exitSync.val2 = 0;
+            }
         }
     }
     else
     {
-        unique_lock<mutex> lck(exitSync.mtx);
         while (!exitSync.val)
-            exitSync.cv.wait(lck)
+            exitSync.cv.wait(exitLock)
     }
 
-    mg_exit_library();
+    cout << "Shutting down." << endl;
 
+    mg_exit_library();
     delete prsonaServer;
 
-    if (bgnDealer)
-    {
-        for (size_t i = 0; i < serverIPs.size(); i++)
-        {
-            if (serverIPs[i] == selfIP)
-                continue;
-
-            stringstream sysString;
-            string data;
-
-            struct mg_connection *conn =
-                mg_connect_client(
-                    serverIPs[i], 
-                    PRSONA_PORT,
-                    USE_SSL, 
-                    NULL,
-                    0);
-
-            sysString << "GET " << EXIT_URI << " HTTP/1.1\r\n";
-            sysString << "Host: " << serverIPs[i] << "\r\n\r\n";
-            data = sysString.str();
-
-            mg_write(conn, data.c_str(), data.length());
-            mg_close_connection(conn);
-        }
-    }
-
     return 0;
 }