PRSONA/prsona/src/networkServer.cpp

#include <iostream>
#include <fstream>
#include <sstream>
#include <ctime>
#include <chrono>

#include "networkServer.hpp"

/********************************************************
 *********                                      *********
 *********  server networking public functions  *********
 *********                                      *********
 ********************************************************/

/*
 * CREATOR FOR A NEW SERVER
 */

PrsonaServer *create_server(
    std::default_random_engine& rng,
    std::string dealerIP,
    int dealerPort,
    bool bgnDealer,
    size_t numServers)
{
    // We simulate the distributed BGN key generation; in our case, the dealer makes the BGN individually and shares it with other servers
    if (bgnDealer)
        return new PrsonaServer(numServers);

    // If we're not the dealer, get the BGN private key from the dealer
    BGN privateKey = get_bgn_private_key(rng, dealerIP, dealerPort);

    // And make a server object accounting for that
    return new PrsonaServer(numServers, privateKey);
}

/*
 * CHECK IN FUNCTION USED FOR SYNCHRONIZATION IN SETUP
 */

void check_in_with_dealer(
    std::string dealerIP,
    int dealerPort)
{
    std::stringstream buffer;
    std::string data;

    // The actual check in process is very simple; just make the correct GET request
    buffer << "GET " << SERVER_CHECK_IN_URI << " HTTP/1.1\r\n";
    buffer << "Host: " << dealerIP << ":" << dealerPort << "\r\n\r\n";
    data = buffer.str();

    // This also means things are slightly less cumbersome than making a websocket connection would be
    struct mg_connection *conn = mg_connect_client(dealerIP.c_str(), dealerPort, USE_SSL, NULL, 0);

    // Make the actual GET request
    mg_write(conn, data.c_str(), data.length());

    // We don't really care about the response, so we can just ignore it
    mg_close_connection(conn);
}

/*
 * INITIATER FOR SHARED GLOBAL VALUES
 */

void initiate_generators(
    std::default_random_engine& rng,
    PrsonaServer* prsonaServer,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort)
{
    // Form and distribute the first fresh generator
    std::vector<Proof> pi;
    Twistpoint freshGenerator = make_generator(rng, prsonaServer, serverIPs, serverPorts, selfIP, selfPort, true, pi);

    distribute_generator(prsonaServer, serverIPs, serverPorts, selfIP, selfPort, true, pi, freshGenerator);

    // Form and distribute the H used in ElGamal operations
    Twistpoint blindGenerator = make_generator(rng, prsonaServer, serverIPs, serverPorts, selfIP, selfPort, false, pi);

    distribute_generator(prsonaServer, serverIPs, serverPorts, selfIP, selfPort, false, pi, blindGenerator);
}

/*
 * FUNCTION TO PERFORM OPERATIONS FOR EXPERIMENT
 */

void make_epoch(
    std::default_random_engine& rng,
    PrsonaServer *prsonaServer,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort,
    std::mutex& updateMtx,
    std::atomic<size_t>& epochNum,
    const CivetServer& civetServer,
    std::mutex& buildUpOutputMtx,
    const std::string& buildUpOutputFilename,
    std::mutex& breakDownOutputMtx,
    const std::string& breakDownOutputFilename,
    std::mutex& fullOutputMtx,
    const std::string& fullOutputFilename)
{
    // As before, the fresh generator always starts from the same G
    Twistpoint nextGenerator = PrsonaServer::EL_GAMAL_GENERATOR;

    std::unique_lock<std::mutex> updateLock(updateMtx, std::defer_lock);

    std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer.getContext());
    std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
    clock_t cpuTimeBefore = clock();

    // Take update locks on every machine
    obtain_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);

    // Do the first half of the epoch calculations (building up the intermediary values)
    std::vector<Proof> generatorProof = epoch_build_up(rng, prsonaServer, serverIPs, serverPorts, selfIP, selfPort, nextGenerator, civetServer, buildUpOutputMtx, buildUpOutputFilename);

    // Tally up the current scores at the end of the epoch for the users
    std::vector<EGCiphertext> currentUserEncryptedTallies;
    std::vector<CurveBipoint> currentServerEncryptedTallies;
    tally_scores(prsonaServer, serverIPs, serverPorts, selfIP, selfPort, nextGenerator, currentUserEncryptedTallies, currentServerEncryptedTallies);

    // And distribute these to each server
    distribute_tallied_scores(prsonaServer, serverIPs, serverPorts, selfIP, selfPort, nextGenerator, currentUserEncryptedTallies, currentServerEncryptedTallies);

    // Do the second half of the epoch calculations (breaking down values to their final values, to be given to users)
    epoch_break_down(rng, prsonaServer, serverIPs, serverPorts, selfIP, selfPort, generatorProof, nextGenerator, civetServer, breakDownOutputMtx, breakDownOutputFilename);

    // Indicate we are in a new epoch
    epochNum.fetch_add(1);

    // Release the update locks from every machine
    release_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);
    
    clock_t cpuTimeAfter = clock();
    std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
    std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer.getContext());

    std::vector<double> timingData(2);
    timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
    timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;

    std::vector<size_t> bandwidthData(2);
    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0];
    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];

    write_log_data(fullOutputMtx, fullOutputFilename, timingData, bandwidthData);
}

/*********************************************************
 *********                                       *********
 *********  server networking private functions  *********
 *********                                       *********
 *********************************************************/

/*
 * SHARED GLOBAL UPDATE LOCK GETTERS AND RELEASERS
 */

void obtain_update_locks(
    std::unique_lock<std::mutex> &updateLock,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort)
{
    // Get locks on each machine (in a predetermined order, defined universally for all servers)
    size_t i = 0;
    while (i < serverIPs.size())
    {
        // When it's our turn, it's easy to take the lock
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
        {
            updateLock.lock();
            i++;
            continue;
        }

        struct synchronization_tool sync;
        struct mg_connection *conn = NULL;

        // Connect to the server
        std::unique_lock<std::mutex> lck(sync.mtx);
        sync.val = 0;
        sync.val2 = 0;
        while (!conn)
        {
            conn = mg_connect_websocket_client(serverIPs[i].c_str(), serverPorts[i], USE_SSL, NULL, 0, UPDATE_LOCK_URI, "null", synchro_websocket_data_handler, synchro_websocket_close_handler, &sync);

            if (!conn)
                std::cerr << "Couldn't connect to server " << i << " to obtain its lock" << std::endl;
        }

        // Ask for its lock
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        // Wait for its response (or the connection to die)
        while (!sync.val2)
            sync.cv.wait(lck);

        // Close connection
        mg_close_connection(conn);

        // Only move forward once we've confirmed we have the lock (or else we risk deadlock!)
        if (sync.val)
            i++;
    }
}

void release_update_locks(
    std::unique_lock<std::mutex> &updateLock,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort)
{
    // Release locks on each machine (in the opposite of the predetermined order we used to take them)
    ssize_t i = serverIPs.size() - 1;
    while (i >= 0)
    {
        // When it's our turn, it's easy to release the lock
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
        {
            updateLock.unlock();
            i--;
            continue;
        }

        struct synchronization_tool sync;
        struct mg_connection *conn = NULL;

        // Connect to the server
        std::unique_lock<std::mutex> lck(sync.mtx);
        sync.val = 0;
        sync.val2 = 0;
        while (!conn)
        {
            conn = mg_connect_websocket_client(serverIPs[i].c_str(), serverPorts[i], USE_SSL, NULL, 0, UPDATE_UNLOCK_URI, "null", synchro_websocket_data_handler, synchro_websocket_close_handler, &sync);

            if (!conn)
                std::cerr << "Couldn't connect to server " << i << " to release its lock" << std::endl;
        }

        // Return its lock
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        // Wait for its response (or the connection to die)
        while (!sync.val2)
            sync.cv.wait(lck);

        // Close connection
        mg_close_connection(conn);

        // Only move forward once we've confirmed we released the lock (or else we risk deadlock!)
        if (sync.val)
            i--;
    }
}

/*
 * GETTER FOR DEALER VALUE
 */

BGN get_bgn_private_key(
    std::default_random_engine& rng,
    std::string dealerIP,
    int dealerPort)
{
    struct synchronization_tool sync;
    char *filename;
    struct mg_connection *conn = NULL;

    // Set up connection to the dealer
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 0;
    while (!conn)
    {
        conn = mg_connect_websocket_client(dealerIP.c_str(), dealerPort, USE_SSL, NULL, 0, REQUEST_BGN_PRIVKEY_URI, "null", file_websocket_data_handler, file_websocket_close_handler, &sync);

        if (!conn)
            std::cerr << "Couldn't connect to dealer to obtain BGN details." << std::endl;
    }

    // Establish a file to receive BGN data at
    filename = set_temp_filename(rng, conn);

    // Tell dealer to go ahead with data
    mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

    // Wait for data
    while (!sync.val)
        sync.cv.wait(lck);

    // Close connection
    mg_close_connection(conn);

    // Un-serialize BGN private key
    BGN retval = get_bgn_private_key_from_file(filename);

    remove(filename);
    delete [] filename;

    return retval;
}

/*
 * HELPERS TO INITIATE SHARED GLOBAL VALUES
 */

Twistpoint make_generator(
    std::default_random_engine& rng,
    PrsonaServer *prsonaServer,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort,
    bool fresh,
    std::vector<Proof>& pi)
{
    // Either way, we always start from the same, default G
    Twistpoint retval = PrsonaServer::EL_GAMAL_GENERATOR;
    pi.clear();

    // Make sure we instruct the servers to contribute correctly (in practice, there is basically no difference between these)
    const char* which = (fresh ? REQUEST_ADD_CURR_SEED_FOR_FRESH_GENERATOR_URI : REQUEST_ADD_RAND_SEED_FOR_EG_BLIND_GENERATOR_URI);

    // Ask each server for its contribution
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        // Add our own contribution
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
        {
            if (fresh)
                retval = prsonaServer->add_curr_seed_to_generator(pi, retval);
            else
                retval = prsonaServer->add_rand_seed_to_generator(pi, retval);

            continue;
        }

        struct synchronization_tool sync;
        char *filename;
        struct mg_connection *conn = NULL;

        // Serialize current state of generator
        std::stringstream buffer;
        std::string data;
        buffer << retval;
        data = buffer.str();

        // Set up connection to current server
        std::unique_lock<std::mutex> lck(sync.mtx);
        sync.val = 0;
        while (!conn)
        {
            conn = mg_connect_websocket_client(serverIPs[i].c_str(), serverPorts[i], USE_SSL, NULL, 0, which, "null", file_websocket_data_handler, file_websocket_close_handler, &sync);

            if (!conn)
                std::cerr << "Couldn't get server " << i << "'s update on generator" << std::endl;
        }

        // Establish a file to receive the new generator at
        filename = set_temp_filename(rng, conn);

        // Send the current state of the generator
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        // Wait for response
        while (!sync.val)
            sync.cv.wait(lck);

        // Close connection
        mg_close_connection(conn);

        // Un-serialize the new state of the generator
        Proof currProof;
        retval = get_generator_from_file(filename, currProof);
        pi.push_back(currProof);

        remove(filename);
        delete [] filename;
    }

    return retval;
}

void distribute_generator(
    PrsonaServer *prsonaServer,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort,
    bool fresh,
    const std::vector<Proof>& pi,
    const Twistpoint& generator)
{
    // Serialize the final generator, and its proof of correctness
    std::stringstream buffer;
    std::string data;
    BinarySizeT sizeOfVector(pi.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << pi[i];
    buffer << generator;
    data = buffer.str();

    // Make sure we tell the servers which generator we're giving them correctly
    const char* which = (fresh ? SUBMIT_FRESH_GENERATOR_URI : SUBMIT_EG_BLIND_GENERATOR_URI);
    
    // Distribute the generator to each server
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        // Load the generator into our own server object
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
        {
            if (fresh)
                prsonaServer->initialize_fresh_generator(pi, generator);
            else
                prsonaServer->set_EG_blind_generator(pi, generator);

            continue;
        }

        struct mg_connection *conn = NULL;

        // Connect to the server
        while (!conn)
        {
            conn = mg_connect_websocket_client(serverIPs[i].c_str(), serverPorts[i], USE_SSL, NULL, 0, which, "null", empty_websocket_data_handler, empty_websocket_close_handler, NULL);

            if (!conn)
                std::cerr << "Couldn't connect to server " << i << " to give them the " << (fresh ? "fresh" : "blind") << " generator." << std::endl;
        }

        // Send the generator to the server
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        // Don't bother waiting for a response; we can just move on to the next
        mg_close_connection(conn);
    }
}

/*
 * HELPERS FOR EPOCH CALCULATIONS
 */

std::vector<Proof> epoch_build_up(
    std::default_random_engine& rng,
    PrsonaServer *prsonaServer,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort,
    Twistpoint& nextGenerator,
    const CivetServer& civetServer,
    std::mutex& outputMtx,
    const std::string& outputFilename)
{
    std::vector<std::vector<std::vector<Proof>>> pi;
    std::vector<std::vector<std::vector<Twistpoint>>> permutationCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymSeedCommits;
    std::vector<std::vector<std::vector<CurveBipoint>>> serverTallyCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> partwayVoteMatrixCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> finalVoteMatrixCommits;

    std::vector<std::vector<Proof>> generatorProofHolder(1);

    // Go through each server to perform the epoch calculation at hand
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        // When it's our turn, do things as normal
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
        {
            pi.clear();
            pi.push_back(generatorProofHolder);
            permutationCommits.clear();
            freshPseudonymCommits.clear();
            freshPseudonymSeedCommits.clear();
            serverTallyCommits.clear();
            partwayVoteMatrixCommits.clear();
            finalVoteMatrixCommits.clear();

            std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer.getContext());
            std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
            clock_t cpuTimeBefore = clock();
            
            // Perform the actual calculation
            prsonaServer->build_up_midway_pseudonyms(pi, permutationCommits, freshPseudonymCommits, freshPseudonymSeedCommits, serverTallyCommits, partwayVoteMatrixCommits, finalVoteMatrixCommits, nextGenerator);

            std::vector<std::vector<Twistpoint>> currUserTallyMaskCommits;
            std::vector<std::vector<Twistpoint>> currUserTallyMessageCommits;
            std::vector<std::vector<Twistpoint>> currUserTallySeedCommits;

            // Serialize the relevant data
            std::string data = make_epoch_update_string(pi[1], permutationCommits[0], freshPseudonymCommits[0], freshPseudonymSeedCommits[0], serverTallyCommits[0], partwayVoteMatrixCommits[0], finalVoteMatrixCommits[0], currUserTallyMaskCommits, currUserTallyMessageCommits, currUserTallySeedCommits, nextGenerator, false);

            struct synchronization_tool sync;
            std::vector<struct mg_connection *> conns;

            // Distribute the data to each server (in parallel, roughly)
            std::unique_lock<std::mutex> lck(sync.mtx);
            sync.val = 1;
            for (size_t j = 0; j < serverIPs.size(); j++)
            {
                // But, obviously, don't send it back to ourselves
                if (i == j)
                    continue;

                // Send that data
                struct mg_connection *currConn = distribute_epoch_updates(serverIPs[j], serverPorts[j], data, &sync);

                // But keep track of that connection, as we can't close it until we know the server's gotten its data
                conns.push_back(currConn);
            }

            // Wait for the other servers to all report back that they have received the update
            while (sync.val < serverIPs.size())
                sync.cv.wait(lck);

            for (size_t j = 0; j < conns.size(); j++)
                mg_close_connection(conns[j]);

            clock_t cpuTimeAfter = clock();
            std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
            std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer.getContext());

            std::vector<double> timingData(2);
            timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
            timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;

            std::vector<size_t> bandwidthData(2);
            bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0];
            bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];

            write_log_data(outputMtx, outputFilename, timingData, bandwidthData);

            // Keep an up-to-date version of the proof of the new fresh generator
            generatorProofHolder = pi[0];
        }
        else    // When it's another server's turn, tell them to do their part
        {
            // Serialize the request
            std::string data = make_epoch_initiator_string(generatorProofHolder[0], nextGenerator);

            // And have them do that request
            nextGenerator = initiate_epoch_updates(rng, serverIPs[i], serverPorts[i], data, false, generatorProofHolder);
        }
    }

    // Return the proof of the fresh generator
    return generatorProofHolder[0];
}

void epoch_break_down(
    std::default_random_engine& rng,
    PrsonaServer *prsonaServer,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort,
    const std::vector<Proof>& generatorProof,
    const Twistpoint& nextGenerator,
    const CivetServer& civetServer,
    std::mutex& outputMtx,
    const std::string& outputFilename)
{
    std::vector<std::vector<std::vector<Proof>>> pi;
    std::vector<std::vector<std::vector<Twistpoint>>> permutationCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymSeedCommits;
    std::vector<std::vector<std::vector<CurveBipoint>>> serverTallyCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> partwayVoteMatrixCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> finalVoteMatrixCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> userTallyMaskCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> userTallyMessageCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> userTallySeedCommits;

    // Go through each server to perform the epoch calculation at hand
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        // When it's our turn, do things as normal
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
        {
            pi.clear();
            permutationCommits.clear();
            freshPseudonymCommits.clear();
            freshPseudonymSeedCommits.clear();
            serverTallyCommits.clear();
            partwayVoteMatrixCommits.clear();
            finalVoteMatrixCommits.clear();
            userTallyMaskCommits.clear();
            userTallyMessageCommits.clear();
            userTallySeedCommits.clear();

            std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer.getContext());
            std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
            clock_t cpuTimeBefore = clock();

            // Perform the actual calculation
            prsonaServer->break_down_midway_pseudonyms(generatorProof, pi, permutationCommits, freshPseudonymCommits, freshPseudonymSeedCommits, serverTallyCommits, partwayVoteMatrixCommits, finalVoteMatrixCommits, userTallyMaskCommits, userTallyMessageCommits, userTallySeedCommits, nextGenerator);

            // Serialize the relevant data
            std::string data = make_epoch_update_string(pi[0], permutationCommits[0], freshPseudonymCommits[0], freshPseudonymSeedCommits[0], serverTallyCommits[0], partwayVoteMatrixCommits[0], finalVoteMatrixCommits[0], userTallyMaskCommits[0], userTallyMessageCommits[0], userTallySeedCommits[0], nextGenerator, true);

            struct synchronization_tool sync;
            std::vector<struct mg_connection *> conns;

            // Distribute the data to each server (in parallel, roughly)
            std::unique_lock<std::mutex> lck(sync.mtx);
            sync.val = 1;
            for (size_t j = 0; j < serverIPs.size(); j++)
            {
                // But, obviously, don't send it back to ourselves
                if (i == j)
                    continue;

                // Send that data
                struct mg_connection *currConn = distribute_epoch_updates(serverIPs[j], serverPorts[j], data, &sync);

                // But keep track of that connection, as we can't close it until we know the server's gotten its data
                conns.push_back(currConn);
            }

            // Wait for the other servers to all report back that they have received the update
            while (sync.val < serverIPs.size())
                sync.cv.wait(lck);

            for (size_t j = 0; j < conns.size(); j++)
                mg_close_connection(conns[j]);

            clock_t cpuTimeAfter = clock();
            std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
            std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer.getContext());

            std::vector<double> timingData(2);
            timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
            timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;

            std::vector<size_t> bandwidthData(2);
            bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0];
            bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];
        }
        else    // When it's another server's turn, tell them to do their part
        {
            std::vector<std::vector<Proof>> unused;

            // Serialize the request
            std::string data = make_epoch_initiator_string(generatorProof, nextGenerator);
            
            // And have them do that request
            initiate_epoch_updates(rng, serverIPs[i], serverPorts[i], data, true, unused);
        }
    }
}

/*
 * HELPERS FOR EPOCH HELPERS
 */

Twistpoint initiate_epoch_updates(
    std::default_random_engine& rng,
    const std::string& recipient,
    int recipientPort,
    const std::string& data,
    bool isBreakdown,
    std::vector<std::vector<Proof>>& generatorProofHolder)
{
    Twistpoint retval;
    struct synchronization_tool sync;
    char * filename = NULL;
    const char* which = (isBreakdown ? REQUEST_EPOCH_BREAK_DOWN_URI : REQUEST_EPOCH_BUILD_UP_URI);

    // Make sure we don't move on until this server has conducted its epoch calculations
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 0;
    while (!sync.val)
    {
        struct mg_connection *conn = NULL;
        
        // Connect to a server
        sync.val2 = 0;
        while (!conn)
        {
            conn = mg_connect_websocket_client(recipient.c_str(), recipientPort, USE_SSL, NULL, 0, which, "null", epoch_websocket_data_handler, epoch_websocket_close_handler, &sync);

            if (!conn)
                std::cerr << "Couldn't initiate epoch update with server at " << recipient << ":" << recipientPort << std::endl;
        }
        
        // Establish a file to receive update data (when relevant)
        if (!isBreakdown)
            filename = set_temp_filename(rng, conn);
            
        // Send the relevant data
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        // Wait for a response (or the connection to die)
        while (!sync.val2)
            sync.cv.wait(lck);

        // Close connection
        mg_close_connection(conn);
    }
    
    // In the second half of the epoch, we don't have a data response to un-serialize, so just move on
    if (isBreakdown)
        return retval;

    std::vector<Proof> generatorProof;
    generatorProofHolder.clear();

    // Otherwise, un-serialize the updated fresh generator (and its proof of correctness)
    retval = get_generator_from_file(filename, generatorProof);
    generatorProofHolder.push_back(generatorProof);

    remove(filename);
    delete [] filename;

    return retval;
}

struct mg_connection *distribute_epoch_updates(
    const std::string& recipient,
    int recipientPort,
    const std::string& data,
    struct synchronization_tool* sync)
{
    struct mg_connection *conn = NULL;

    // Connect to the server
    while (!conn)
    {
        conn = mg_connect_websocket_client(recipient.c_str(), recipientPort, USE_SSL, NULL, 0, SUBMIT_EPOCH_UPDATES_URI, "null", synchro_websocket_data_handler, synchro_websocket_close_handler, sync);

        if (!conn)
            std::cerr << "Couldn't give epoch updates to server at " << recipient << ":" << recipientPort << std::endl;
    }

    // Send the update
    mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
    mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

    return conn;
}

/*
 * SCORE TALLYING AND DISTRIBUTION HELPERS
 */

void tally_scores(
    PrsonaServer *prsonaServer,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort,
    const Twistpoint& nextGenerator,
    std::vector<EGCiphertext>& userTallyScores,
    std::vector<CurveBipoint>& serverTallyScores)
{
    struct synchronization_tool sync;
    std::vector<struct mg_connection *> conns;

    // Connect to each server (roughly in parallel)
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 1;
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        // Except, skip ourselves (obviously)
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
            continue;
        
        struct mg_connection *currConn = NULL;

        while (!currConn)
        {
            currConn = mg_connect_websocket_client(serverIPs[i].c_str(), serverPorts[i], USE_SSL, NULL, 0, REQUEST_PARTIAL_DECRYPTION_URI, "null", synchro_websocket_data_handler, synchro_websocket_close_handler, &sync);

            if (!currConn)
                std::cerr << "Trouble getting partial decryption from server at " << serverIPs[i] << ":" << serverPorts[i] << std::endl;
        }

        // Ping server for simulated distributed BGN
        mg_websocket_client_write(currConn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        conns.push_back(currConn);
    }

    // Since we're only simulating distributed BGN, we're only waiting for ACKs here
    while (sync.val < serverIPs.size())
        sync.cv.wait(lck);

    // Close connections
    for (size_t i = 0; i < conns.size(); i++)
        mg_close_connection(conns[i]);

    // Now we do the actual calculations
    std::vector<EGCiphertext> retval;
    std::vector<Twistpoint> currentPseudonyms = prsonaServer->get_current_pseudonyms();
    std::vector<Scalar> decryptedTalliedScores = prsonaServer->tally_scores();
    mpz_class maxScorePossibleThisRound = prsonaServer->get_max_possible_score().toInt() * PrsonaBase::get_max_allowed_vote();
    mpz_class topOfScoreRange = decryptedTalliedScores.size() * PrsonaBase::get_max_allowed_vote();

    userTallyScores.clear();
    serverTallyScores.clear();
    for (size_t i = 0; i < decryptedTalliedScores.size(); i++)
    {
        // Scale scores correctly to keep in the desired range
        decryptedTalliedScores[i] = Scalar((decryptedTalliedScores[i].toInt() * topOfScoreRange) / maxScorePossibleThisRound);

        EGCiphertext currCiphertext;
        userTallyScores.push_back(currCiphertext);

        CurveBipoint currServerScore;
        serverTallyScores.push_back(currServerScore);

        Scalar currMask;
        currMask.set_random();

        // Give the server the new weights for the next epoch's calculation, to get passed around to the other servers
        prsonaServer->encrypt(serverTallyScores[i], decryptedTalliedScores[i]);

        // Encrypt the scores for the partway pseudonyms (since we're in the middle of the epoch calculations)
        userTallyScores[i].mask = currentPseudonyms[i] * currMask;
        userTallyScores[i].encryptedMessage = (nextGenerator * currMask) + (prsonaServer->get_blinding_generator() * decryptedTalliedScores[i]);
    }
}

void distribute_tallied_scores(
    PrsonaServer *prsonaServer,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort,
    const Twistpoint& nextGenerator,
    const std::vector<EGCiphertext>& userTallyScores,
    const std::vector<CurveBipoint>& serverTallyScores)
{
    // Serialize scores
    std::stringstream buffer;
    std::string data;
    BinarySizeT sizeOfVector(userTallyScores.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << userTallyScores[i];
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << serverTallyScores[i];
    data = buffer.str();

    struct synchronization_tool sync;
    std::vector<struct mg_connection *> conns;

    // Connect to each server (roughly in parallel)
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 1;
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        // When it's our turn, receive our actual new scores for the next epoch
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
        {
            prsonaServer->receive_tallied_scores(userTallyScores, serverTallyScores);
            continue;
        }
        
        struct mg_connection *currConn = NULL;

        while (!currConn)
        {
            currConn = mg_connect_websocket_client(serverIPs[i].c_str(), serverPorts[i], USE_SSL, NULL, 0, SUBMIT_PARTIAL_DECRYPTION_URI, "null", synchro_websocket_data_handler, synchro_websocket_close_handler, &sync);

            if (!currConn)
                std::cerr << "Trouble giving full re-encryption to server at " << serverIPs[i] << ":" << serverPorts[i] << std::endl;
        }

        // Send the relevant data
        mg_websocket_client_write(currConn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
        mg_websocket_client_write(currConn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        conns.push_back(currConn);
    }

    // Wait for each server to ACK the scores
    while (sync.val < serverIPs.size())
        sync.cv.wait(lck);

    // Close connections
    for (size_t i = 0; i < conns.size(); i++)
        mg_close_connection(conns[i]);
}

/*
 * FILE I/O HELPERS
 */

BGN get_bgn_private_key_from_file(
    const char *filename)
{
    std::ifstream bgnFile(filename);
    BGN privateKey;

    bgnFile >> privateKey;

    return privateKey;
}

Twistpoint get_generator_from_file(
    const char *filename,
    Proof& pi)
{
    std::ifstream genFile(filename);
    Twistpoint retval;

    genFile >> pi;
    genFile >> retval;

    return retval;
}

Twistpoint get_generator_from_file(
    const char *filename,
    std::vector<Proof>& pi)
{
    std::ifstream epochFile(filename);
    
    Twistpoint retval;
    BinarySizeT sizeOfVector;
    pi.clear();

    epochFile >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Proof currProof;
        epochFile >> currProof;

        pi.push_back(currProof);
    }
    epochFile >> retval;

    return retval;
}

/*
 * EPOCH DATA SERIALIZERS/UN-SERIALIZERS
 */

std::string make_epoch_initiator_string(
    const std::vector<Proof>& generatorProof,
    const Twistpoint& nextGenerator)
{
    std::stringstream buffer;
    
    BinarySizeT sizeOfVector(generatorProof.size());
    buffer << sizeOfVector;

    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << generatorProof[i];
    
    buffer << nextGenerator;

    return buffer.str();
}

ssize_t read_epoch_initiator_string(
    const char *filename,
    std::vector<Proof>& generatorProof,
    Twistpoint& nextGenerator)
{
    std::ifstream file(filename);

    file.ignore(std::numeric_limits<std::streamsize>::max());
    std::streamsize retval = file.gcount();
    file.clear();
    file.seekg(0, std::ios_base::beg);

    BinarySizeT sizeOfVector;

    generatorProof.clear();

    file >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Proof currProof;
        file >> currProof;

        generatorProof.push_back(currProof);
    }
    
    file >> nextGenerator;

    return retval;
}

std::string make_epoch_update_string(
    const std::vector<std::vector<Proof>>& pi,
    const std::vector<std::vector<Twistpoint>>& permutationCommits,
    const std::vector<std::vector<Twistpoint>>& freshPseudonymCommits,
    const std::vector<std::vector<Twistpoint>>& freshPseudonymSeedCommits,
    const std::vector<std::vector<CurveBipoint>>& serverTallyCommits,
    const std::vector<std::vector<std::vector<TwistBipoint>>>& partwayVoteMatrixCommits,
    const std::vector<std::vector<std::vector<TwistBipoint>>>& finalVoteMatrixCommits,
    const std::vector<std::vector<Twistpoint>>& userTallyMaskCommits,
    const std::vector<std::vector<Twistpoint>>& userTallyMessageCommits,
    const std::vector<std::vector<Twistpoint>>& userTallySeedCommits,
    const Twistpoint& nextGenerator,
    bool doUserTallies)
{
    std::stringstream buffer;
    
    BinarySizeT sizeOfVectorI, sizeOfVectorJ;

    sizeOfVectorI.set(pi.size());
    buffer << sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        sizeOfVectorJ.set(pi[i].size());
        buffer << sizeOfVectorJ;

        for (size_t j = 0; j < sizeOfVectorJ.val(); j++)
            buffer << pi[i][j];
    }

    sizeOfVectorI.set(permutationCommits.size());
    buffer << sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
            buffer << permutationCommits[i][j];

    sizeOfVectorI.set(freshPseudonymCommits.size());
    buffer << sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
            buffer << freshPseudonymCommits[i][j];

    sizeOfVectorI.set(freshPseudonymSeedCommits.size());
    buffer << sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
            buffer << freshPseudonymSeedCommits[i][j];

    sizeOfVectorI.set(serverTallyCommits.size());
    buffer << sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
            buffer << serverTallyCommits[i][j];

    sizeOfVectorI.set(partwayVoteMatrixCommits.size());
    buffer << sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
            for (size_t k = 0; k < sizeOfVectorI.val(); k++)
                buffer << partwayVoteMatrixCommits[i][j][k];

    sizeOfVectorI.set(finalVoteMatrixCommits.size());
    buffer << sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
            for (size_t k = 0; k < sizeOfVectorI.val(); k++)
                buffer << finalVoteMatrixCommits[i][j][k];

    sizeOfVectorI.set(userTallyMaskCommits.size());
    buffer << sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
            buffer << userTallyMaskCommits[i][j];

    sizeOfVectorI.set(userTallyMessageCommits.size());
    buffer << sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
            buffer << userTallyMessageCommits[i][j];

    sizeOfVectorI.set(userTallySeedCommits.size());
    buffer << sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
            buffer << userTallySeedCommits[i][j];

    buffer << nextGenerator;

    BinaryBool flag(doUserTallies);
    buffer << flag;

    return buffer.str();
}

ssize_t read_epoch_update_string(
    const char *filename,
    std::vector<std::vector<Proof>>& pi,
    std::vector<std::vector<Twistpoint>>& permutationCommits,
    std::vector<std::vector<Twistpoint>>& freshPseudonymCommits,
    std::vector<std::vector<Twistpoint>>& freshPseudonymSeedCommits,
    std::vector<std::vector<CurveBipoint>>& serverTallyCommits,
    std::vector<std::vector<std::vector<TwistBipoint>>>& partwayVoteMatrixCommits,
    std::vector<std::vector<std::vector<TwistBipoint>>>& finalVoteMatrixCommits,
    std::vector<std::vector<Twistpoint>>& userTallyMaskCommits,
    std::vector<std::vector<Twistpoint>>& userTallyMessageCommits,
    std::vector<std::vector<Twistpoint>>& userTallySeedCommits,
    Twistpoint& nextGenerator,
    bool& doUserTallies)
{
    std::ifstream file(filename);

    file.ignore(std::numeric_limits<std::streamsize>::max());
    std::streamsize retval = file.gcount();
    file.clear();
    file.seekg(0, std::ios_base::beg);

    BinarySizeT sizeOfVectorI, sizeOfVectorJ;
    
    pi.clear();
    permutationCommits.clear();
    freshPseudonymCommits.clear();
    freshPseudonymSeedCommits.clear();
    serverTallyCommits.clear();
    partwayVoteMatrixCommits.clear();
    finalVoteMatrixCommits.clear();
    userTallyMaskCommits.clear();
    userTallyMessageCommits.clear();
    userTallySeedCommits.clear();

    file >> sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        std::vector<Proof> currRow;

        file >> sizeOfVectorJ;
        for (size_t j = 0; j < sizeOfVectorJ.val(); j++)
        {
            Proof currProof;
            file >> currProof;

            currRow.push_back(currProof);
        }

        pi.push_back(currRow);
    }

    file >> sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        std::vector<Twistpoint> currRow;
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
        {
            Twistpoint currCommit;
            file >> currCommit;

            currRow.push_back(currCommit);
        }
        permutationCommits.push_back(currRow);
    }

    file >> sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        std::vector<Twistpoint> currRow;
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
        {
            Twistpoint currCommit;
            file >> currCommit;

            currRow.push_back(currCommit);
        }
        freshPseudonymCommits.push_back(currRow);
    }

    file >> sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        std::vector<Twistpoint> currRow;
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
        {
            Twistpoint currCommit;
            file >> currCommit;

            currRow.push_back(currCommit);
        }
        freshPseudonymSeedCommits.push_back(currRow);
    }

    file >> sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        std::vector<CurveBipoint> currRow;
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
        {
            CurveBipoint currCommit;
            file >> currCommit;

            currRow.push_back(currCommit);
        }
        serverTallyCommits.push_back(currRow);
    }

    file >> sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        std::vector<std::vector<TwistBipoint>> currMatrix;
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
        {
            std::vector<TwistBipoint> currRow;
            for (size_t k = 0; k < sizeOfVectorI.val(); k++)
            {
                TwistBipoint currCommit;
                file >> currCommit;

                currRow.push_back(currCommit);    
            }
            currMatrix.push_back(currRow);
        }
        partwayVoteMatrixCommits.push_back(currMatrix);
    }

    file >> sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        std::vector<std::vector<TwistBipoint>> currMatrix;
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
        {
            std::vector<TwistBipoint> currRow;
            for (size_t k = 0; k < sizeOfVectorI.val(); k++)
            {
                TwistBipoint currCommit;
                file >> currCommit;

                currRow.push_back(currCommit);    
            }
            currMatrix.push_back(currRow);
        }
        finalVoteMatrixCommits.push_back(currMatrix);
    }

    file >> sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        std::vector<Twistpoint> currRow;
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
        {
            Twistpoint currCommit;
            file >> currCommit;

            currRow.push_back(currCommit);
        }
        userTallyMaskCommits.push_back(currRow);
    }

    file >> sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        std::vector<Twistpoint> currRow;
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
        {
            Twistpoint currCommit;
            file >> currCommit;

            currRow.push_back(currCommit);
        }
        userTallyMessageCommits.push_back(currRow);
    }

    file >> sizeOfVectorI;
    for (size_t i = 0; i < sizeOfVectorI.val(); i++)
    {
        std::vector<Twistpoint> currRow;
        for (size_t j = 0; j < sizeOfVectorI.val(); j++)
        {
            Twistpoint currCommit;
            file >> currCommit;

            currRow.push_back(currCommit);
        }
        userTallySeedCommits.push_back(currRow);
    }

    file >> nextGenerator;
    
    BinaryBool binaryDoUserTallies;
    file >> binaryDoUserTallies;

    doUserTallies = binaryDoUserTallies.val();

    return retval;
}

/**********************************************************
 ****                                                  ****
 ****  other server-relevant handler member functions  ****
 ****                                                  ****
 **********************************************************/

/*
 * EPOCH READY HANDLER
 */

EpochReadyHandler::EpochReadyHandler(
    struct synchronization_tool *exitSync,
    struct synchronization_tool *readySync,
    std::mutex& updateMtx,
    size_t numServers)
: exitSync(exitSync), readySync(readySync), updateMtx(updateMtx), numServers(numServers)
{ /* */ }

bool EpochReadyHandler::handleGet(
    CivetServer *server,
    struct mg_connection *conn)
{
    std::unique_lock<std::mutex> exitLock(exitSync->mtx, std::defer_lock);
    std::unique_lock<std::mutex> updateLock(updateMtx, std::defer_lock);
    std::unique_lock<std::mutex> readyLock(readySync->mtx);

    if (readySync->val < numServers)
    {
        mg_printf(conn, "HTTP/1.1 503 Service Unavailable\r\n"
                        "Content-Type: text/plain\r\n"
                        "Connection: close\r\n\r\n");
        mg_printf(conn, "Server is waiting for other servers to begin.\n");
    }
    else if (!exitLock.try_lock())
    {
        mg_printf(conn, "HTTP/1.1 503 Service Unavailable\r\n"
                        "Content-Type: text/plain\r\n"
                        "Connection: close\r\n\r\n");
        mg_printf(conn, "Server is still in a previous epoch.\n");   
    }
    else if (!updateLock.try_lock())
    {
        mg_printf(conn, "HTTP/1.1 503 Service Unavailable\r\n"
                        "Content-Type: text/plain\r\n"
                        "Connection: close\r\n\r\n");
        mg_printf(conn, "Server is handling other updates.\n");   
    }
    else
    {
        mg_printf(conn, "HTTP/1.1 200 OK\r\n"
                        "Content-Type: text/plain\r\n"
                        "Connection: close\r\n\r\n");
        mg_printf(conn, "Server is ready for epoch.\n");
    }

    return true;
}

/*
 * EPOCH NUM HANDLER
 */

EpochNumHandler::EpochNumHandler(
    std::atomic<size_t>& epochNum)
: epochNum(epochNum)
{ /* */ }

bool EpochNumHandler::handleGet(
    CivetServer *server,
    struct mg_connection *conn)
{
    mg_printf(conn, "HTTP/1.1 200 OK\r\n"
                    "Content-Type: text/plain\r\n"
                    "Connection: close\r\n\r\n");
    mg_printf(conn, "Epoch num: %lu\n", epochNum.load());

    return true;
}

/*
 * UPDATE LOCK HANDLER
 */

UpdateLockWebSocketHandler::UpdateLockWebSocketHandler(
    std::mutex& updateMtx,
    std::unique_lock<std::mutex> **lockHolder,
    bool isLocking)
: updateMtx(updateMtx), lockHolder(lockHolder), isLocking(isLocking)
{ /* */ }

UpdateLockWebSocketHandler::~UpdateLockWebSocketHandler()
{
    delete *lockHolder;
    *lockHolder = NULL;
}

bool UpdateLockWebSocketHandler::handleConnection(
    CivetServer *server,
    const struct mg_connection *conn)
{
    return true;
}

void UpdateLockWebSocketHandler::handleReadyState(
    CivetServer *server,
    struct mg_connection *conn)
{ /* */ }

bool UpdateLockWebSocketHandler::handleData(
    CivetServer *server,
    struct mg_connection *conn,
    int bits,
    char *data,
    size_t data_len)
{
    switch (bits & 0xf)
    {
        case MG_WEBSOCKET_OPCODE_DATACOMPLETE:
            if (isLocking)
            {
                std::unique_lock<std::mutex> *tempHolder = new std::unique_lock<std::mutex>(updateMtx);
                
                // Once you get to this line, we now hold the lock, and lockHolder is guaranteed to be NULL
                *lockHolder = tempHolder;

                // Respond to notify that the requesting process holds the lock
                mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
            }
            else
            {
                // You must do things in this order so that *lockHolder will be guaranteed to be NULL at the time the lock unlocks (deletion of the lock object)
                std::unique_lock<std::mutex> *tempHolder = *lockHolder;
                *lockHolder = NULL;

                delete tempHolder;
                
                // Respond to notify that the requesting process has released the lock
                mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
            }
            break;

        case MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE:
            break;

        default:
            std::cerr << "Unknown opcode: failing." << std::endl;
            break;
    }

    return false;
}

void UpdateLockWebSocketHandler::handleClose(
    CivetServer *server,
    const struct mg_connection *conn)
{ /* */ }

/*********************************************************
 ****                                                 ****
 ****  PrsonaServerWebSocketHandler member functions  ****
 ****                                                 ****
 *********************************************************/

/*
 * CONSTRUCTOR
 */

PrsonaServerWebSocketHandler::PrsonaServerWebSocketHandler(
    std::default_random_engine& rng,
    PrsonaServer *prsonaServer,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& selfIP,
    int selfPort,
    std::mutex& updateMtx,
    std::atomic<size_t>& epochNum,
    std::mutex& buildUpOutputMtx,
    const std::string& buildUpOutputFilename,
    std::mutex& breakDownOutputMtx,
    const std::string& breakDownOutputFilename,
    std::mutex& updateOutputMtx,
    const std::string& updateOutputFilename,
    std::mutex& voteOutputMtx,
    const std::string& voteOutputFilename)
: rng(rng), prsonaServer(prsonaServer), serverIPs(serverIPs), serverPorts(serverPorts), selfIP(selfIP), selfPort(selfPort), updateMtx(updateMtx), epochNum(epochNum), buildUpOutputMtx(buildUpOutputMtx), buildUpOutputFilename(buildUpOutputFilename), breakDownOutputMtx(breakDownOutputMtx), breakDownOutputFilename(breakDownOutputFilename), updateOutputMtx(updateOutputMtx), updateOutputFilename(updateOutputFilename), voteOutputMtx(voteOutputMtx), voteOutputFilename(voteOutputFilename)
{ /* */ }

/*
 * REQUIRED BY INHERITED CLASS
 */

bool PrsonaServerWebSocketHandler::handleConnection(
    CivetServer *server,
    const struct mg_connection *conn)
{
    const struct mg_request_info *info = mg_get_request_info(conn);
    
    // Check if the request being made is something this server can respond to
    bool flag = info->query_string && info->query_string[0] >= PRSONA_GIVE_BGN_PUBKEY && info->query_string[0] <= PRSONA_RECEIVE_EPOCH_UPDATE;
    flag = flag || (info->query_string && info->query_string[0] == PRSONA_GIVE_PARTIAL_DECRYPTION);
    flag = flag || (info->query_string && info->query_string[0] == PRSONA_RECEIVE_PARTIAL_DECRYPTION);

    return flag;
}

void PrsonaServerWebSocketHandler::handleReadyState(
    CivetServer *server,
    struct mg_connection *conn)
{
    const struct mg_request_info *info = mg_get_request_info(conn);

    // Set filenames for query types that will need to un-serialize data to respond correctly
    switch (info->query_string[0])
    {
        case PRSONA_GIVE_VOTE_ROW:
        case PRSONA_GIVE_VOTE_MATRIX:
        case PRSONA_GIVE_CLIENT_TALLY:
        case PRSONA_GIVE_SERVER_TALLY:
        case PRSONA_GIVE_PSEUDONYMS:

        case PRSONA_GIVE_VOTE_ROW_COMMITMENT:
        case PRSONA_GIVE_VOTE_MATRIX_COMMITMENT:
        case PRSONA_GIVE_CLIENT_TALLY_COMMITMENT:
        case PRSONA_GIVE_SERVER_TALLY_COMMITMENT:
        case PRSONA_GIVE_PSEUDONYMS_COMMITMENT:

        case PRSONA_RECEIVE_NEW_CLIENT:
        case PRSONA_RECEIVE_VOTE:

        case PRSONA_RECEIVE_UPDATE_WITH_NEW_USER:

        case PRSONA_ADD_CURR_SEED_TO_GENERATOR:
        case PRSONA_RECEIVE_FRESH_GENERATOR:
        case PRSONA_ADD_RAND_SEED_TO_GENERATOR:
        case PRSONA_RECEIVE_EG_BLIND_GENERATOR:

        case PRSONA_PERFORM_EPOCH_BUILD_UP:
        case PRSONA_PERFORM_EPOCH_BREAK_DOWN:
        case PRSONA_RECEIVE_EPOCH_UPDATE:

        case PRSONA_RECEIVE_PARTIAL_DECRYPTION:
            set_temp_filename(rng, conn);
            break;

        default:
            mg_set_user_connection_data(conn, NULL);
            break;
    }
}

bool PrsonaServerWebSocketHandler::handleData(
    CivetServer *server,
    struct mg_connection *conn,
    int bits,
    char *data,
    size_t data_len)
{
    char *filename = (char *) mg_get_user_connection_data(conn);
    FILE *currFile = NULL;

    switch (bits & 0xf)
    {
        // Requester has indicated they have sent all relevant data
        case MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE:
        case MG_WEBSOCKET_OPCODE_DATACOMPLETE:
            generate_response(server, conn, filename);
            break;

        // Requester has sent more data (which may theoretically be broken up into multiple packets)
        case MG_WEBSOCKET_OPCODE_BINARY:
        case MG_WEBSOCKET_OPCODE_CONTINUATION:
            currFile = fopen(filename, "ab");
            fwrite(data, sizeof(char), data_len, currFile);
            fclose(currFile);
            return true;

        // Something strange has happened
        default:
            std::cerr << "Unknown packet type received. Failing." << std::endl;
            break;
    }

    return false;
}

void PrsonaServerWebSocketHandler::handleClose(
    CivetServer *server,
    const struct mg_connection *conn)
{
    char *filename = (char *) mg_get_user_connection_data(conn);

    // If we didn't have a temp file for this request, don't do anything
    if (!filename)
        return;

    // If we did, delete it
    remove(filename);
    delete [] filename;
}

/*
 * RESPONSE ROUTER FUNCTION
 */

void PrsonaServerWebSocketHandler::generate_response(
    CivetServer *server,
    struct mg_connection *conn,
    const char *filename)
{
    const struct mg_request_info *info = mg_get_request_info(conn);

    // Select the correct response for this type of request
    switch (info->query_string[0])
    {
        // BASIC PUBLIC SYSTEM INFO GETTERS
        case PRSONA_GIVE_BGN_PUBKEY:
            get_bgn_public_key(conn);
            break;

        case PRSONA_GIVE_NUM_CLIENTS:
            get_num_clients(conn);
            break;

        case PRSONA_GIVE_NUM_SERVERS:
            get_num_servers(conn);
            break;

        case PRSONA_GIVE_FRESH_GENERATOR:
            get_generator(conn, true);
            break;

        case PRSONA_GIVE_EG_BLIND_GENERATOR:
            get_generator(conn, false);
            break;

        // ENCRYPTED DATA GETTERS
        case PRSONA_GIVE_VOTE_ROW:
            get_current_votes_by(conn, filename);
            break;

        case PRSONA_GIVE_VOTE_MATRIX:
            get_all_current_votes(conn);
            break;

        case PRSONA_GIVE_CLIENT_TALLY:
            get_current_user_encrypted_tally(conn, filename);
            break;

        case PRSONA_GIVE_SERVER_TALLY:
            get_current_server_encrypted_tally(conn, filename);
            break;

        case PRSONA_GIVE_PSEUDONYMS:
            get_current_pseudonyms(conn);
            break;

        // PROOF COMMITMENT GETTERS
        case PRSONA_GIVE_VOTE_ROW_COMMITMENT:
            get_vote_row_commitment(conn, filename);
            break;

        case PRSONA_GIVE_VOTE_MATRIX_COMMITMENT:
            get_vote_matrix_commitment(conn);
            break;

        case PRSONA_GIVE_CLIENT_TALLY_COMMITMENT:
            get_user_tally_commitment(conn, filename);
            break;

        case PRSONA_GIVE_SERVER_TALLY_COMMITMENT:
            get_server_tally_commitment(conn, filename);
            break;

        case PRSONA_GIVE_PSEUDONYMS_COMMITMENT:
            get_pseudonyms_commitment(conn);
            break;

        // CLIENT INTERACTIONS
        case PRSONA_RECEIVE_NEW_CLIENT:
            add_new_client(conn, filename);
            break;

        case PRSONA_RECEIVE_VOTE:
            receive_vote(server, conn, filename);
            break;

        // CLIENT INTERACTION HELPER
        case PRSONA_RECEIVE_UPDATE_WITH_NEW_USER:
            import_new_user_update(conn, filename);
            break;

        // CONSTRUCTOR HELPERS
        case PRSONA_GIVE_BGN_PRIVKEY:
            get_bgn_details(conn);
            break;

        case PRSONA_ADD_CURR_SEED_TO_GENERATOR:
            add_seed_to_generator(conn, filename, true);
            break;

        case PRSONA_RECEIVE_FRESH_GENERATOR:
            set_generator(filename, true);
            break;

        case PRSONA_ADD_RAND_SEED_TO_GENERATOR:
            add_seed_to_generator(conn, filename, false);
            break;

        case PRSONA_RECEIVE_EG_BLIND_GENERATOR:
            set_generator(filename, false);
            break;

        // EPOCH ROUNDS
        case PRSONA_PERFORM_EPOCH_BUILD_UP:
            build_up_midway_pseudonyms(server, conn, filename);
            break;

        case PRSONA_PERFORM_EPOCH_BREAK_DOWN:
            break_down_midway_pseudonyms(server, conn, filename);
            break;

        case PRSONA_RECEIVE_EPOCH_UPDATE:
            accept_epoch_updates(server, conn, filename);
            break;

        // DISTRIBUTED BGN
        case PRSONA_GIVE_PARTIAL_DECRYPTION:
            get_partial_decryption(conn);
            break;

        case PRSONA_RECEIVE_PARTIAL_DECRYPTION:
            receive_tallied_scores(conn, filename);
            break;

        default:
            break;
    }
}

/*
 * BASIC PUBLIC SYSTEM INFO GETTERS
 */

void PrsonaServerWebSocketHandler::get_bgn_public_key(
    struct mg_connection *conn) const
{
    // Retrieve value
    BGNPublicKey pubKey = prsonaServer->get_bgn_public_key();

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pubKey;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_num_clients(
    struct mg_connection *conn) const
{
    // Retrieve value
    BinarySizeT numClients(prsonaServer->get_num_clients());

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << numClients;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_num_servers(
    struct mg_connection *conn) const
{
    // Retrieve value
    BinarySizeT numServers(prsonaServer->get_num_servers());

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << numServers;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_generator(
    struct mg_connection *conn,
    bool fresh)
{
    // Retrieve value
    Twistpoint generator;
    std::vector<Proof> pi;
    if (fresh)
        generator = prsonaServer->get_fresh_generator(pi);
    else
        generator = prsonaServer->get_blinding_generator(pi);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    BinarySizeT sizeOfVector(pi.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << pi[i];
    buffer << generator;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
}

/*
 * ENCRYPTED DATA GETTERS
 */

void PrsonaServerWebSocketHandler::get_current_votes_by(
    struct mg_connection *conn,
    const char *filename) const
{
    std::ifstream file(filename);

    // Un-serialize request
    Twistpoint shortTermPublicKey;
    file >> shortTermPublicKey;

    // Retrieve value
    Proof pi;
    std::vector<TwistBipoint> votes = prsonaServer->get_current_votes_by(pi, shortTermPublicKey);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi;
    BinarySizeT sizeOfVector(votes.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << votes[i];
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_all_current_votes(
    struct mg_connection *conn) const
{
    // Retrieve value
    Proof pi;
    std::vector<std::vector<TwistBipoint>> votes = prsonaServer->get_all_current_votes(pi);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi;
    BinarySizeT sizeOfVector(votes.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        for (size_t j = 0; j < sizeOfVector.val(); j++)
            buffer << votes[i][j];
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_current_user_encrypted_tally(
    struct mg_connection *conn,
    const char *filename) const
{
    std::ifstream file(filename);

    // Un-serialize request
    Twistpoint shortTermPublicKey;
    file >> shortTermPublicKey;

    // Retrieve value
    Proof pi;
    EGCiphertext tally = prsonaServer->get_current_user_encrypted_tally(pi, shortTermPublicKey);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi;
    buffer << tally;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_current_server_encrypted_tally(
    struct mg_connection *conn,
    const char *filename) const
{
    std::ifstream file(filename);

    // Un-serialize request
    Twistpoint shortTermPublicKey;
    file >> shortTermPublicKey;

    // Retrieve value
    Proof pi;
    CurveBipoint tally = prsonaServer->get_current_server_encrypted_tally(pi, shortTermPublicKey);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi;
    buffer << tally;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_current_pseudonyms(
    struct mg_connection *conn) const
{
    // Retrieve value
    Proof pi;
    std::vector<Twistpoint> pseudonyms = prsonaServer->get_current_pseudonyms(pi);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi;
    BinarySizeT sizeOfVector(pseudonyms.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << pseudonyms[i];
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

/*
 * PROOF COMMITMENT GETTERS
 */

void PrsonaServerWebSocketHandler::get_vote_row_commitment(
    struct mg_connection *conn,
    const char *filename) const
{
    std::ifstream file(filename);

    // Un-serialize request
    Twistpoint shortTermPublicKey;
    file >> shortTermPublicKey;

    // Retrieve value
    Proof pi = prsonaServer->get_vote_row_commitment(shortTermPublicKey);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_vote_matrix_commitment(
    struct mg_connection *conn) const
{
    // Retrieve value
    Proof pi = prsonaServer->get_vote_matrix_commitment();

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_user_tally_commitment(
    struct mg_connection *conn,
    const char *filename) const
{
    std::ifstream file(filename);

    // Un-serialize request
    Twistpoint shortTermPublicKey;
    file >> shortTermPublicKey;

    // Retrieve value
    Proof pi = prsonaServer->get_user_tally_commitment(shortTermPublicKey);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_server_tally_commitment(
    struct mg_connection *conn,
    const char *filename) const
{
    std::ifstream file(filename);

    // Un-serialize request
    Twistpoint shortTermPublicKey;
    file >> shortTermPublicKey;

    // Retrieve value
    Proof pi = prsonaServer->get_server_tally_commitment(shortTermPublicKey);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::get_pseudonyms_commitment(
    struct mg_connection *conn) const
{
    // Retrieve value
    Proof pi = prsonaServer->get_pseudonyms_commitment();

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

/*
 * CLIENT INTERACTIONS
 */

void PrsonaServerWebSocketHandler::add_new_client(
    struct mg_connection *conn,
    const char *filename)
{
    std::ifstream file(filename);

    // Un-serialize request
    Proof proofOfValidKey;
    file >> proofOfValidKey;
    Twistpoint shortTermPublicKey, empty;
    file >> shortTermPublicKey;

    // Obtain global update lock
    std::unique_lock<std::mutex> updateLock(updateMtx, std::defer_lock);
    obtain_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);

    // Add new client to server object
    std::vector<Proof> proofOfValidAddition;
    prsonaServer->add_new_client(proofOfValidAddition, proofOfValidKey, shortTermPublicKey);

    // Share this update with other servers
    std::vector<CurveBipoint> previousVoteTallies;
    std::vector<Twistpoint> currentPseudonyms;
    std::vector<EGCiphertext> currentUserEncryptedTallies;
    std::vector<std::vector<TwistBipoint>> voteMatrix;
    prsonaServer->export_new_user_update(previousVoteTallies, currentPseudonyms, currentUserEncryptedTallies, voteMatrix);
    distribute_new_user_updates(proofOfValidAddition, previousVoteTallies, currentPseudonyms, currentUserEncryptedTallies, voteMatrix);

    // Release global update lock
    release_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    BinarySizeT sizeOfVector(proofOfValidAddition.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << proofOfValidAddition[i];
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::receive_vote(
    CivetServer *civetServer,
    struct mg_connection *conn,
    const char *filename)
{
    std::ifstream file(filename);

    file.ignore(std::numeric_limits<std::streamsize>::max());
    std::streamsize bandwidthRcv = file.gcount();
    file.clear();
    file.seekg(0, std::ios_base::beg);

    // Un-serialize request
    BinarySizeT sizeOfVector;
    file >> sizeOfVector;

    std::vector<Proof> pi;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Proof currProof;
        file >> currProof;

        pi.push_back(currProof);
    }

    file >> sizeOfVector;

    std::vector<TwistBipoint> newVotes;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        TwistBipoint currVote;
        file >> currVote;

        newVotes.push_back(currVote);
    }

    Twistpoint shortTermPublicKey;
    file >> shortTermPublicKey;

    BinaryBool shouldDeal;
    file >> shouldDeal;

    // If we're dealing this update to the other servers, obtain global update lock
    std::unique_lock<std::mutex> updateLock(updateMtx, std::defer_lock);

    std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer->getContext());
    std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
    clock_t cpuTimeBefore = clock();

    if (shouldDeal.val())
        obtain_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);

    // Load votes into server object
    prsonaServer->receive_vote(pi, newVotes, shortTermPublicKey);

    // If we're dealing this update to the other servers, actually do that
    if (shouldDeal.val())
    {
        distribute_new_vote(pi, newVotes, shortTermPublicKey);

        // Then release the global update lock
        release_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);
    }

    clock_t cpuTimeAfter = clock();
    std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
    std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer->getContext());

    std::vector<double> timingData(2);
    timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
    timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;

    std::vector<size_t> bandwidthData(2);
    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];

    write_log_data(voteOutputMtx, voteOutputFilename, timingData, bandwidthData);

    // Notify client their request has been completed
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
}

/*
 * DISTRIBUTION HELPERS FOR CLIENT INTERACTIONS
 */

void PrsonaServerWebSocketHandler::distribute_new_user_updates(
    const std::vector<Proof>& proofOfValidAddition,
    const std::vector<CurveBipoint>& previousVoteTallies,
    const std::vector<Twistpoint>& currentPseudonyms,
    const std::vector<EGCiphertext>& currentUserEncryptedTallies,
    const std::vector<std::vector<TwistBipoint>>& voteMatrix) const
{
    struct synchronization_tool sync;

    // Serialize data
    std::stringstream buffer;
    std::string data;
    BinarySizeT sizeOfVector;

    sizeOfVector.set(proofOfValidAddition.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << proofOfValidAddition[i];

    sizeOfVector.set(previousVoteTallies.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << previousVoteTallies[i];

    sizeOfVector.set(currentPseudonyms.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << currentPseudonyms[i];

    sizeOfVector.set(currentUserEncryptedTallies.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << currentUserEncryptedTallies[i];

    sizeOfVector.set(voteMatrix.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        for (size_t j = 0; j < sizeOfVector.val(); j++)
            buffer << voteMatrix[i][j];

    data = buffer.str();

    // Connect to each server and give them data on the new user
    size_t i = 0;
    while (i < serverIPs.size())
    {
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
        {
            i++;
            continue;
        }

        struct mg_connection *conn = NULL;

        std::unique_lock<std::mutex> lck(sync.mtx);
        sync.val = 0;
        sync.val2 = 0;
        while (!conn)
        {
            conn = mg_connect_websocket_client(serverIPs[i].c_str(), serverPorts[i], USE_SSL, NULL, 0, SUBMIT_UPDATE_WITH_NEW_USER_URI, "null", synchro_websocket_data_handler, synchro_websocket_close_handler, &sync);
            
            if (!conn)
                std::cerr << "Couldn't give server " << i << " new user" << std::endl;
        }

        // Send the server the new user data
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        // Wait for its response (or the connection to die)
        while (!sync.val2)
            sync.cv.wait(lck);

        // Close connection
        mg_close_connection(conn);

        // Only move forward once we've confirmed we correctly gave the update (or else we risk synchronization issues!)
        if (sync.val)
            i++;
    }
}

void PrsonaServerWebSocketHandler::distribute_new_vote(
    std::vector<Proof> pi,
    std::vector<TwistBipoint> newVotes,
    Twistpoint shortTermPublicKey) const
{
    struct synchronization_tool sync;

    // Serialize data
    std::stringstream buffer;
    std::string data;
    BinarySizeT sizeOfVector;

    sizeOfVector.set(pi.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << pi[i];

    sizeOfVector.set(newVotes.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << newVotes[i];

    buffer << shortTermPublicKey;

    BinaryBool flag(false);
    buffer << flag;

    data = buffer.str();

    // Connect to each server and give them the new votes
    size_t i = 0;
    while (i < serverIPs.size())
    {
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
        {
            i++;
            continue;
        }

        struct mg_connection *conn = NULL;

        std::unique_lock<std::mutex> syncLock(sync.mtx);
        sync.val = 0;
        sync.val2 = 0;
        while (!conn)
        {
            conn = mg_connect_websocket_client(serverIPs[i].c_str(), serverPorts[i], USE_SSL, NULL, 0, SUBMIT_VOTE_URI, "null", synchro_websocket_data_handler, synchro_websocket_close_handler, &sync);

            if (!conn)
                std::cerr << "Couldn't give server " << i << " new vote data" << std::endl;
        }
        
        // Send the server the new vote data
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        // Wait for its response (or the connection to die)
        while (!sync.val2)
            sync.cv.wait(syncLock);

        // Close connection
        mg_close_connection(conn);

        // Only move forward once we've confirmed we correctly gave the update (or else we risk synchronization issues!)
        if (sync.val)
            i++;
    }
}

void PrsonaServerWebSocketHandler::import_new_user_update(
    struct mg_connection *conn,
    const char *filename)
{
    std::vector<Proof> proofOfValidAddition;
    std::vector<CurveBipoint> previousVoteTallies;
    std::vector<Twistpoint> currentPseudonyms;
    std::vector<EGCiphertext> currentUserEncryptedTallies;
    std::vector<std::vector<TwistBipoint>> voteMatrix;

    std::ifstream file(filename);

    // Un-serialize request
    BinarySizeT sizeOfVector;
    
    file >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Proof currProof;
        file >> currProof;

        proofOfValidAddition.push_back(currProof);
    }

    file >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        CurveBipoint currTally;
        file >> currTally;

        previousVoteTallies.push_back(currTally);
    }

    file >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Twistpoint currNym;
        file >> currNym;

        currentPseudonyms.push_back(currNym);
    }

    file >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        EGCiphertext currTally;
        file >> currTally;

        currentUserEncryptedTallies.push_back(currTally);
    }

    file >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        std::vector<TwistBipoint> currRow;
        for (size_t j = 0; j < sizeOfVector.val(); j++)
        {
            TwistBipoint currVote;
            file >> currVote;

            currRow.push_back(currVote);
        }
        voteMatrix.push_back(currRow);
    }

    // Load data into server object
    prsonaServer->import_new_user_update(proofOfValidAddition, previousVoteTallies, currentPseudonyms, currentUserEncryptedTallies, voteMatrix);

    // Acknowledge successful request
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
}

/*
 * CONSTRUCTOR HELPERS
 */

void PrsonaServerWebSocketHandler::get_bgn_details(
    struct mg_connection *conn) const
{
    // Retrieve value
    const BGN& sharedBGN = prsonaServer->get_bgn_details();

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << sharedBGN;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::add_seed_to_generator(
    struct mg_connection *conn,
    const char *filename,
    bool fresh) const
{
    std::ifstream file(filename);

    // Un-serialize request
    Twistpoint currGenerator;
    file >> currGenerator;

    // Retrieve value
    std::vector<Proof> pi;
    if (fresh)
        currGenerator = prsonaServer->add_curr_seed_to_generator(pi, currGenerator);
    else
        currGenerator = prsonaServer->add_rand_seed_to_generator(pi, currGenerator);

    // Serialize response
    std::stringstream buffer;
    std::string data;
    buffer << pi[0];
    buffer << currGenerator;
    data = buffer.str();

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
}

void PrsonaServerWebSocketHandler::set_generator(
    const char *filename,
    bool fresh)
{
    std::ifstream file(filename);

    // Un-serialize request
    BinarySizeT sizeOfVector;
    file >> sizeOfVector;
    std::vector<Proof> pi;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Proof currProof;
        file >> currProof;

        pi.push_back(currProof);
    }
    Twistpoint newGenerator;
    file >> newGenerator;

    // Load value into server object
    if (fresh)
        prsonaServer->initialize_fresh_generator(pi, newGenerator);
    else
        prsonaServer->set_EG_blind_generator(pi, newGenerator);
}

/*
 * EPOCH ROUNDS
 */

void PrsonaServerWebSocketHandler::build_up_midway_pseudonyms(
    CivetServer *civetServer,
    struct mg_connection *conn,
    const char *filename)
{
    std::vector<std::vector<Proof>> generatorProofHolder;
    std::vector<Proof> generatorProof;
    Twistpoint nextGenerator;

    // Un-serialize request
    ssize_t bandwidthRcv = read_epoch_initiator_string(filename, generatorProof, nextGenerator);
    generatorProofHolder.push_back(generatorProof);

    std::vector<std::vector<std::vector<Proof>>> pi;
    pi.push_back(generatorProofHolder);
    std::vector<std::vector<std::vector<Twistpoint>>> permutationCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymSeedCommits;
    std::vector<std::vector<std::vector<CurveBipoint>>> serverTallyCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> partwayVoteMatrixCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> finalVoteMatrixCommits;

    std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer->getContext());
    std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
    clock_t cpuTimeBefore = clock();

    // Do actual epoch calculation
    prsonaServer->build_up_midway_pseudonyms(pi, permutationCommits, freshPseudonymCommits, freshPseudonymSeedCommits, serverTallyCommits, partwayVoteMatrixCommits, finalVoteMatrixCommits, nextGenerator);

    std::vector<std::vector<Twistpoint>> userTallyMaskCommits, userTallyMessageCommits, userTallySeedCommits;

    // Serialize update data
    std::string data = make_epoch_update_string(pi[1], permutationCommits[0], freshPseudonymCommits[0], freshPseudonymSeedCommits[0], serverTallyCommits[0], partwayVoteMatrixCommits[0], finalVoteMatrixCommits[0], userTallyMaskCommits, userTallyMessageCommits, userTallySeedCommits, nextGenerator, false);

    struct synchronization_tool sync;
    std::vector<struct mg_connection *> conns;

    // Connect to all other servers (roughly in parallel)
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 1;
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
            continue;

        // Send them update data
        struct mg_connection *currConn = distribute_epoch_updates(serverIPs[i], serverPorts[i], data, &sync);

        conns.push_back(currConn);
    }

    // Wait for all to acknowledge receipt of the update data
    while (sync.val < serverIPs.size())
        sync.cv.wait(lck);

    // Close connections
    for (size_t i = 0; i < conns.size(); i++)
        mg_close_connection(conns[i]);

    clock_t cpuTimeAfter = clock();
    std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
    std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer->getContext());

    std::vector<double> timingData(2);
    timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
    timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;

    std::vector<size_t> bandwidthData(2);
    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];

    write_log_data(buildUpOutputMtx, buildUpOutputFilename, timingData, bandwidthData);
    
    // Serialize response
    data = make_epoch_initiator_string(pi[0][0], nextGenerator);

    // Send response
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
}

void PrsonaServerWebSocketHandler::break_down_midway_pseudonyms(
    CivetServer *civetServer,
    struct mg_connection *conn,
    const char *filename)
{
    std::vector<Proof> generatorProof;
    Twistpoint nextGenerator;

    // Un-serialize request
    ssize_t bandwidthRcv = read_epoch_initiator_string(filename, generatorProof, nextGenerator);
    
    std::vector<std::vector<std::vector<Proof>>> pi;
    std::vector<std::vector<std::vector<Twistpoint>>> permutationCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymSeedCommits;
    std::vector<std::vector<std::vector<CurveBipoint>>> serverTallyCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> partwayVoteMatrixCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> finalVoteMatrixCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> userTallyMaskCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> userTallyMessageCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> userTallySeedCommits;

    std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer->getContext());
    std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
    clock_t cpuTimeBefore = clock();

    // Do actual epoch calculation
    prsonaServer->break_down_midway_pseudonyms(generatorProof, pi, permutationCommits, freshPseudonymCommits, freshPseudonymSeedCommits, serverTallyCommits, partwayVoteMatrixCommits, finalVoteMatrixCommits, userTallyMaskCommits, userTallyMessageCommits, userTallySeedCommits, nextGenerator);

    // Serialize update data
    std::string data = make_epoch_update_string(pi[0], permutationCommits[0], freshPseudonymCommits[0], freshPseudonymSeedCommits[0], serverTallyCommits[0], partwayVoteMatrixCommits[0], finalVoteMatrixCommits[0], userTallyMaskCommits[0], userTallyMessageCommits[0], userTallySeedCommits[0], nextGenerator, true);

    struct synchronization_tool sync;
    std::vector<struct mg_connection *> conns;

    // Connect to all other servers (roughly in parallel)
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 1;
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        if (serverIPs[i] == selfIP && serverPorts[i] == selfPort)
            continue;

        // Send them update data
        struct mg_connection *currConn = distribute_epoch_updates(serverIPs[i], serverPorts[i], data, &sync);

        conns.push_back(currConn);
    }

    // Wait for all to acknowledge receipt of the update data
    while (sync.val < serverIPs.size())
        sync.cv.wait(lck);

    // Close connections
    for (size_t i = 0; i < conns.size(); i++)
        mg_close_connection(conns[i]);

    clock_t cpuTimeAfter = clock();
    std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
    std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer->getContext());

    std::vector<double> timingData(2);
    timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
    timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;

    std::vector<size_t> bandwidthData(2);
    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];

    write_log_data(breakDownOutputMtx, breakDownOutputFilename, timingData, bandwidthData);

    // Keep our epoch value up-to-date
    epochNum.fetch_add(1);

    // Tell initiator we have finished
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
}

void PrsonaServerWebSocketHandler::accept_epoch_updates(
    CivetServer *civetServer,
    struct mg_connection *conn,
    const char *filename)
{
    std::vector<std::vector<Proof>> pi;
    std::vector<std::vector<Twistpoint>> permutationCommits;
    std::vector<std::vector<Twistpoint>> freshPseudonymCommits;
    std::vector<std::vector<Twistpoint>> freshPseudonymSeedCommits;
    std::vector<std::vector<CurveBipoint>> serverTallyCommits;
    std::vector<std::vector<std::vector<TwistBipoint>>> partwayVoteMatrixCommits;
    std::vector<std::vector<std::vector<TwistBipoint>>> finalVoteMatrixCommits;
    std::vector<std::vector<Twistpoint>> userTallyMaskCommits;
    std::vector<std::vector<Twistpoint>> userTallyMessageCommits;
    std::vector<std::vector<Twistpoint>> userTallySeedCommits;
    Twistpoint nextGenerator;
    bool doUserTallies;

    // Un-serialize request
    ssize_t bandwidthRcv = read_epoch_update_string(filename, pi, permutationCommits, freshPseudonymCommits, freshPseudonymSeedCommits, serverTallyCommits, partwayVoteMatrixCommits, finalVoteMatrixCommits, userTallyMaskCommits, userTallyMessageCommits, userTallySeedCommits, nextGenerator, doUserTallies);

    std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer->getContext());
    std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
    clock_t cpuTimeBefore = clock();

    // Load data into server object
    prsonaServer->accept_epoch_updates(pi, permutationCommits, freshPseudonymCommits, freshPseudonymSeedCommits, serverTallyCommits, partwayVoteMatrixCommits, finalVoteMatrixCommits, userTallyMaskCommits, userTallyMessageCommits, userTallySeedCommits, nextGenerator, doUserTallies);

    clock_t cpuTimeAfter = clock();
    std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
    std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer->getContext());

    std::vector<double> timingData(2);
    timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
    timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;

    std::vector<size_t> bandwidthData(2);
    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];

    write_log_data(breakDownOutputMtx, breakDownOutputFilename, timingData, bandwidthData);

    // Acknowledge receipt of request
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
}

/*
 * DISTRIBUTED BGN
 */

void PrsonaServerWebSocketHandler::get_partial_decryption(
    struct mg_connection *conn) const
{
    // Send back ACK (since all we do here is simulate distributed BGN)
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
}

void PrsonaServerWebSocketHandler::receive_tallied_scores(
    struct mg_connection *conn,
    const char *filename)
{
    std::ifstream file(filename);

    // Un-serialize request
    std::vector<EGCiphertext> userScores;
    std::vector<CurveBipoint> serverScores;

    BinarySizeT sizeOfVector;
    file >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        EGCiphertext currScore;
        file >> currScore;

        userScores.push_back(currScore);
    }

    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        CurveBipoint currScore;
        file >> currScore;

        serverScores.push_back(currScore);
    }

    // Load into server object
    prsonaServer->receive_tallied_scores(userScores, serverScores);

    // Acknowledge receipt of data
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
}