PRSONA/prsona/src/networkClient.cpp

#include <iostream>
#include <fstream>
#include <sstream>

#include "networkClient.hpp"

/********************************************************
 *********                                      *********
 *********  client networking public functions  *********
 *********                                      *********
 ********************************************************/

/*
 * CREATOR FOR A NEW CLIENT
 */

PrsonaClient *create_client(
    std::default_random_engine& rng,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    size_t numServers)
{
    std::vector<size_t> bandwidthData(2);
    // Get the servers' public BGN key
    BGNPublicKey publicKey = get_bgn_public_key(rng, serverIPs, serverPorts);

    // Get the H point used in ElGamal operations
    std::vector<Proof> generatorProof;
    Twistpoint blindGenerator = get_generator(rng, serverIPs, serverPorts, false, generatorProof, bandwidthData);

    // Make the actual client object
    PrsonaClient *retval = new PrsonaClient(generatorProof, blindGenerator, publicKey, numServers);

    // Get the current fresh generator
    generatorProof.clear();
    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof, bandwidthData);

    // Load this fresh generator into the client object
    retval->receive_fresh_generator(generatorProof, freshGenerator);

    // Make the client's current short term public key
    Proof proofOfValidSTPK;
    Twistpoint shortTermPublicKey = retval->get_short_term_public_key(proofOfValidSTPK);

    // Register this client with the servers
    register_new_client(rng, retval, serverIPs, serverPorts, proofOfValidSTPK, shortTermPublicKey);

    return retval;
}

/*
 * FUNCTIONS TO PERFORM OPERATIONS FOR EXPERIMENT
 */

void make_vote(
    std::default_random_engine& rng,
    PrsonaClient* prsonaClient,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& target,
    int targetPort,
    size_t numClients,
    const CivetServer& civetServer,
    std::mutex& outputMtx,
    const std::string& outputFilename,
    std::mutex& usageMtx,
    const std::string& usageFilename)
{
    std::uniform_int_distribution<int> voteDistribution(0, PrsonaBase::get_max_allowed_vote());
    std::uniform_int_distribution<int> numVoteDistribution(0, numClients);
    size_t numVotes = numVoteDistribution(rng);
    std::vector<size_t> bandwidthData(2);
    bandwidthData[0] = 0;
    bandwidthData[1] = 0;

    // Make the correct number of new votes, but shuffle where they go
    std::vector<Scalar> votes;
    std::vector<bool> replaces;
    for (size_t j = 0; j < numClients; j++)
    {
        votes.push_back(Scalar(voteDistribution(rng)));
        replaces.push_back(j < numVotes);
    }
    shuffle(replaces.begin(), replaces.end(), rng);

    std::vector<size_t> bandwidthDataBefore = get_server_log_data(civetServer.getContext());
    std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
    clock_t cpuTimeBefore = clock();

    // Get current fresh generator (it's not guaranteed we've done this in the current epoch)
    std::vector<Proof> generatorProof;
    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof, bandwidthData);

    // Load fresh generator into client object
    prsonaClient->receive_fresh_generator(generatorProof, freshGenerator);
    
    // Make current short term public key
    Twistpoint shortTermPublicKey = prsonaClient->get_short_term_public_key();

    // Get this client's current encrypted votes
    std::vector<Proof> fullProof;
    std::vector<TwistBipoint> encryptedVotes = get_server_committed_val<std::vector<TwistBipoint>>(rng, serverIPs, serverPorts, REQUEST_VOTE_ROW_URI, REQUEST_VOTE_ROW_COMMITMENT_URI, fullProof, shortTermPublicKey, bandwidthData);

    // Use the client's method to make valid new votes (and their proof)
    std::vector<Proof> voteProof;
    encryptedVotes = prsonaClient->make_votes(voteProof, fullProof, encryptedVotes, votes, replaces);

    // Serialize this data
    std::string data = make_vote_string(voteProof, encryptedVotes, shortTermPublicKey);

    // Send the new votes (and their proof) to the chosen server
    send_item(rng, target, targetPort, SUBMIT_VOTE_URI, data, false, bandwidthData);

    clock_t cpuTimeAfter = clock();
    std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
    std::vector<size_t> bandwidthDataAfter = get_server_log_data(civetServer.getContext());

    std::vector<double> timingData(2);
    timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
    timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;

    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];

    write_log_data(outputMtx, outputFilename, timingData, bandwidthData);
    write_usage_data(usageMtx, usageFilename);
}

bool make_reputation_proof(
    std::default_random_engine& rng,
    PrsonaClient* prsonaClient,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& target,
    int targetPort,
    size_t numClients,
    const CivetServer& civetServer,
    std::mutex& outputMtx,
    const std::string& outputFilename,
    std::mutex& usageMtx,
    const std::string& usageFilename)
{
    std::vector<double> timingData(2);
    std::vector<size_t> bandwidthData(2), bandwidthDataMidA(2), bandwidthDataMidB(2);
    bandwidthData[0] = 0;
    bandwidthData[1] = 0;

    std::string extraOutput = outputFilename + ".extra";

    std::vector<size_t> bandwidthDataBefore = get_server_log_data(civetServer.getContext());
    std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
    clock_t cpuTimeBefore = clock();

    // Get current fresh generator (it's not guaranteed we've done this in the current epoch)
    std::vector<Proof> generatorProof;
    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof, bandwidthData);

    // EXTRA 1
    bandwidthDataMidA = bandwidthData;
    write_log_data(outputMtx, extraOutput, timingData, bandwidthData);

    // Load fresh generator into client object
    prsonaClient->receive_fresh_generator(generatorProof, freshGenerator);

    // Make current short term public key
    Twistpoint shortTermPublicKey = prsonaClient->get_short_term_public_key();

    // Get this client's current encrypted score
    std::vector<Proof> encryptedScoreProof;
    EGCiphertext encryptedScore = get_server_committed_val<EGCiphertext>(rng, serverIPs, serverPorts, REQUEST_CLIENT_TALLY_URI, REQUEST_CLIENT_TALLY_COMMITMENT_URI, encryptedScoreProof, shortTermPublicKey, bandwidthData);

    // EXTRA 2
    bandwidthDataMidB[0] = bandwidthData[0] - bandwidthDataMidA[0];
    bandwidthDataMidB[1] = bandwidthData[1] - bandwidthDataMidA[1];
    bandwidthDataMidA = bandwidthData;
    write_log_data(outputMtx, extraOutput, timingData, bandwidthDataMidB);

    // Load this current encrypted score into client object
    prsonaClient->receive_vote_tally(encryptedScoreProof, encryptedScore);

    // Zero will always be a valid threshold
    Scalar threshold(0);

    // Use client object to generate a correct reputation proof with the chosen parameters
    std::vector<Proof> repProof = prsonaClient->generate_reputation_proof(threshold, numClients);

    // Serialize that proof
    std::string data = make_rep_proof_string(repProof, shortTermPublicKey, threshold);

    // Send that proof to a chosen client (and set up a file to receive whether or not the client accepted the proof)
    char *responseFile = send_item(rng, target, targetPort, VERIFY_REPUTATION_PROOF_URI, data, true, bandwidthData);

    // EXTRA 3
    bandwidthDataMidB[0] = bandwidthData[0] - bandwidthDataMidA[0];
    bandwidthDataMidB[1] = bandwidthData[1] - bandwidthDataMidA[1];
    write_log_data(outputMtx, extraOutput, timingData, bandwidthDataMidB);

    clock_t cpuTimeAfter = clock();
    std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
    std::vector<size_t> bandwidthDataAfter = get_server_log_data(civetServer.getContext());

    timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
    timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;

    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];

    write_log_data(outputMtx, outputFilename, timingData, bandwidthData);
    write_usage_data(usageMtx, usageFilename);

    // The other client will give one byte back, containing whether or not it accepted the proof
    std::ifstream response(responseFile);
    char passed = response.get();

    // Delete the temp file
    remove(responseFile);
    delete [] responseFile;

    return passed == '\x01';
}

/*********************************************************
 *********                                       *********
 *********  client networking private functions  *********
 *********                                       *********
 *********************************************************/

/*
 * HELPERS TO ADD THIS CLIENT TO SERVERS
 */

void register_new_client(
    std::default_random_engine& rng,
    PrsonaClient *newUser,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const Proof& proofOfValidSTPK,
    const Twistpoint& shortTermPublicKey)
{
    struct synchronization_tool sync;
    char *filename = NULL;
    struct mg_connection *conn = NULL;

    // Serialize the relevant data that needs to be sent
    std::stringstream buffer;
    std::string data;
    buffer << proofOfValidSTPK;
    buffer << shortTermPublicKey;
    data = buffer.str();

    // Set up connection to a server
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 0;
    while (!conn)
    {
        // Pick a (pseudo-)random server to register this client with
        std::uniform_int_distribution<size_t> distribution(0, serverIPs.size() - 1);
        size_t whichServer = distribution(rng);

        conn = mg_connect_websocket_client(serverIPs[whichServer].c_str(), serverPorts[whichServer], USE_SSL, NULL, 0, SUBMIT_NEW_CLIENT_URI, "null", file_websocket_data_handler, file_websocket_close_handler, &sync);

        if (!conn)
            std::cerr << "Couldn't register new client" << std::endl;
    }

    // Establish a file to receive proof of addition to system at
    filename = set_temp_filename(rng, conn);

    // Send client data
    mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
    mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

    // Wait for response
    while (!sync.val)
        sync.cv.wait(lck);

    // Close connection
    mg_close_connection(conn);

    // Un-serialize proof of addition to system
    std::vector<Proof> proofOfValidAddition = get_valid_addition_proof_from_file(filename);

    // Remove temp file used to receive serialized data
    remove(filename);
    delete [] filename;

    // Verify that the client was correctly added to the server
    verify_valid_addition(rng, newUser, serverIPs, serverPorts, proofOfValidAddition, shortTermPublicKey);
}

void verify_valid_addition(
    std::default_random_engine& rng,
    PrsonaClient *newUser,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::vector<Proof>& proofOfValidAddition,
    const Twistpoint& shortTermPublicKey)
{
    std::vector<size_t> bandwidthData(2);

    // Get general information on state of system from servers
    std::vector<Proof> serverEncryptedScoreProof;
    CurveBipoint serverEncryptedScore = get_server_committed_val<CurveBipoint>(rng, serverIPs, serverPorts, REQUEST_SERVER_TALLY_URI, REQUEST_SERVER_TALLY_COMMITMENT_URI, serverEncryptedScoreProof, shortTermPublicKey, bandwidthData);

    std::vector<Proof> userEncryptedScoreProof;
    EGCiphertext userEncryptedScore = get_server_committed_val<EGCiphertext>(rng, serverIPs, serverPorts, REQUEST_CLIENT_TALLY_URI, REQUEST_CLIENT_TALLY_COMMITMENT_URI, userEncryptedScoreProof, shortTermPublicKey, bandwidthData);

    std::vector<Proof> voteMatrixProof;
    std::vector<std::vector<TwistBipoint>> voteMatrix = get_server_committed_val<std::vector<std::vector<TwistBipoint>>>(rng, serverIPs, serverPorts, REQUEST_VOTE_MATRIX_URI, REQUEST_VOTE_MATRIX_COMMITMENT_URI, voteMatrixProof, shortTermPublicKey, bandwidthData);

    std::vector<Proof> pseudonymsProof;
    std::vector<Twistpoint> currentPseudonyms = get_server_committed_val<std::vector<Twistpoint>>(rng, serverIPs, serverPorts, REQUEST_PSEUDONYMS_URI, REQUEST_PSEUDONYMS_COMMITMENT_URI, pseudonymsProof, shortTermPublicKey, bandwidthData);

    // Use client's normal verification method
    newUser->receive_new_user_data(proofOfValidAddition, serverEncryptedScoreProof, serverEncryptedScore, userEncryptedScoreProof, userEncryptedScore, voteMatrixProof, voteMatrix, pseudonymsProof, currentPseudonyms);
}

/*
 * GETTERS FOR VARIOUS SERVER VALUES
 */

Twistpoint get_generator(
    std::default_random_engine& rng,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    bool fresh,
    std::vector<Proof>& pi,
    std::vector<size_t>& bandwidthData)
{
    pi.clear();
    struct synchronization_tool sync;
    char *filename = NULL;
    struct mg_connection *conn = NULL;
    const char* whichUri = (fresh ? REQUEST_FRESH_GENERATOR_URI : REQUEST_EG_BLIND_GENERATOR_URI);

    // Set up connection to a server
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 0;
    while (!conn)
    {
        // Pick a (pseudo-)random server to get the generator from
        std::uniform_int_distribution<size_t> distribution(0, serverIPs.size() - 1);
        size_t whichServer = distribution(rng);

        conn = mg_connect_websocket_client(serverIPs[whichServer].c_str(), serverPorts[whichServer], USE_SSL, NULL, 0, whichUri, "null", file_websocket_data_handler, file_websocket_close_handler, &sync);

        if (!conn)
            std::cerr << "Couldn't connect to servers to get generator" << std::endl;
    }

    std::vector<size_t> bandwidthDataBefore = get_conn_log_data(mg_get_context(conn), true);

    // Establish a file to receive generator at
    filename = set_temp_filename(rng, conn);    

    // Tell server to go ahead with data
    mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

    // Wait for data
    while (!sync.val)
        sync.cv.wait(lck);

    std::vector<size_t> bandwidthDataAfter = get_conn_log_data(mg_get_context(conn), true);

    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];

    // Close connection
    mg_close_connection(conn);

    // Un-serialize generator
    Twistpoint retval = get_generator_from_file(filename, pi);

    // Remove temp file used to receive serialized data
    remove(filename);
    delete [] filename;

    return retval;
}

BGNPublicKey get_bgn_public_key(
    std::default_random_engine& rng,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts)
{
    struct synchronization_tool sync;
    char *filename = NULL;
    struct mg_connection *conn = NULL;

    // Set up connection to a server
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 0;
    while (!conn)
    {
        // Pick a (pseudo-)random server to get bgn data from
        std::uniform_int_distribution<size_t> distribution(0, serverIPs.size() - 1);
        size_t whichServer = distribution(rng);

        conn = mg_connect_websocket_client(serverIPs[whichServer].c_str(), serverPorts[whichServer], USE_SSL, NULL, 0, REQUEST_BGN_PUBKEY_URI, "null", file_websocket_data_handler, file_websocket_close_handler, &sync);

        if (!conn)
            std::cerr << "Couldn't connect to servers to obtain BGN details" << std::endl;
    }

    // Establish a file to receive BGN public key at
    filename = set_temp_filename(rng, conn);
        
    // Tell server to go ahead with data
    mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

    // Wait for data
    while (!sync.val)
        sync.cv.wait(lck);

    // Close connection
    mg_close_connection(conn);

    // Un-serialize BGN public key
    BGNPublicKey retval = get_bgn_public_key_from_file(filename);

    // Remove temp file used to receive serialized data
    remove(filename);
    delete [] filename;

    return retval;
}

template <typename T>
T get_server_committed_val(
    std::default_random_engine& rng,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const char *firstUri,
    const char *commitUri,
    std::vector<Proof>& pi,
    const Twistpoint& shortTermPublicKey,
    std::vector<size_t>& bandwidthData)
{
    pi.clear();

    // Pick a (pseudo-)random server to get a committed-to value from
    std::uniform_int_distribution<size_t> distribution(0, serverIPs.size() - 1);
    size_t whichServer = distribution(rng);

    // Get the value itself
    Proof firstProof;
    T retval = get_first_committed_val<T>(rng, serverIPs[whichServer], serverPorts[whichServer], firstUri, firstProof, shortTermPublicKey, bandwidthData);

    // Get all the other server's hashes of the value (to confirm they all agree on it)
    pi.push_back(firstProof);
    get_additional_commitment(rng, serverIPs, serverPorts, serverIPs[whichServer], serverPorts[whichServer], commitUri, pi, shortTermPublicKey, bandwidthData);

    return retval;
}

template EGCiphertext get_server_committed_val<EGCiphertext>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &, std::vector<size_t>&);
template CurveBipoint get_server_committed_val<CurveBipoint>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &, std::vector<size_t>&);
template std::vector<Twistpoint> get_server_committed_val<std::vector<Twistpoint>>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &, std::vector<size_t>&);
template std::vector<TwistBipoint> get_server_committed_val<std::vector<TwistBipoint>>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &, std::vector<size_t>&);
template std::vector<std::vector<TwistBipoint>> get_server_committed_val<std::vector<std::vector<TwistBipoint>>>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &, std::vector<size_t>&);

/*
 * HELPERS FOR GENERALIZED GETTER FUNCTION
 */

template <typename T>
T get_first_committed_val(
    std::default_random_engine& rng,
    const std::string& serverIP,
    int serverPort,
    const char *firstUri,
    Proof& pi,
    const Twistpoint& shortTermPublicKey,
    std::vector<size_t>& bandwidthData)
{
    struct synchronization_tool sync;
    char *filename = NULL;
    struct mg_connection *conn = NULL;

    // Serialize the relevant data that needs to be sent
    std::stringstream buffer;
    std::string data;
    buffer << shortTermPublicKey;
    data = buffer.str();

    // Set up connection to a server
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 0;
    while (!conn)
    {
        conn = mg_connect_websocket_client(serverIP.c_str(), serverPort, USE_SSL, NULL, 0, firstUri, "null", file_websocket_data_handler, file_websocket_close_handler, &sync);

        if (!conn)
            std::cerr << "Trouble getting encrypted score from server at " << serverIP << ":" << serverPort << std::endl;
    }

    std::vector<size_t> bandwidthDataBefore = get_conn_log_data(mg_get_context(conn), true);

    // Establish a file to receive committed-to value at
    filename = set_temp_filename(rng, conn);

    // Send request data
    mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
    mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

    // Wait for response
    while (!sync.val)
        sync.cv.wait(lck);

    std::vector<size_t> bandwidthDataAfter = get_conn_log_data(mg_get_context(conn), true);

    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];

    // Close connection
    mg_close_connection(conn);

    // Un-serialize committed-to value
    T retval = get_committed_val_from_file<T>(filename, pi);

    // Remove temp file used to receive serialized data
    remove(filename);
    delete [] filename;

    return retval;
}

void get_additional_commitment(
    std::default_random_engine& rng,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    const std::string& skipIP,
    int skipPort,
    const char *commitUri,
    std::vector<Proof>& pi,
    const Twistpoint& shortTermPublicKey,
    std::vector<size_t>& bandwidthData)
{
    std::vector<char *> commitmentFilenames;
    std::vector<struct synchronization_tool *> commitmentSyncs;

    // Serialize the relevant data that needs to be sent
    std::stringstream buffer;
    std::string data;
    buffer << shortTermPublicKey;
    data = buffer.str();

    // Ask each server (besides the one we got the value from) what the hash of it is
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        if (serverIPs[i] == skipIP && serverPorts[i] == skipPort)
            continue;

        struct synchronization_tool *currSync = new struct synchronization_tool;
        commitmentSyncs.push_back(currSync);
        struct mg_connection *conn = NULL;

        // Set up connection to a server
        std::unique_lock<std::mutex> lck(currSync->mtx);
        currSync->val = 0;
        while (!conn)
        {
            conn = mg_connect_websocket_client(serverIPs[i].c_str(), serverPorts[i], USE_SSL, NULL, 0, commitUri, "null", file_websocket_data_handler, file_websocket_close_handler, currSync);

            if (!conn)
                std::cerr << "Trouble getting commitment from server at " << serverIPs[i] << ":" << serverPorts[i] << std::endl;
        }

        std::vector<size_t> bandwidthDataBefore = get_conn_log_data(mg_get_context(conn), true);

        // Establish a file to receive hash at
        commitmentFilenames.push_back(set_temp_filename(rng, conn));

        // Send request data
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        // Wait for response
        while (!currSync->val)
            currSync->cv.wait(lck);

        std::vector<size_t> bandwidthDataAfter = get_conn_log_data(mg_get_context(conn), true);

        bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
        bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];

        // Close connection
        mg_close_connection(conn);
    }
    
    for (size_t i = 0; i < commitmentFilenames.size(); i++)
    {
        // Un-serialize hash
        pi.push_back(get_commitment_from_file(commitmentSyncs[i], commitmentFilenames[i]));

        // Clean up the std::mutex used for this hash
        delete commitmentSyncs[i];

        // Delete temp file used to receive serialized data
        remove(commitmentFilenames[i]);
        delete [] commitmentFilenames[i];
    }
}

/*
 * FILE I/O HELPERS FOR ALL GETTERS
 */

std::vector<Proof> get_valid_addition_proof_from_file(
    const char *filename)
{
    std::ifstream additionFile(filename);

    std::vector<Proof> retval;

    BinarySizeT sizeOfVector;
    additionFile >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Proof currProof;
        additionFile >> currProof;
        retval.push_back(currProof);
    }

    return retval;
}

Twistpoint get_generator_from_file(
    const char *filename,
    std::vector<Proof>& pi)
{
    std::ifstream genFile(filename);

    BinarySizeT sizeOfVector;
    genFile >> sizeOfVector;

    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Proof currProof;
        genFile >> currProof;

        pi.push_back(currProof);
    }

    Twistpoint retval;
    genFile >> retval;

    return retval;
}

BGNPublicKey get_bgn_public_key_from_file(
    const char *filename)
{
    std::ifstream bgnFile(filename);

    BGNPublicKey publicKey;
    bgnFile >> publicKey;

    return publicKey;
}

// User-encrytped score
template <>
EGCiphertext get_committed_val_from_file<EGCiphertext>(
    const char *filename,
    Proof& pi)
{
    std::ifstream valFile(filename);

    EGCiphertext retval;
    valFile >> pi;
    valFile >> retval;

    return retval;
}

// Server-encrytped score
template <>
CurveBipoint get_committed_val_from_file<CurveBipoint>(
    const char *filename,
    Proof& pi)
{
    std::ifstream valFile(filename);

    CurveBipoint retval;
    valFile >> pi;
    valFile >> retval;

    return retval;
}

// Current pseudonyms
template <>
std::vector<Twistpoint> get_committed_val_from_file<std::vector<Twistpoint>>(
    const char *filename,
    Proof& pi)
{
    std::ifstream valFile(filename);

    valFile >> pi;

    std::vector<Twistpoint> retval;

    BinarySizeT sizeOfVector;
    valFile >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Twistpoint currVote;
        valFile >> currVote;

        retval.push_back(currVote);
    }

    return retval;
}

// Vote row
template <>
std::vector<TwistBipoint> get_committed_val_from_file<std::vector<TwistBipoint>>(
    const char *filename,
    Proof& pi)
{
    std::ifstream valFile(filename);

    valFile >> pi;

    std::vector<TwistBipoint> retval;

    BinarySizeT sizeOfVector;
    valFile >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        TwistBipoint currVote;
        valFile >> currVote;

        retval.push_back(currVote);
    }

    return retval;
}

// Full vote matrix
template <>
std::vector<std::vector<TwistBipoint>> get_committed_val_from_file<std::vector<std::vector<TwistBipoint>>>(
    const char *filename,
    Proof& pi)
{
    std::ifstream valFile(filename);

    valFile >> pi;

    std::vector<std::vector<TwistBipoint>> retval;

    BinarySizeT sizeOfVector;
    valFile >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        std::vector<TwistBipoint> currRow;
        for (size_t j = 0; j < sizeOfVector.val(); j++)
        {
            TwistBipoint currVote;
            valFile >> currVote;

            currRow.push_back(currVote);    
        }
        retval.push_back(currRow);
    }

    return retval;
}

// NOTE: this function is the weird only case in which its usage will not assume you already have possession of the file lock to read from
Proof get_commitment_from_file(
    struct synchronization_tool *sync,
    const char *filename)
{
    std::unique_lock<std::mutex> lck(sync->mtx);
    std::ifstream scoreFile(filename);

    Proof retval;
    scoreFile >> retval;

    return retval;
}

/*
 * GENERALIZED SENDER FOR ORCHESTRATOR-SIGNALED OPERATIONS
 */

char *send_item(
    std::default_random_engine& rng,
    const std::string& target,
    int targetPort,
    const char* whichUri,
    const std::string& data,
    bool responseExpected,
    std::vector<size_t>& bandwidthData)
{
    struct synchronization_tool sync;
    char *retval = NULL;
    struct mg_connection *conn = NULL;

    // Keep looping until item has been correctly received
    std::unique_lock<std::mutex> lck(sync.mtx);
    sync.val = 0;
    sync.val2 = 0;
    while (!sync.val)
    {
        // Set up connection to a server
        while (!conn)
        {
            if (responseExpected)
                conn = mg_connect_websocket_client(target.c_str(), targetPort, USE_SSL, NULL, 0, whichUri, "null", file_websocket_data_handler, file_websocket_close_handler, &sync);
            else
                conn = mg_connect_websocket_client(target.c_str(), targetPort, USE_SSL, NULL, 0, whichUri, "null", synchro_websocket_data_handler, synchro_websocket_close_handler, &sync);    

            if (!conn)
                std::cerr << "Couldn't connect to server for purposes of sending item." << std::endl;
        }

        std::vector<size_t> bandwidthDataBefore = get_conn_log_data(mg_get_context(conn), false);

        // Set up a file to receive a response at (if it's expected)
        if (responseExpected)
            retval = set_temp_filename(rng, conn);

        // Send request data
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
        mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);

        // Wait until the server has closed the connection
        while (!sync.val2)
            sync.cv.wait(lck);

        std::vector<size_t> bandwidthDataAfter = get_conn_log_data(mg_get_context(conn), false);

        bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
        bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];

        // Close connection
        mg_close_connection(conn);
        conn = NULL;
    }

    return retval;
}

/*
 * DATA SERIALIZERS
 */

std::string make_vote_string(
    const std::vector<Proof>& pi,
    const std::vector<TwistBipoint>& newVotes,
    const Twistpoint& shortTermPublicKey)
{
    std::stringstream buffer;
    BinarySizeT sizeOfVector;

    sizeOfVector.set(pi.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << pi[i];

    sizeOfVector.set(newVotes.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << newVotes[i];

    buffer << shortTermPublicKey;

    // Clients always tell the server they should share the vote row with other servers; servers never tell other servers to do this
    BinaryBool shouldDeal(true);
    buffer << shouldDeal;

    return buffer.str();
}

std::string make_rep_proof_string(
    const std::vector<Proof>& pi,
    const Twistpoint& shortTermPublicKey,
    const Scalar& threshold)
{
    std::stringstream buffer;
    BinarySizeT sizeOfVector;

    sizeOfVector.set(pi.size());
    
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << pi[i];

    buffer << shortTermPublicKey;
    buffer << threshold;

    return buffer.str();
}

/**********************************************************
 ****                                                  ****
 ****  other client-relevant handler member functions  ****
 ****                                                  ****
 **********************************************************/

/*
 * CLIENT READY HANDLER
 */

ClientReadyHandler::ClientReadyHandler(
    struct synchronization_tool *exitSync)
: exitSync(exitSync)
{ /* */ }

bool ClientReadyHandler::handleGet(
    CivetServer *server,
    struct mg_connection *conn)
{
    std::unique_lock<std::mutex> exitLock(exitSync->mtx, std::defer_lock);

    if (!exitLock.try_lock())
    {
        mg_printf(conn, "HTTP/1.1 503 Service Unavailable\r\n"
                        "Content-Type: text/plain\r\n"
                        "Connection: close\r\n\r\n");
        mg_printf(conn, "Client is still making previous votes or a reputation proof.\n");   
    }
    else
    {
        mg_printf(conn, "HTTP/1.1 200 OK\r\n"
                        "Content-Type: text/plain\r\n"
                        "Connection: close\r\n\r\n");
        mg_printf(conn, "Client is ready to move forward.\n");
    }

    return true;
}

/*********************************************************
 ****                                                 ****
 ****  PrsonaClientWebSocketHandler member functions  ****
 ****                                                 ****
 *********************************************************/

/*
 * CONSTRUCTOR
 */

PrsonaClientWebSocketHandler::PrsonaClientWebSocketHandler(
    std::default_random_engine& rng,
    PrsonaClient *prsonaClient,
    const std::vector<std::string>& serverIPs,
    const std::vector<int>& serverPorts,
    std::mutex& outputMtx,
    const std::string& outputFilename,
    std::mutex& usageMtx,
    const std::string& usageFilename)
: rng(rng), prsonaClient(prsonaClient), serverIPs(serverIPs), serverPorts(serverPorts), outputMtx(outputMtx), outputFilename(outputFilename), usageMtx(usageMtx), usageFilename(usageFilename)
{ /* */ }

/*
 * REQUIRED BY INHERITED CLASS
 */

bool PrsonaClientWebSocketHandler::handleConnection(
    CivetServer *server,
    const struct mg_connection *conn)
{
    const struct mg_request_info *info = mg_get_request_info(conn);
    
    // Check if the request being made is something this client can respond to
    bool flag = (info->query_string && info->query_string[0] == PRSONA_VERIFY_REPUTATION_PROOF);
    return flag;
}

void PrsonaClientWebSocketHandler::handleReadyState(
    CivetServer *server,
    struct mg_connection *conn)
{
    const struct mg_request_info *info = mg_get_request_info(conn);

    // Set filenames for query types that will need to un-serialize data to respond correctly
    switch (info->query_string[0])
    {
        case PRSONA_VERIFY_REPUTATION_PROOF:
            set_temp_filename(rng, conn);
            break;

        default:
            mg_set_user_connection_data(conn, NULL);
            break;
    }
}

bool PrsonaClientWebSocketHandler::handleData(
    CivetServer *server,
    struct mg_connection *conn,
    int bits,
    char *data,
    size_t data_len)
{
    char *filename = (char *) mg_get_user_connection_data(conn);
    FILE *currFile = NULL;

    switch (bits & 0xf)
    {
        // Requester has indicated they have sent all relevant data
        case MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE:
        case MG_WEBSOCKET_OPCODE_DATACOMPLETE:
            generate_response(server, conn, filename);
            break;

        // Requester has sent more data (which may theoretically be broken up into multiple packets)
        case MG_WEBSOCKET_OPCODE_BINARY:
        case MG_WEBSOCKET_OPCODE_CONTINUATION:
            currFile = fopen(filename, "ab");
            fwrite(data, sizeof(char), data_len, currFile);
            fclose(currFile);
            return true;

        // Something strange has happened
        default:
            std::cerr << "Unknown packet type received. Failing." << std::endl;
            break;
    }

    return false;
}

void PrsonaClientWebSocketHandler::handleClose(
    CivetServer *server,
    const struct mg_connection *conn)
{
    char *filename = (char *) mg_get_user_connection_data(conn);

    // If we didn't have a temp file for this request, don't do anything
    if (!filename)
        return;

    // If we did, delete it
    remove(filename);
    delete [] filename;
}

/*
 * RESPONSE ROUTER FUNCTION
 */

void PrsonaClientWebSocketHandler::generate_response(
    CivetServer *server,
    struct mg_connection *conn,
    const char *filename)
{
    const struct mg_request_info *info = mg_get_request_info(conn);

    // Select the correct response for this type of request
    switch (info->query_string[0])
    {
        case PRSONA_VERIFY_REPUTATION_PROOF:
            verify_reputation_proof(server, conn, filename);
            break;

        default:
            break;
    }
}

/*
 * REPUTATION PROOF RESPONSE
 */

void PrsonaClientWebSocketHandler::verify_reputation_proof(
    CivetServer *civetServer,
    struct mg_connection *conn,
    const char *filename)
{
    std::vector<Proof> pi;
    Twistpoint shortTermPublicKey;
    Scalar threshold;
    std::vector<size_t> bandwidthData(2);
    bandwidthData[0] = 0;
    bandwidthData[1] = 0;

    // Un-serialize the reputation proof
    std::ifstream file(filename);

    file.ignore(std::numeric_limits<std::streamsize>::max());
    std::streamsize bandwidthRcv = file.gcount();
    file.clear();
    file.seekg(0, std::ios_base::beg);

    BinarySizeT sizeOfVector;
    file >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Proof currProof;
        file >> currProof;

        pi.push_back(currProof);
    }
    file >> shortTermPublicKey;
    file >> threshold;

    std::vector<size_t> bandwidthDataBefore = get_server_log_data(civetServer->getContext());
    std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
    clock_t cpuTimeBefore = clock();

    // Get current fresh generator (it's not guaranteed we've done this in the current epoch)
    std::vector<Proof> generatorProof;
    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof, bandwidthData);

    // Load fresh generator into client object
    prsonaClient->receive_fresh_generator(generatorProof, freshGenerator);

    // Get what the servers say is this user's encrypted score
    std::vector<Proof> encryptedScoreProof;
    EGCiphertext encryptedScore = get_server_committed_val<EGCiphertext>(rng, serverIPs, serverPorts, REQUEST_CLIENT_TALLY_URI, REQUEST_CLIENT_TALLY_COMMITMENT_URI, encryptedScoreProof, shortTermPublicKey, bandwidthData);

    // Check if the proof verifies correctly
    bool flag = prsonaClient->verify_reputation_proof(pi, shortTermPublicKey, threshold, encryptedScoreProof, encryptedScore);

    clock_t cpuTimeAfter = clock();
    std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
    std::vector<size_t> bandwidthDataAfter = get_server_log_data(civetServer->getContext());

    std::vector<double> timingData(2);
    timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
    timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;

    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1] + 1;

    write_log_data(outputMtx, outputFilename, timingData, bandwidthData);
    write_usage_data(usageMtx, usageFilename);

    // Tell the prover whether or not we accept the proof
    std::string data = flag ? "\x01" : "\x00";
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), 1);
    mg_websocket_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
}