PRSONA/prsona/src/serverMain.cpp

#include <atomic>
#include <chrono>
#include <iostream>
#include <fstream>
#include <cstring>
#include <cstdlib>
#include <vector>
#include <string>

#include "networkServer.hpp"

#define BGN_TMP_FILE (TMP_DIR "bgn")
#define GEN_TMP_FILE (TMP_DIR "generator")
#define EPOCH_GEN_TMP_FILE (TMP_DIR "epoch")

using namespace std;

struct synchronization_tool exitSync, bgnSync, generatorSync, readySync, updateSync, epochSync, tallySync;
mutex updateMtx;

atomic<size_t> epochNum(0);

// Initialize the classes we use
void initialize_prsona_classes()
{
    Scalar::init();
    PrsonaBase::init();
    PrsonaBase::set_client_malicious();
}

PrsonaServer *create_server_from_bgn_file(size_t numServers)
{
    unique_lock<mutex> lck(bgnSync.mtx);
    ifstream bgnFile(BGN_TMP_FILE);
    BGN privateKey;
    bgnFile >> privateKey;

    return new PrsonaServer(numServers, privateKey);
}

static int bgn_websocket_data_handler(
    struct mg_connection *conn,
    int bits,
    char *data,
    size_t data_len,
    void *user_data)
{
    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE)
        return false;

    if ((bits & 0xf) != MG_WEBSOCKET_OPCODE_BINARY && (bits & 0xf) != MG_WEBSOCKET_OPCODE_CONTINUATION)
    {
        std::cerr << "Unknown opcode: failing." << std::endl;
        return false;
    }

    unique_lock<mutex> lck(bgnSync.mtx);
    FILE *currFile = fopen(BGN_TMP_FILE, "ab");
    fwrite(data, sizeof(char), data_len, currFile);
    fclose(currFile);

    return true;
}

static void bgn_websocket_close_handler(
    const struct mg_connection *conn,
    void *user_data)
{
    unique_lock<mutex> lck(bgnSync.mtx);
    bgnSync.val = 1;
    bgnSync.cv.notify_all();
}

Twistpoint update_generator_from_gen_file(Proof& pi)
{
    unique_lock<mutex> lck(generatorSync.mtx);
    ifstream genFile(GEN_TMP_FILE);
    Twistpoint retval;

    genFile >> pi;
    genFile >> retval;

    return retval;
}

Twistpoint update_data_from_epoch_gen_file(vector<Proof>& pi)
{
    unique_lock<mutex> lck(epochSync.mtx);
    ifstream epochFile(EPOCH_GEN_TMP_FILE);
    
    Twistpoint retval;
    BinarySizeT sizeOfVector;
    pi.clear();

    epochFile >> sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
    {
        Proof currProof;
        epochFile >> currProof;

        pi.push_back(currProof);
    }
    epochFile >> retval;

    return retval;
}

static int generator_websocket_data_handler(
    struct mg_connection *conn,
    int bits,
    char *data,
    size_t data_len,
    void *user_data)
{
    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE || (bits & 0xf) == MG_WEBSOCKET_OPCODE_DATACOMPLETE)
        return false;

    if ((bits & 0xf) != MG_WEBSOCKET_OPCODE_BINARY && (bits & 0xf) != MG_WEBSOCKET_OPCODE_CONTINUATION)
    {
        std::cerr << "Unknown opcode: failing." << std::endl;
        return false;
    }

    unique_lock<mutex> lck(generatorSync.mtx);
    FILE *currFile = fopen(GEN_TMP_FILE, "ab");
    fwrite(data, sizeof(char), data_len, currFile);
    fclose(currFile);

    return true;
}

static void generator_websocket_close_handler(
    const struct mg_connection *conn,
    void *user_data)
{
    unique_lock<mutex> lck(generatorSync.mtx);
    generatorSync.val = 1;
    generatorSync.cv.notify_all();
}

static int epoch_websocket_data_handler(
    struct mg_connection *conn,
    int bits,
    char *data,
    size_t data_len,
    void *user_data)
{
    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE)
        return false;
    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_DATACOMPLETE)
    {
        unique_lock<mutex> lck(epochSync.mtx);
        epochSync.val++;

        return false;
    }

    if ((bits & 0xf) != MG_WEBSOCKET_OPCODE_BINARY && (bits & 0xf) != MG_WEBSOCKET_OPCODE_CONTINUATION)
    {
        std::cerr << "Unknown opcode: failing." << std::endl;
        return false;
    }

    unique_lock<mutex> lck(epochSync.mtx);
    FILE *currFile = fopen(EPOCH_GEN_TMP_FILE, "ab");
    fwrite(data, sizeof(char), data_len, currFile);
    fclose(currFile);

    return true;
}

static void epoch_websocket_close_handler(
    const struct mg_connection *conn,
    void *user_data)
{
    unique_lock<mutex> lck(epochSync.mtx);
    epochSync.val2 = 0;
    epochSync.cv.notify_all();
}

static int tally_websocket_data_handler(
    struct mg_connection *conn,
    int bits,
    char *data,
    size_t data_len,
    void *user_data)
{
    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_CONNECTION_CLOSE)
        return false;

    if ((bits & 0xf) == MG_WEBSOCKET_OPCODE_DATACOMPLETE)
    {
        unique_lock<mutex> lck(tallySync.mtx);
        tallySync.val++;

        tallySync.cv.notify_all();

        return false;
    }

    if ((bits & 0xf) != MG_WEBSOCKET_OPCODE_BINARY && (bits & 0xf) != MG_WEBSOCKET_OPCODE_CONTINUATION)
    {
        std::cerr << "Unknown opcode: failing." << std::endl;
        return false;
    }

    return true;
}

Twistpoint get_generator(
    vector<Proof>& pi,
    PrsonaServer *prsonaServer,
    const vector<string>& serverIPs,
    const string& selfIP,
    bool fresh)
{
    Twistpoint retval = PrsonaServer::EL_GAMAL_GENERATOR;
    pi.clear();

    if (fresh)
        retval = prsonaServer->add_curr_seed_to_generator(pi, retval);
    else
        retval = prsonaServer->add_rand_seed_to_generator(pi, retval);

    const char* which = (fresh ? GET_FRESH_GEN_URI : GET_BLIND_GEN_URI);

    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        if (serverIPs[i] == selfIP)
            continue;

        bool flag = false;
        while (!flag)
        {
            struct mg_connection *conn =
                mg_connect_websocket_client(
                    serverIPs[i].c_str(),
                    PRSONA_PORT,
                    USE_SSL,
                    NULL,
                    0,
                    which,
                    "null",
                    generator_websocket_data_handler,
                    generator_websocket_close_handler,
                    NULL);

            if (!conn)
            {
                cerr << "Couldn't get server " << i << "'s update on generator" << endl;
                continue;
            }

            stringstream buffer;
            string data;
            buffer << retval;
            data = buffer.str();

            mg_websocket_client_write(
                conn,
                MG_WEBSOCKET_OPCODE_BINARY,
                data.c_str(),
                data.length());

            unique_lock<mutex> lck(generatorSync.mtx);
            remove(GEN_TMP_FILE);
            generatorSync.val = 0;
            mg_websocket_client_write(
                conn,
                MG_WEBSOCKET_OPCODE_DATACOMPLETE,
                "",
                0);

            while (!generatorSync.val)
                generatorSync.cv.wait(lck);

            mg_close_connection(conn);

            flag = true;
        }

        Proof currProof;
        retval = update_generator_from_gen_file(currProof);
        pi.push_back(currProof);
    }

    return retval;
}

void handout_generator(
    const vector<Proof>& pi,
    const Twistpoint generator,
    PrsonaServer *prsonaServer,
    const vector<string> serverIPs,
    string selfIP,
    bool fresh)
{
    if (fresh)
        prsonaServer->initialize_fresh_generator(pi, generator);
    else
        prsonaServer->set_EG_blind_generator(pi, generator);

    stringstream buffer;
    string data;

    const char* which = (fresh ? GIVE_FRESH_GEN_URI : GIVE_BLIND_GEN_URI);
    
    BinarySizeT sizeOfVector(pi.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << pi[i];

    buffer << generator;
    data = buffer.str();

    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        if (serverIPs[i] == selfIP)
            continue;

        bool flag = false;
        while (!flag)
        {
            struct mg_connection *conn =
                mg_connect_websocket_client(
                    serverIPs[i].c_str(),
                    PRSONA_PORT,
                    USE_SSL,
                    NULL,
                    0,
                    which,
                    "null",
                    empty_websocket_data_handler,
                    empty_websocket_close_handler,
                    NULL);

            if (!conn)
            {
                cerr << "Couldn't give " << (fresh ? "fresh" : "blind") << " generator to server " << i << endl;
                continue;
            }

            mg_websocket_client_write(
                conn,
                MG_WEBSOCKET_OPCODE_BINARY,
                data.c_str(),
                data.length());

            mg_websocket_client_write(
                conn,
                MG_WEBSOCKET_OPCODE_DATACOMPLETE,
                "",
                0);

            mg_close_connection(conn);

            flag = true;
        }
    }
}

Twistpoint initiate_epoch_updates(
    const string& recipient,
    const string& data,
    vector<vector<Proof>>& generatorProofHolder,
    bool isBreakdown)
{
    Twistpoint retval;
    struct synchronization_tool epochSync;
    const char* which = (isBreakdown ? EPOCH_BREAK_DOWN_URI : EPOCH_BUILD_UP_URI);

    bool flag = false;
    while (!flag)
    {
        struct mg_connection *conn =
            mg_connect_websocket_client(
                recipient.c_str(),
                PRSONA_PORT,
                USE_SSL,
                NULL,
                0,
                which,
                "null",
                epoch_websocket_data_handler,
                epoch_websocket_close_handler,
                NULL);

        if (!conn)
        {
            std::cerr << "Trouble initiating epoch update with server at " << recipient << std::endl;
            continue;
        }
        
        unique_lock<mutex> lck(epochSync.mtx);
        remove(EPOCH_GEN_TMP_FILE);
        epochSync.val = 0;
        epochSync.val2 = 1;

        mg_websocket_client_write(
            conn,
            MG_WEBSOCKET_OPCODE_BINARY,
            data.c_str(),
            data.length());

        mg_websocket_client_write(
            conn,
            MG_WEBSOCKET_OPCODE_DATACOMPLETE,
            "",
            0);

        while (epochSync.val2)
            epochSync.cv.wait(lck);

        if (!epochSync.val)
            flag = true;

        mg_close_connection(conn);
    }

    if (isBreakdown)
        return retval;

    vector<Proof> generatorProof;
    generatorProofHolder.clear();

    retval = update_data_from_epoch_gen_file(generatorProof);

    generatorProofHolder.push_back(generatorProof);

    return retval;
}

vector<Proof> epoch_build_up(
    PrsonaServer *prsonaServer,
    const vector<string>& serverIPs,
    const string& selfIP,
    Twistpoint& nextGenerator)
{
    std::vector<std::vector<std::vector<Proof>>> pi;
    std::vector<std::vector<std::vector<Twistpoint>>> permutationCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymSeedCommits;
    std::vector<std::vector<std::vector<CurveBipoint>>> serverTallyCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> partwayVoteMatrixCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> finalVoteMatrixCommits;

    std::vector<std::vector<Proof>> generatorProofHolder(1);

    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        if (serverIPs[i] == selfIP)
        {
            pi.clear();
            permutationCommits.clear();
            freshPseudonymCommits.clear();
            freshPseudonymSeedCommits.clear();
            serverTallyCommits.clear();
            partwayVoteMatrixCommits.clear();
            finalVoteMatrixCommits.clear();

            pi.push_back(generatorProofHolder);

            prsonaServer->build_up_midway_pseudonyms(
                pi,
                permutationCommits,
                freshPseudonymCommits,
                freshPseudonymSeedCommits,
                serverTallyCommits,
                partwayVoteMatrixCommits,
                finalVoteMatrixCommits,
                nextGenerator);

            vector<vector<Twistpoint>> currUserTallyMaskCommits;
            vector<vector<Twistpoint>> currUserTallyMessageCommits;
            vector<vector<Twistpoint>> currUserTallySeedCommits;

            string data = make_epoch_update_string(
                pi[1],
                permutationCommits[0],
                freshPseudonymCommits[0],
                freshPseudonymSeedCommits[0],
                serverTallyCommits[0],
                partwayVoteMatrixCommits[0],
                finalVoteMatrixCommits[0],
                currUserTallyMaskCommits,
                currUserTallyMessageCommits,
                currUserTallySeedCommits,
                nextGenerator,
                false);

            struct synchronization_tool epochSync;
            epochSync.val = 1;
            for (size_t j = 0; j < serverIPs.size(); j++)
            {
                if (i == j)
                    continue;

                distribute_epoch_updates(
                    serverIPs[j],
                    data,
                    &epochSync);
            }
            unique_lock<mutex> lck(epochSync.mtx);
            while (epochSync.val < serverIPs.size())
                epochSync.cv.wait(lck);

            generatorProofHolder = pi[0];
        }
        else
        {
            string data = make_epoch_initiator_string(
                generatorProofHolder[0],
                nextGenerator);
            nextGenerator = initiate_epoch_updates(
                serverIPs[i],
                data,
                generatorProofHolder,
                false);
        }
    }

    return generatorProofHolder[0];
}

void epoch_break_down(
    PrsonaServer *prsonaServer,
    const vector<string>& serverIPs,
    const string& selfIP,
    const vector<Proof>& generatorProof,
    const Twistpoint& nextGenerator)
{
    std::vector<std::vector<std::vector<Proof>>> pi;
    std::vector<std::vector<std::vector<Twistpoint>>> permutationCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> freshPseudonymSeedCommits;
    std::vector<std::vector<std::vector<CurveBipoint>>> serverTallyCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> partwayVoteMatrixCommits;
    std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> finalVoteMatrixCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> userTallyMaskCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> userTallyMessageCommits;
    std::vector<std::vector<std::vector<Twistpoint>>> userTallySeedCommits;

    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        pi.clear();
        permutationCommits.clear();
        freshPseudonymCommits.clear();
        freshPseudonymSeedCommits.clear();
        serverTallyCommits.clear();
        partwayVoteMatrixCommits.clear();
        finalVoteMatrixCommits.clear();
        userTallyMaskCommits.clear();
        userTallyMessageCommits.clear();
        userTallySeedCommits.clear();

        if (serverIPs[i] == selfIP)
        {
            prsonaServer->break_down_midway_pseudonyms(
                generatorProof,
                pi,
                permutationCommits,
                freshPseudonymCommits,
                freshPseudonymSeedCommits,
                serverTallyCommits,
                partwayVoteMatrixCommits,
                finalVoteMatrixCommits,
                userTallyMaskCommits,
                userTallyMessageCommits,
                userTallySeedCommits,
                nextGenerator);

            string data = make_epoch_update_string(
                pi[0],
                permutationCommits[0],
                freshPseudonymCommits[0],
                freshPseudonymSeedCommits[0],
                serverTallyCommits[0],
                partwayVoteMatrixCommits[0],
                finalVoteMatrixCommits[0],
                userTallyMaskCommits[0],
                userTallyMessageCommits[0],
                userTallySeedCommits[0],
                nextGenerator,
                true);

            struct synchronization_tool epochSync;
            epochSync.val = 1;
            for (size_t j = 0; j < serverIPs.size(); j++)
            {
                if (i == j)
                    continue;

                distribute_epoch_updates(
                    serverIPs[j],
                    data,
                    &epochSync);
            }
            unique_lock<mutex> lck(epochSync.mtx);
            while (epochSync.val < serverIPs.size())
                epochSync.cv.wait(lck);
        }
        else
        {
            vector<vector<Proof>> unused;
            string data = make_epoch_initiator_string(
                generatorProof,
                nextGenerator);
            initiate_epoch_updates(
                serverIPs[i],
                data,
                unused,
                true);
        }
    }
}

void tally_scores(
    PrsonaServer *prsonaServer,
    const vector<string>& serverIPs,
    const string& selfIP,
    const Twistpoint& nextGenerator,
    std::vector<EGCiphertext>& userTallyScores,
    std::vector<CurveBipoint>& serverTallyScores)
{
    tallySync.val = 0;
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        if (serverIPs[i] == selfIP)
        {
            unique_lock<mutex> lck(tallySync.mtx);
            tallySync.val++;
            continue;
        }
        else
        {
            bool flag = false;
            while (!flag)
            {
                struct mg_connection *conn =
                    mg_connect_websocket_client(
                        serverIPs[i].c_str(),
                        PRSONA_PORT,
                        USE_SSL,
                        NULL,
                        0,
                        GET_DECRYPTION_URI,
                        "null",
                        tally_websocket_data_handler,
                        empty_websocket_close_handler,
                        NULL);

                if (!conn)
                {
                    std::cerr << "Trouble initiating epoch update with server at " << serverIPs[i] << std::endl;
                    continue;
                }

                mg_websocket_client_write(
                    conn,
                    MG_WEBSOCKET_OPCODE_DATACOMPLETE,
                    "",
                    0);

                mg_close_connection(conn);

                flag = true;
            }
        }
    }

    unique_lock<mutex> lck(tallySync.mtx);
    while (tallySync.val < serverIPs.size())
        tallySync.cv.wait(lck);

    std::vector<EGCiphertext> retval;
    std::vector<Twistpoint> currentPseudonyms = prsonaServer->get_current_pseudonyms();
    std::vector<Scalar> decryptedTalliedScores = prsonaServer->tally_scores();
    mpz_class maxScorePossibleThisRound =
        prsonaServer->get_max_possible_score().toInt() *
        PrsonaBase::get_max_allowed_vote();
    mpz_class topOfScoreRange =
        decryptedTalliedScores.size() * PrsonaBase::get_max_allowed_vote();

    userTallyScores.clear();
    serverTallyScores.clear();
    for (size_t i = 0; i < decryptedTalliedScores.size(); i++)
    {
        decryptedTalliedScores[i] =
            Scalar(
                (decryptedTalliedScores[i].toInt() * topOfScoreRange) /
                maxScorePossibleThisRound
            );

        EGCiphertext currCiphertext;
        userTallyScores.push_back(currCiphertext);
        CurveBipoint currServerScore;
        serverTallyScores.push_back(currServerScore);
        Scalar currMask;
        currMask.set_random();

        // Give the server the new weights,
        // to get passed around to the other servers
        prsonaServer->encrypt(
            serverTallyScores[i], decryptedTalliedScores[i]);

        userTallyScores[i].mask = currentPseudonyms[i] * currMask;
        userTallyScores[i].encryptedMessage =
            (nextGenerator * currMask) +
            (prsonaServer->get_blinding_generator() * decryptedTalliedScores[i]);
    }
}

void distribute_tallied_scores(
    PrsonaServer *prsonaServer,
    const vector<string>& serverIPs,
    const string& selfIP,
    const Twistpoint& nextGenerator,
    const std::vector<EGCiphertext>& userTallyScores,
    const std::vector<CurveBipoint>& serverTallyScores)
{
    stringstream buffer;
    string data;
    
    BinarySizeT sizeOfVector(userTallyScores.size());
    buffer << sizeOfVector;
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << userTallyScores[i];
    for (size_t i = 0; i < sizeOfVector.val(); i++)
        buffer << serverTallyScores[i];

    data = buffer.str();

    tallySync.val = 0;
    for (size_t i = 0; i < serverIPs.size(); i++)
    {
        if (serverIPs[i] == selfIP)
        {
            prsonaServer->receive_tallied_scores(userTallyScores, serverTallyScores);
            unique_lock<mutex> lck(tallySync.mtx);
            tallySync.val++;
            continue;
        }
        else
        {
            bool flag = false;
            while (!flag)
            {
                struct mg_connection *conn =
                    mg_connect_websocket_client(
                        serverIPs[i].c_str(),
                        PRSONA_PORT,
                        USE_SSL,
                        NULL,
                        0,
                        GIVE_DECRYPTION_URI,
                        "null",
                        tally_websocket_data_handler,
                        empty_websocket_close_handler,
                        NULL);

                if (!conn)
                {
                    std::cerr << "Trouble initiating epoch update with server at " << serverIPs[i] << std::endl;
                    continue;
                }

                mg_websocket_client_write(
                    conn,
                    MG_WEBSOCKET_OPCODE_BINARY,
                    data.c_str(),
                    data.length());

                mg_websocket_client_write(
                    conn,
                    MG_WEBSOCKET_OPCODE_DATACOMPLETE,
                    "",
                    0);

                mg_close_connection(conn);

                flag = true;
            }
        }
    }

    unique_lock<mutex> lck(tallySync.mtx);
    while (tallySync.val < serverIPs.size())
        tallySync.cv.wait(lck);
}

void epoch(
    PrsonaServer *prsonaServer,
    const vector<string>& serverIPs,
    const string& selfIP)
{
    Twistpoint nextGenerator = PrsonaServer::EL_GAMAL_GENERATOR;
    
    unique_lock<mutex> lck(updateMtx, defer_lock);
    obtain_update_locks(
        lck,
        serverIPs,
        selfIP,
        &updateSync);

    vector<Proof> generatorProof =
        epoch_build_up(
            prsonaServer,
            serverIPs,
            selfIP,
            nextGenerator);

    std::vector<EGCiphertext> currentUserEncryptedTallies;
    std::vector<CurveBipoint> currentServerEncryptedTallies;

    tally_scores(
        prsonaServer,
        serverIPs,
        selfIP,
        nextGenerator,
        currentUserEncryptedTallies,
        currentServerEncryptedTallies);

    distribute_tallied_scores(
        prsonaServer,
        serverIPs,
        selfIP,
        nextGenerator,
        currentUserEncryptedTallies,
        currentServerEncryptedTallies);

    epoch_break_down(
        prsonaServer,
        serverIPs,
        selfIP,
        generatorProof,
        nextGenerator);

    epochNum.fetch_add(1);

    release_update_locks(
        lck,
        serverIPs,
        selfIP,
        &updateSync);
}

class EpochReadyHandler : public CivetHandler
{
    public:
        EpochReadyHandler(size_t numServers)
            : numServers(numServers) { /* */ }

        bool handleGet(CivetServer *server, struct mg_connection *conn)
        {
            unique_lock<mutex> exitLock(exitSync.mtx, defer_lock);
            unique_lock<mutex> readyLock(readySync.mtx);

            if (readySync.val < numServers)
            {
                mg_printf(conn,
                          "HTTP/1.1 503 Service Unavailable\r\nContent-Type: "
                          "text/plain\r\nConnection: close\r\n\r\n");
                mg_printf(conn, "Server is waiting for other servers to begin.\n");
            }
            else if (exitLock.try_lock())
            {
                mg_printf(conn,
                          "HTTP/1.1 200 OK\r\nContent-Type: "
                          "text/plain\r\nConnection: close\r\n\r\n");
                mg_printf(conn, "Server is ready for epoch.\n");
            }
            else
            {
                mg_printf(conn,
                          "HTTP/1.1 503 Service Unavailable\r\nContent-Type: "
                          "text/plain\r\nConnection: close\r\n\r\n");
                mg_printf(conn, "Server is still in a previous epoch.\n");   
            }

            return true;
        }

    private:
        const size_t numServers;

};

class EpochNumHandler : public CivetHandler
{
    public:
        bool handleGet(CivetServer *server, struct mg_connection *conn)
        {
            mg_printf(conn,
                      "HTTP/1.1 200 OK\r\nContent-Type: "
                      "text/plain\r\nConnection: close\r\n\r\n");
            mg_printf(conn, "Epoch num: %lu\n", epochNum.load());

            return true;
        }
};

int main(int argc, char *argv[])
{
    initialize_prsona_classes();

#if USE_SSL
    mg_init_library(0);
#else
    mg_init_library(MG_FEATURES_SSL);
#endif
    
    const char *options[] = {"listening_ports", PRSONA_PORT_STR, 0};

    vector<string> serverIPs;
    string selfIP, dealerIP;

    char buffer[40];
    ifstream serverConfig("serverIPs.cfg");
    while (!serverConfig.eof())
    {
        serverConfig.getline(buffer, 40);
        if (strlen(buffer) > 0)
            serverIPs.push_back(string(buffer));
    }

    ifstream selfConfig("selfIP.cfg");
    while (!selfConfig.eof())
    {
        selfConfig.getline(buffer, 40);
        if (strlen(buffer) > 0)
            selfIP = buffer;
    }

    ifstream dealerConfig("dealerIP.cfg");
    while (!dealerConfig.eof())
    {
        dealerConfig.getline(buffer, 40);
        if (strlen(buffer) > 0)
            dealerIP = buffer;
    }

    // Defaults
    size_t numServers = serverIPs.size();
    bool bgnDealer = selfIP == dealerIP;
    bool maliciousServers = true;

    if (argc > 1)
    {
        bool setting = argv[1][0] == 't' || argv[1][0] == 'T';
        maliciousServers = setting;
    }

    cout << "Establishing PRSONA server with the following parameters: " << endl;
    cout << numServers << " PRSONA servers" << endl;
    cout << "This server " << (bgnDealer ? "IS" : "is NOT") << "the trusted BGN dealer" << endl;
    cout << "Servers are set to " << (maliciousServers ? "MALICIOUS" : "HBC") << " security" << endl;
    cout << "This server is at IP address: " << selfIP << endl;
    cout << "The BGN dealer is at IP address: " << dealerIP << endl;
    cout << endl;

    // Set malicious flags where necessary
    if (maliciousServers)
        PrsonaBase::set_server_malicious();

    cout << "Creating PrsonaServer entity." << endl;

    // Entities we operate with
    PrsonaServer *prsonaServer;
    if (bgnDealer)
        prsonaServer = new PrsonaServer(numServers);
    else
    {
        cout << "Retrieving BGN details." << endl;

        bool flag = false;
        while (!flag)
        {
            struct mg_connection *conn =
                mg_connect_websocket_client(
                    dealerIP.c_str(),
                    PRSONA_PORT,
                    USE_SSL,
                    NULL,
                    0,
                    PRIVATE_BGN_URI,
                    "null",
                    bgn_websocket_data_handler,
                    bgn_websocket_close_handler,
                    NULL);

            if (!conn)
            {
                cerr << "Couldn't obtain BGN details." << endl;
                continue;
            }

            unique_lock<mutex> lck(bgnSync.mtx);
            remove(BGN_TMP_FILE);
            bgnSync.val = 0;
            mg_websocket_client_write(
                conn,
                MG_WEBSOCKET_OPCODE_DATACOMPLETE,
                "",
                0);

            while (!bgnSync.val)
                bgnSync.cv.wait(lck);

            mg_close_connection(conn);

            flag = true;
        }

        prsonaServer = create_server_from_bgn_file(numServers);
    }

    CivetServer server(options);

    PrsonaServerWebSocketHandler wsHandler(prsonaServer, &updateMtx, &epochNum, serverIPs, selfIP);
    server.addWebSocketHandler("/ws", wsHandler);

    if (bgnDealer)
    {
        cout << "Waiting for other servers to check in and retrieve BGN details." << endl;

        unique_lock<mutex> lck(readySync.mtx);
        RemoteControlHandler serverReadyHandler(&readySync, "ACK");
        server.addHandler(SERVER_READY_URI, serverReadyHandler);

        readySync.val++;
        while (readySync.val < numServers)
            readySync.cv.wait(lck);

        vector<Proof> pi;
        Twistpoint freshGenerator =
            get_generator(pi, prsonaServer, serverIPs, selfIP, true);

        handout_generator(pi, freshGenerator, prsonaServer, serverIPs, selfIP, true);

        Twistpoint blindGenerator =
            get_generator(pi, prsonaServer, serverIPs, selfIP, false);

        handout_generator(pi, blindGenerator, prsonaServer, serverIPs, selfIP, false);
    }
    else
    {
        cout << "Notifying BGN dealer that this server is ready." << endl;

        stringstream sysString;
        string data;

        struct mg_connection *conn =
            mg_connect_client(
                dealerIP.c_str(), 
                PRSONA_PORT,
                USE_SSL, 
                NULL,
                0);

        sysString << "GET " << SERVER_READY_URI << " HTTP/1.1\r\n";
        sysString << "Host: " << dealerIP << "\r\n\r\n";
        data = sysString.str();

        mg_write(conn, data.c_str(), data.length());
        mg_close_connection(conn);
    }

    unique_lock<mutex> exitLock(exitSync.mtx);
    exitSync.val = 0;
    exitSync.val2 = 0;
    RemoteControlHandler exitHandler(&exitSync, "Server coming down!");
    server.addHandler(EXIT_URI, exitHandler);

    cout << "Entering main ready loop." << endl;

    if (bgnDealer)
    {
        AltRemoteControlHandler triggerEpochHandler(1, &exitSync, "Server will initiate epoch!");
        server.addHandler(TRIGGER_EPOCH_URI, triggerEpochHandler);

        EpochReadyHandler epochReadyHandler(numServers);
        server.addHandler(EPOCH_READY_URI, epochReadyHandler);

        EpochNumHandler epochNumHandler;
        server.addHandler(WHICH_EPOCH_URI, epochNumHandler);

        while (!exitSync.val)
        {
            while (!exitSync.val && !exitSync.val2)
                exitSync.cv.wait(exitLock);

            if (exitSync.val2)
            {
                cout << "Executing epoch." << endl;

                epoch(prsonaServer, serverIPs, selfIP);

                exitSync.val2 = 0;
            }
        }
    }
    else
    {
        while (!exitSync.val)
            exitSync.cv.wait(exitLock);
    }

    cout << "Shutting down." << endl;

    mg_exit_library();
    delete prsonaServer;

    return 0;
}