fixing some instrumentation stuff

prsona/inc/networkClient.hpp

@@ -72,7 +72,8 @@ Twistpoint get_generator(
     const std::vector<std::string>& serverIPs,
     const std::vector<int>& serverPorts,
     bool fresh,
-    std::vector<Proof>& pi);
+    std::vector<Proof>& pi,
+    std::vector<size_t>& bandwidthData);
 
 BGNPublicKey get_bgn_public_key(
     std::default_random_engine& rng,
@@ -87,17 +88,19 @@ T get_server_committed_val(
     const char *firstUri,
     const char *commitUri,
     std::vector<Proof>& pi,
-    const Twistpoint& shortTermPublicKey);
+    const Twistpoint& shortTermPublicKey,
+    std::vector<size_t>& bandwidthData);
 
 // HELPERS FOR GENERALIZED GETTER FUNCTION
 template <typename T>
 T get_first_committed_val(
     std::default_random_engine& rng,
-    const std::string& server,
+    const std::string& serverIP,
     int serverPort,
     const char *firstUri,
     Proof& pi,
-    const Twistpoint& shortTermPublicKey);
+    const Twistpoint& shortTermPublicKey,
+    std::vector<size_t>& bandwidthData);
 
 void get_additional_commitment(
     std::default_random_engine& rng,
@@ -107,7 +110,8 @@ void get_additional_commitment(
     int skipPort,
     const char *commitUri,
     std::vector<Proof>& pi,
-    const Twistpoint& shortTermPublicKey);
+    const Twistpoint& shortTermPublicKey,
+    std::vector<size_t>& bandwidthData);
 
 // FILE I/O HELPERS FOR ALL GETTERS
 std::vector<Proof> get_valid_addition_proof_from_file(
@@ -136,7 +140,8 @@ char *send_item(
     int targetPort,
     const char* whichUri,
     const std::string& data,
-    bool responseExpected);
+    bool responseExpected,
+    std::vector<size_t>& bandwidthData);
 
 // DATA SERIALIZERS
 std::string make_vote_string(

prsona/inc/networkServer.hpp

@@ -59,14 +59,16 @@ void obtain_update_locks(
     const std::vector<std::string>& serverIPs,
     const std::vector<int>& serverPorts,
     const std::string& selfIP,
-    int selfPort);
+    int selfPort,
+    std::vector<size_t>& bandwidthData);
 
 void release_update_locks(
     std::unique_lock<std::mutex> &updateLock,
     const std::vector<std::string>& serverIPs,
     const std::vector<int>& serverPorts,
     const std::string& selfIP,
-    int selfPort);
+    int selfPort,
+    std::vector<size_t>& bandwidthData);
 
 // GETTER FOR DEALER VALUE
 BGN get_bgn_private_key(
@@ -106,7 +108,8 @@ std::vector<Proof> epoch_build_up(
     Twistpoint& nextGenerator,
     const CivetServer& civetServer,
     std::mutex& outputMtx,
-    const std::string& outputFilename);
+    const std::string& outputFilename,
+    std::vector<size_t>& bandwidthData);
 
 void epoch_break_down(
     std::default_random_engine& rng,
@@ -119,7 +122,8 @@ void epoch_break_down(
     const Twistpoint& nextGenerator,
     const CivetServer& civetServer,
     std::mutex& outputMtx,
-    const std::string& outputFilename);
+    const std::string& outputFilename,
+    std::vector<size_t>& bandwidthData);
 
 // HELPERS FOR EPOCH HELPERS
 Twistpoint initiate_epoch_updates(
@@ -128,7 +132,8 @@ Twistpoint initiate_epoch_updates(
     int recipientPort,
     const std::string& data,
     bool isBreakdown,
-    std::vector<std::vector<Proof>>& generatorProofHolder);
+    std::vector<std::vector<Proof>>& generatorProofHolder,
+    std::vector<size_t>& bandwidthData);
 
 struct mg_connection *distribute_epoch_updates(
     const std::string& recipient,
@@ -145,7 +150,8 @@ void tally_scores(
     int selfPort,
     const Twistpoint& nextGenerator,
     std::vector<EGCiphertext>& userTallyScores,
-    std::vector<CurveBipoint>& serverTallyScores);
+    std::vector<CurveBipoint>& serverTallyScores,
+    std::vector<size_t>& bandwidthData);
 
 void distribute_tallied_scores(
     PrsonaServer *prsonaServer,
@@ -155,7 +161,8 @@ void distribute_tallied_scores(
     int selfPort,
     const Twistpoint& nextGenerator,
     const std::vector<EGCiphertext>& userTallyScores,
-    const std::vector<CurveBipoint>& serverTallyScores);
+    const std::vector<CurveBipoint>& serverTallyScores,
+    std::vector<size_t>& bandwidthData);
 
 // FILE I/O HELPERS
 BGN get_bgn_private_key_from_file(
@@ -447,7 +454,8 @@ class PrsonaServerWebSocketHandler : public CivetWebSocketHandler  {
         void distribute_new_vote(
             std::vector<Proof> pi,
             std::vector<TwistBipoint> newVotes,
-            Twistpoint shortTermPublicKey
+            Twistpoint shortTermPublicKey,
+            std::vector<size_t>& bandwidthData
         ) const;
 
         void import_new_user_update(

prsona/src/networkClient.cpp

@@ -20,19 +20,20 @@ PrsonaClient *create_client(
     const std::vector<int>& serverPorts,
     size_t numServers)
 {
+    std::vector<size_t> bandwidthData;
     // Get the servers' public BGN key
     BGNPublicKey publicKey = get_bgn_public_key(rng, serverIPs, serverPorts);
 
     // Get the H point used in ElGamal operations
     std::vector<Proof> generatorProof;
-    Twistpoint blindGenerator = get_generator(rng, serverIPs, serverPorts, false, generatorProof);
+    Twistpoint blindGenerator = get_generator(rng, serverIPs, serverPorts, false, generatorProof, bandwidthData);
 
     // Make the actual client object
     PrsonaClient *retval = new PrsonaClient(generatorProof, blindGenerator, publicKey, numServers);
 
     // Get the current fresh generator
     generatorProof.clear();
-    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof);
+    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof, bandwidthData);
 
     // Load this fresh generator into the client object
     retval->receive_fresh_generator(generatorProof, freshGenerator);
@@ -66,6 +67,7 @@ void make_vote(
     std::uniform_int_distribution<int> voteDistribution(0, PrsonaBase::get_max_allowed_vote());
     std::uniform_int_distribution<int> numVoteDistribution(0, numClients);
     size_t numVotes = numVoteDistribution(rng);
+    std::vector<size_t> bandwidthData(2);
 
     // Make the correct number of new votes, but shuffle where they go
     std::vector<Scalar> votes;
@@ -83,7 +85,7 @@ void make_vote(
 
     // Get current fresh generator (it's not guaranteed we've done this in the current epoch)
     std::vector<Proof> generatorProof;
-    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof);
+    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof, bandwidthData);
 
     // Load fresh generator into client object
     prsonaClient->receive_fresh_generator(generatorProof, freshGenerator);
@@ -93,7 +95,7 @@ void make_vote(
 
     // Get this client's current encrypted votes
     std::vector<Proof> fullProof;
-    std::vector<TwistBipoint> encryptedVotes = get_server_committed_val<std::vector<TwistBipoint>>(rng, serverIPs, serverPorts, REQUEST_VOTE_ROW_URI, REQUEST_VOTE_ROW_COMMITMENT_URI, fullProof, shortTermPublicKey);
+    std::vector<TwistBipoint> encryptedVotes = get_server_committed_val<std::vector<TwistBipoint>>(rng, serverIPs, serverPorts, REQUEST_VOTE_ROW_URI, REQUEST_VOTE_ROW_COMMITMENT_URI, fullProof, shortTermPublicKey, bandwidthData);
 
     // Use the client's method to make valid new votes (and their proof)
     std::vector<Proof> voteProof;
@@ -103,7 +105,7 @@ void make_vote(
     std::string data = make_vote_string(voteProof, encryptedVotes, shortTermPublicKey);
 
     // Send the new votes (and their proof) to the chosen server
-    send_item(rng, target, targetPort, SUBMIT_VOTE_URI, data, false);
+    send_item(rng, target, targetPort, SUBMIT_VOTE_URI, data, false, bandwidthData);
 
     clock_t cpuTimeAfter = clock();
     std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
@@ -113,11 +115,8 @@ void make_vote(
     timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
     timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;
 
-    std::vector<size_t> bandwidthData(2);
-    std::cout << "Bytes read before: " << bandwidthDataBefore[0] << std::endl;
-    std::cout << "Bytes read after: " << bandwidthDataAfter[0] << std::endl;
-    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0];
-    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];
+    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
+    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
 
     write_log_data(outputMtx, outputFilename, timingData, bandwidthData);
 }
@@ -134,13 +133,15 @@ bool make_reputation_proof(
     std::mutex& outputMtx,
     const std::string& outputFilename)
 {
+    std::vector<size_t> bandwidthData(2);
+
     std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer.getContext());
     std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
     clock_t cpuTimeBefore = clock();
 
     // Get current fresh generator (it's not guaranteed we've done this in the current epoch)
     std::vector<Proof> generatorProof;
-    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof);
+    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof, bandwidthData);
 
     // Load fresh generator into client object
     prsonaClient->receive_fresh_generator(generatorProof, freshGenerator);
@@ -150,7 +151,7 @@ bool make_reputation_proof(
 
     // Get this client's current encrypted score
     std::vector<Proof> encryptedScoreProof;
-    EGCiphertext encryptedScore = get_server_committed_val<EGCiphertext>(rng, serverIPs, serverPorts, REQUEST_CLIENT_TALLY_URI, REQUEST_CLIENT_TALLY_COMMITMENT_URI, encryptedScoreProof, shortTermPublicKey);
+    EGCiphertext encryptedScore = get_server_committed_val<EGCiphertext>(rng, serverIPs, serverPorts, REQUEST_CLIENT_TALLY_URI, REQUEST_CLIENT_TALLY_COMMITMENT_URI, encryptedScoreProof, shortTermPublicKey, bandwidthData);
 
     // Load this current encrypted score into client object
     prsonaClient->receive_vote_tally(encryptedScoreProof, encryptedScore);
@@ -170,7 +171,7 @@ bool make_reputation_proof(
     std::string data = make_rep_proof_string(repProof, shortTermPublicKey, threshold);
 
     // Send that proof to a chosen client (and set up a file to receive whether or not the client accepted the proof)
-    char *responseFile = send_item(rng, target, targetPort, VERIFY_REPUTATION_PROOF_URI, data, true);
+    char *responseFile = send_item(rng, target, targetPort, VERIFY_REPUTATION_PROOF_URI, data, true, bandwidthData);
 
     clock_t cpuTimeAfter = clock();
     std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
@@ -180,9 +181,8 @@ bool make_reputation_proof(
     timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
     timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;
 
-    std::vector<size_t> bandwidthData(2);
-    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0];
-    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];
+    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
+    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
 
     write_log_data(outputMtx, outputFilename, timingData, bandwidthData);
 
@@ -274,18 +274,20 @@ void verify_valid_addition(
     const std::vector<Proof>& proofOfValidAddition,
     const Twistpoint& shortTermPublicKey)
 {
+    std::vector<size_t> bandwidthData(2);
+
     // Get general information on state of system from servers
     std::vector<Proof> serverEncryptedScoreProof;
-    CurveBipoint serverEncryptedScore = get_server_committed_val<CurveBipoint>(rng, serverIPs, serverPorts, REQUEST_SERVER_TALLY_URI, REQUEST_SERVER_TALLY_COMMITMENT_URI, serverEncryptedScoreProof, shortTermPublicKey);
+    CurveBipoint serverEncryptedScore = get_server_committed_val<CurveBipoint>(rng, serverIPs, serverPorts, REQUEST_SERVER_TALLY_URI, REQUEST_SERVER_TALLY_COMMITMENT_URI, serverEncryptedScoreProof, shortTermPublicKey, bandwidthData);
 
     std::vector<Proof> userEncryptedScoreProof;
-    EGCiphertext userEncryptedScore = get_server_committed_val<EGCiphertext>(rng, serverIPs, serverPorts, REQUEST_CLIENT_TALLY_URI, REQUEST_CLIENT_TALLY_COMMITMENT_URI, userEncryptedScoreProof, shortTermPublicKey);
+    EGCiphertext userEncryptedScore = get_server_committed_val<EGCiphertext>(rng, serverIPs, serverPorts, REQUEST_CLIENT_TALLY_URI, REQUEST_CLIENT_TALLY_COMMITMENT_URI, userEncryptedScoreProof, shortTermPublicKey, bandwidthData);
 
     std::vector<Proof> voteMatrixProof;
-    std::vector<std::vector<TwistBipoint>> voteMatrix = get_server_committed_val<std::vector<std::vector<TwistBipoint>>>(rng, serverIPs, serverPorts, REQUEST_VOTE_MATRIX_URI, REQUEST_VOTE_MATRIX_COMMITMENT_URI, voteMatrixProof, shortTermPublicKey);
+    std::vector<std::vector<TwistBipoint>> voteMatrix = get_server_committed_val<std::vector<std::vector<TwistBipoint>>>(rng, serverIPs, serverPorts, REQUEST_VOTE_MATRIX_URI, REQUEST_VOTE_MATRIX_COMMITMENT_URI, voteMatrixProof, shortTermPublicKey, bandwidthData);
 
     std::vector<Proof> pseudonymsProof;
-    std::vector<Twistpoint> currentPseudonyms = get_server_committed_val<std::vector<Twistpoint>>(rng, serverIPs, serverPorts, REQUEST_PSEUDONYMS_URI, REQUEST_PSEUDONYMS_COMMITMENT_URI, pseudonymsProof, shortTermPublicKey);
+    std::vector<Twistpoint> currentPseudonyms = get_server_committed_val<std::vector<Twistpoint>>(rng, serverIPs, serverPorts, REQUEST_PSEUDONYMS_URI, REQUEST_PSEUDONYMS_COMMITMENT_URI, pseudonymsProof, shortTermPublicKey, bandwidthData);
 
     // Use client's normal verification method
     newUser->receive_new_user_data(proofOfValidAddition, serverEncryptedScoreProof, serverEncryptedScore, userEncryptedScoreProof, userEncryptedScore, voteMatrixProof, voteMatrix, pseudonymsProof, currentPseudonyms);
@@ -300,7 +302,8 @@ Twistpoint get_generator(
     const std::vector<std::string>& serverIPs,
     const std::vector<int>& serverPorts,
     bool fresh,
-    std::vector<Proof>& pi)
+    std::vector<Proof>& pi,
+    std::vector<size_t>& bandwidthData)
 {
     pi.clear();
     struct synchronization_tool sync;
@@ -323,6 +326,8 @@ Twistpoint get_generator(
             std::cerr << "Couldn't connect to servers to get generator" << std::endl;
     }
 
+    std::vector<size_t> bandwidthDataBefore = get_log_data(mg_get_context(conn));
+
     // Establish a file to receive generator at
     filename = set_temp_filename(rng, conn);    
 
@@ -333,6 +338,11 @@ Twistpoint get_generator(
     while (!sync.val)
         sync.cv.wait(lck);
 
+    std::vector<size_t> bandwidthDataAfter = get_log_data(mg_get_context(conn));
+
+    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
+    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
+
     // Close connection
     mg_close_connection(conn);
 
@@ -401,7 +411,8 @@ T get_server_committed_val(
     const char *firstUri,
     const char *commitUri,
     std::vector<Proof>& pi,
-    const Twistpoint& shortTermPublicKey)
+    const Twistpoint& shortTermPublicKey,
+    std::vector<size_t>& bandwidthData)
 {
     pi.clear();
 
@@ -411,20 +422,20 @@ T get_server_committed_val(
 
     // Get the value itself
     Proof firstProof;
-    T retval = get_first_committed_val<T>(rng, serverIPs[whichServer], serverPorts[whichServer], firstUri, firstProof, shortTermPublicKey);
+    T retval = get_first_committed_val<T>(rng, serverIPs[whichServer], serverPorts[whichServer], firstUri, firstProof, shortTermPublicKey, bandwidthData);
 
     // Get all the other server's hashes of the value (to confirm they all agree on it)
     pi.push_back(firstProof);
-    get_additional_commitment(rng, serverIPs, serverPorts, serverIPs[whichServer], serverPorts[whichServer], commitUri, pi, shortTermPublicKey);
+    get_additional_commitment(rng, serverIPs, serverPorts, serverIPs[whichServer], serverPorts[whichServer], commitUri, pi, shortTermPublicKey, bandwidthData);
 
     return retval;
 }
 
-template EGCiphertext get_server_committed_val<EGCiphertext>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &);
-template CurveBipoint get_server_committed_val<CurveBipoint>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &);
-template std::vector<Twistpoint> get_server_committed_val<std::vector<Twistpoint>>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &);
-template std::vector<TwistBipoint> get_server_committed_val<std::vector<TwistBipoint>>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &);
-template std::vector<std::vector<TwistBipoint>> get_server_committed_val<std::vector<std::vector<TwistBipoint>>>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &);
+template EGCiphertext get_server_committed_val<EGCiphertext>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &, std::vector<size_t>&);
+template CurveBipoint get_server_committed_val<CurveBipoint>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &, std::vector<size_t>&);
+template std::vector<Twistpoint> get_server_committed_val<std::vector<Twistpoint>>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &, std::vector<size_t>&);
+template std::vector<TwistBipoint> get_server_committed_val<std::vector<TwistBipoint>>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &, std::vector<size_t>&);
+template std::vector<std::vector<TwistBipoint>> get_server_committed_val<std::vector<std::vector<TwistBipoint>>>(std::default_random_engine &, const std::vector<std::string> &, const std::vector<int> &, const char *, const char *, std::vector<Proof> &, const Twistpoint &, std::vector<size_t>&);
 
 /*
  * HELPERS FOR GENERALIZED GETTER FUNCTION
@@ -437,7 +448,8 @@ T get_first_committed_val(
     int serverPort,
     const char *firstUri,
     Proof& pi,
-    const Twistpoint& shortTermPublicKey)
+    const Twistpoint& shortTermPublicKey,
+    std::vector<size_t>& bandwidthData)
 {
     struct synchronization_tool sync;
     char *filename = NULL;
@@ -460,6 +472,8 @@ T get_first_committed_val(
             std::cerr << "Trouble getting encrypted score from server at " << serverIP << ":" << serverPort << std::endl;
     }
 
+    std::vector<size_t> bandwidthDataBefore = get_log_data(mg_get_context(conn));
+
     // Establish a file to receive committed-to value at
     filename = set_temp_filename(rng, conn);
 
@@ -471,6 +485,11 @@ T get_first_committed_val(
     while (!sync.val)
         sync.cv.wait(lck);
 
+    std::vector<size_t> bandwidthDataAfter = get_log_data(mg_get_context(conn));
+
+    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
+    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
+
     // Close connection
     mg_close_connection(conn);
 
@@ -492,7 +511,8 @@ void get_additional_commitment(
     int skipPort,
     const char *commitUri,
     std::vector<Proof>& pi,
-    const Twistpoint& shortTermPublicKey)
+    const Twistpoint& shortTermPublicKey,
+    std::vector<size_t>& bandwidthData)
 {
     std::vector<char *> commitmentFilenames;
     std::vector<struct synchronization_tool *> commitmentSyncs;
@@ -524,6 +544,8 @@ void get_additional_commitment(
                 std::cerr << "Trouble getting commitment from server at " << serverIPs[i] << ":" << serverPorts[i] << std::endl;
         }
 
+        std::vector<size_t> bandwidthDataBefore = get_log_data(mg_get_context(conn));
+
         // Establish a file to receive hash at
         commitmentFilenames.push_back(set_temp_filename(rng, conn));
 
@@ -535,6 +557,11 @@ void get_additional_commitment(
         while (!currSync->val)
             currSync->cv.wait(lck);
 
+        std::vector<size_t> bandwidthDataAfter = get_log_data(mg_get_context(conn));
+
+        bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
+        bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
+
         // Close connection
         mg_close_connection(conn);
     }
@@ -744,7 +771,8 @@ char *send_item(
     int targetPort,
     const char* whichUri,
     const std::string& data,
-    bool responseExpected)
+    bool responseExpected,
+    std::vector<size_t>& bandwidthData)
 {
     struct synchronization_tool sync;
     char *retval = NULL;
@@ -768,6 +796,8 @@ char *send_item(
                 std::cerr << "Couldn't connect to server for purposes of sending item." << std::endl;
         }
 
+        std::vector<size_t> bandwidthDataBefore = get_log_data(mg_get_context(conn));
+
         // Set up a file to receive a response at (if it's expected)
         if (responseExpected)
             retval = set_temp_filename(rng, conn);
@@ -780,6 +810,11 @@ char *send_item(
         while (!sync.val2)
             sync.cv.wait(lck);
 
+        std::vector<size_t> bandwidthDataAfter = get_log_data(mg_get_context(conn));
+
+        bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
+        bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
+
         // Close connection
         mg_close_connection(conn);
         conn = NULL;
@@ -978,6 +1013,7 @@ void PrsonaClientWebSocketHandler::verify_reputation_proof(
     std::vector<Proof> pi;
     Twistpoint shortTermPublicKey;
     Scalar threshold;
+    std::vector<size_t> bandwidthData(2);
 
     // Un-serialize the reputation proof
     std::ifstream file(filename);
@@ -1005,14 +1041,14 @@ void PrsonaClientWebSocketHandler::verify_reputation_proof(
 
     // Get current fresh generator (it's not guaranteed we've done this in the current epoch)
     std::vector<Proof> generatorProof;
-    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof);
+    Twistpoint freshGenerator = get_generator(rng, serverIPs, serverPorts, true, generatorProof, bandwidthData);
 
     // Load fresh generator into client object
     prsonaClient->receive_fresh_generator(generatorProof, freshGenerator);
 
     // Get what the servers say is this user's encrypted score
     std::vector<Proof> encryptedScoreProof;
-    EGCiphertext encryptedScore = get_server_committed_val<EGCiphertext>(rng, serverIPs, serverPorts, REQUEST_CLIENT_TALLY_URI, REQUEST_CLIENT_TALLY_COMMITMENT_URI, encryptedScoreProof, shortTermPublicKey);
+    EGCiphertext encryptedScore = get_server_committed_val<EGCiphertext>(rng, serverIPs, serverPorts, REQUEST_CLIENT_TALLY_URI, REQUEST_CLIENT_TALLY_COMMITMENT_URI, encryptedScoreProof, shortTermPublicKey, bandwidthData);
 
     // Check if the proof verifies correctly
     bool flag = prsonaClient->verify_reputation_proof(pi, shortTermPublicKey, threshold, encryptedScoreProof, encryptedScore);
@@ -1025,9 +1061,8 @@ void PrsonaClientWebSocketHandler::verify_reputation_proof(
     timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
     timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;
 
-    std::vector<size_t> bandwidthData(2);
-    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
-    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1] + 1;
+    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
+    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1] + 1;
 
     write_log_data(outputMtx, outputFilename, timingData, bandwidthData);
 

prsona/src/networkServer.cpp

@@ -108,6 +108,8 @@ void make_epoch(
     // As before, the fresh generator always starts from the same G
     Twistpoint nextGenerator = PrsonaServer::EL_GAMAL_GENERATOR;
 
+    std::vector<size_t> bandwidthData(2);
+
     std::unique_lock<std::mutex> updateLock(updateMtx, std::defer_lock);
 
     std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer.getContext());
@@ -115,27 +117,27 @@ void make_epoch(
     clock_t cpuTimeBefore = clock();
 
     // Take update locks on every machine
-    obtain_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);
+    obtain_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort, bandwidthData);
 
     // Do the first half of the epoch calculations (building up the intermediary values)
-    std::vector<Proof> generatorProof = epoch_build_up(rng, prsonaServer, serverIPs, serverPorts, selfIP, selfPort, nextGenerator, civetServer, buildUpOutputMtx, buildUpOutputFilename);
+    std::vector<Proof> generatorProof = epoch_build_up(rng, prsonaServer, serverIPs, serverPorts, selfIP, selfPort, nextGenerator, civetServer, buildUpOutputMtx, buildUpOutputFilename, bandwidthData);
 
     // Tally up the current scores at the end of the epoch for the users
     std::vector<EGCiphertext> currentUserEncryptedTallies;
     std::vector<CurveBipoint> currentServerEncryptedTallies;
-    tally_scores(prsonaServer, serverIPs, serverPorts, selfIP, selfPort, nextGenerator, currentUserEncryptedTallies, currentServerEncryptedTallies);
+    tally_scores(prsonaServer, serverIPs, serverPorts, selfIP, selfPort, nextGenerator, currentUserEncryptedTallies, currentServerEncryptedTallies, bandwidthData);
 
     // And distribute these to each server
-    distribute_tallied_scores(prsonaServer, serverIPs, serverPorts, selfIP, selfPort, nextGenerator, currentUserEncryptedTallies, currentServerEncryptedTallies);
+    distribute_tallied_scores(prsonaServer, serverIPs, serverPorts, selfIP, selfPort, nextGenerator, currentUserEncryptedTallies, currentServerEncryptedTallies, bandwidthData);
 
     // Do the second half of the epoch calculations (breaking down values to their final values, to be given to users)
-    epoch_break_down(rng, prsonaServer, serverIPs, serverPorts, selfIP, selfPort, generatorProof, nextGenerator, civetServer, breakDownOutputMtx, breakDownOutputFilename);
+    epoch_break_down(rng, prsonaServer, serverIPs, serverPorts, selfIP, selfPort, generatorProof, nextGenerator, civetServer, breakDownOutputMtx, breakDownOutputFilename, bandwidthData);
 
     // Indicate we are in a new epoch
     epochNum.fetch_add(1);
 
     // Release the update locks from every machine
-    release_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);
+    release_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort, bandwidthData);
     
     clock_t cpuTimeAfter = clock();
     std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
@@ -145,9 +147,8 @@ void make_epoch(
     timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
     timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;
 
-    std::vector<size_t> bandwidthData(2);
-    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0];
-    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];
+    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
+    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
 
     write_log_data(fullOutputMtx, fullOutputFilename, timingData, bandwidthData);
 }
@@ -167,7 +168,8 @@ void obtain_update_locks(
     const std::vector<std::string>& serverIPs,
     const std::vector<int>& serverPorts,
     const std::string& selfIP,
-    int selfPort)
+    int selfPort,
+    std::vector<size_t>& bandwidthData)
 {
     // Get locks on each machine (in a predetermined order, defined universally for all servers)
     size_t i = 0;
@@ -196,6 +198,8 @@ void obtain_update_locks(
                 std::cerr << "Couldn't connect to server " << i << " to obtain its lock" << std::endl;
         }
 
+        std::vector<size_t> bandwidthDataBefore = get_log_data(mg_get_context(conn));
+
         // Ask for its lock
         mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
 
@@ -203,6 +207,11 @@ void obtain_update_locks(
         while (!sync.val2)
             sync.cv.wait(lck);
 
+        std::vector<size_t> bandwidthDataAfter = get_log_data(mg_get_context(conn));
+
+        bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
+        bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
+
         // Close connection
         mg_close_connection(conn);
 
@@ -217,7 +226,8 @@ void release_update_locks(
     const std::vector<std::string>& serverIPs,
     const std::vector<int>& serverPorts,
     const std::string& selfIP,
-    int selfPort)
+    int selfPort,
+    std::vector<size_t>& bandwidthData)
 {
     // Release locks on each machine (in the opposite of the predetermined order we used to take them)
     ssize_t i = serverIPs.size() - 1;
@@ -246,6 +256,8 @@ void release_update_locks(
                 std::cerr << "Couldn't connect to server " << i << " to release its lock" << std::endl;
         }
 
+        std::vector<size_t> bandwidthDataBefore = get_log_data(mg_get_context(conn));
+
         // Return its lock
         mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
 
@@ -253,6 +265,11 @@ void release_update_locks(
         while (!sync.val2)
             sync.cv.wait(lck);
 
+        std::vector<size_t> bandwidthDataAfter = get_log_data(mg_get_context(conn));
+
+        bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
+        bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
+
         // Close connection
         mg_close_connection(conn);
 
@@ -461,7 +478,8 @@ std::vector<Proof> epoch_build_up(
     Twistpoint& nextGenerator,
     const CivetServer& civetServer,
     std::mutex& outputMtx,
-    const std::string& outputFilename)
+    const std::string& outputFilename,
+    std::vector<size_t>& overallBandwidthData)
 {
     std::vector<std::vector<std::vector<Proof>>> pi;
     std::vector<std::vector<std::vector<Twistpoint>>> permutationCommits;
@@ -487,8 +505,10 @@ std::vector<Proof> epoch_build_up(
             serverTallyCommits.clear();
             partwayVoteMatrixCommits.clear();
             finalVoteMatrixCommits.clear();
+            std::vector<size_t> bandwidthData(2);
+            std::vector<std::vector<size_t>> otherBandwidthDataBefore;
 
-            std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer.getContext());
+            std::vector<size_t> serverBandwidthDataBefore = get_log_data(civetServer.getContext());
             std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
             clock_t cpuTimeBefore = clock();
             
@@ -517,6 +537,8 @@ std::vector<Proof> epoch_build_up(
                 // Send that data
                 struct mg_connection *currConn = distribute_epoch_updates(serverIPs[j], serverPorts[j], data, &sync);
 
+                otherBandwidthDataBefore.push_back(get_log_data(mg_get_context(currConn)));
+
                 // But keep track of that connection, as we can't close it until we know the server's gotten its data
                 conns.push_back(currConn);
             }
@@ -526,19 +548,27 @@ std::vector<Proof> epoch_build_up(
                 sync.cv.wait(lck);
 
             for (size_t j = 0; j < conns.size(); j++)
+            {
+                std::vector<size_t> currBandwidthDataAfter = get_log_data(mg_get_context(conns[j]));
+
+                bandwidthData[0] += currBandwidthDataAfter[0] - otherBandwidthDataBefore[j][0];
+                bandwidthData[1] += currBandwidthDataAfter[1] - otherBandwidthDataBefore[j][1];
+                overallBandwidthData[0] += currBandwidthDataAfter[0] - otherBandwidthDataBefore[j][0];
+                overallBandwidthData[1] += currBandwidthDataAfter[1] - otherBandwidthDataBefore[j][1];
+
                 mg_close_connection(conns[j]);
+            }
 
             clock_t cpuTimeAfter = clock();
             std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
-            std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer.getContext());
+            std::vector<size_t> serverBandwidthDataAfter = get_log_data(civetServer.getContext());
 
             std::vector<double> timingData(2);
             timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
             timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;
 
-            std::vector<size_t> bandwidthData(2);
-            bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0];
-            bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];
+            bandwidthData[0] += serverBandwidthDataAfter[0] - serverBandwidthDataBefore[0];
+            bandwidthData[1] += serverBandwidthDataAfter[1] - serverBandwidthDataBefore[1];
 
             write_log_data(outputMtx, outputFilename, timingData, bandwidthData);
 
@@ -551,7 +581,7 @@ std::vector<Proof> epoch_build_up(
             std::string data = make_epoch_initiator_string(generatorProofHolder[0], nextGenerator);
 
             // And have them do that request
-            nextGenerator = initiate_epoch_updates(rng, serverIPs[i], serverPorts[i], data, false, generatorProofHolder);
+            nextGenerator = initiate_epoch_updates(rng, serverIPs[i], serverPorts[i], data, false, generatorProofHolder, overallBandwidthData);
         }
     }
 
@@ -570,7 +600,8 @@ void epoch_break_down(
     const Twistpoint& nextGenerator,
     const CivetServer& civetServer,
     std::mutex& outputMtx,
-    const std::string& outputFilename)
+    const std::string& outputFilename,
+    std::vector<size_t>& overallBandwidthData)
 {
     std::vector<std::vector<std::vector<Proof>>> pi;
     std::vector<std::vector<std::vector<Twistpoint>>> permutationCommits;
@@ -599,8 +630,10 @@ void epoch_break_down(
             userTallyMaskCommits.clear();
             userTallyMessageCommits.clear();
             userTallySeedCommits.clear();
+            std::vector<size_t> bandwidthData(2);
+            std::vector<std::vector<size_t>> otherBandwidthDataBefore;
 
-            std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer.getContext());
+            std::vector<size_t> serverBandwidthDataBefore = get_log_data(civetServer.getContext());
             std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
             clock_t cpuTimeBefore = clock();
 
@@ -625,6 +658,8 @@ void epoch_break_down(
                 // Send that data
                 struct mg_connection *currConn = distribute_epoch_updates(serverIPs[j], serverPorts[j], data, &sync);
 
+                otherBandwidthDataBefore.push_back(get_log_data(mg_get_context(currConn)));
+
                 // But keep track of that connection, as we can't close it until we know the server's gotten its data
                 conns.push_back(currConn);
             }
@@ -634,19 +669,27 @@ void epoch_break_down(
                 sync.cv.wait(lck);
 
             for (size_t j = 0; j < conns.size(); j++)
+            {
+                std::vector<size_t> currBandwidthDataAfter = get_log_data(mg_get_context(conns[j]));
+
+                bandwidthData[0] += currBandwidthDataAfter[0] - otherBandwidthDataBefore[j][0];
+                bandwidthData[1] += currBandwidthDataAfter[1] - otherBandwidthDataBefore[j][1];
+                overallBandwidthData[0] += currBandwidthDataAfter[0] - otherBandwidthDataBefore[j][0];
+                overallBandwidthData[1] += currBandwidthDataAfter[1] - otherBandwidthDataBefore[j][1];
+
                 mg_close_connection(conns[j]);
+            }
 
             clock_t cpuTimeAfter = clock();
             std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
-            std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer.getContext());
+            std::vector<size_t> serverBandwidthDataAfter = get_log_data(civetServer.getContext());
 
             std::vector<double> timingData(2);
             timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
             timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;
 
-            std::vector<size_t> bandwidthData(2);
-            bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0];
-            bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];
+            bandwidthData[0] += serverBandwidthDataAfter[0] - serverBandwidthDataBefore[0];
+            bandwidthData[1] += serverBandwidthDataAfter[1] - serverBandwidthDataBefore[1];
         }
         else    // When it's another server's turn, tell them to do their part
         {
@@ -656,7 +699,7 @@ void epoch_break_down(
             std::string data = make_epoch_initiator_string(generatorProof, nextGenerator);
             
             // And have them do that request
-            initiate_epoch_updates(rng, serverIPs[i], serverPorts[i], data, true, unused);
+            initiate_epoch_updates(rng, serverIPs[i], serverPorts[i], data, true, unused, overallBandwidthData);
         }
     }
 }
@@ -671,7 +714,8 @@ Twistpoint initiate_epoch_updates(
     int recipientPort,
     const std::string& data,
     bool isBreakdown,
-    std::vector<std::vector<Proof>>& generatorProofHolder)
+    std::vector<std::vector<Proof>>& generatorProofHolder,
+    std::vector<size_t>& bandwidthData)
 {
     Twistpoint retval;
     struct synchronization_tool sync;
@@ -764,10 +808,12 @@ void tally_scores(
     int selfPort,
     const Twistpoint& nextGenerator,
     std::vector<EGCiphertext>& userTallyScores,
-    std::vector<CurveBipoint>& serverTallyScores)
+    std::vector<CurveBipoint>& serverTallyScores,
+    std::vector<size_t>& bandwidthData)
 {
     struct synchronization_tool sync;
     std::vector<struct mg_connection *> conns;
+    std::vector<std::vector<size_t>> allBandwidthDataBefore;
 
     // Connect to each server (roughly in parallel)
     std::unique_lock<std::mutex> lck(sync.mtx);
@@ -788,6 +834,8 @@ void tally_scores(
                 std::cerr << "Trouble getting partial decryption from server at " << serverIPs[i] << ":" << serverPorts[i] << std::endl;
         }
 
+        allBandwidthDataBefore.push_back(get_log_data(mg_get_context(currConn)));
+
         // Ping server for simulated distributed BGN
         mg_websocket_client_write(currConn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
 
@@ -800,7 +848,14 @@ void tally_scores(
 
     // Close connections
     for (size_t i = 0; i < conns.size(); i++)
+    {
+        std::vector<size_t> currBandwidthDataAfter = get_log_data(mg_get_context(conns[i]));
+
+        bandwidthData[0] += currBandwidthDataAfter[0] - allBandwidthDataBefore[i][0];
+        bandwidthData[1] += currBandwidthDataAfter[1] - allBandwidthDataBefore[i][1];
+
         mg_close_connection(conns[i]);
+    }
 
     // Now we do the actual calculations
     std::vector<EGCiphertext> retval;
@@ -842,7 +897,8 @@ void distribute_tallied_scores(
     int selfPort,
     const Twistpoint& nextGenerator,
     const std::vector<EGCiphertext>& userTallyScores,
-    const std::vector<CurveBipoint>& serverTallyScores)
+    const std::vector<CurveBipoint>& serverTallyScores,
+    std::vector<size_t>& bandwidthData)
 {
     // Serialize scores
     std::stringstream buffer;
@@ -857,6 +913,7 @@ void distribute_tallied_scores(
 
     struct synchronization_tool sync;
     std::vector<struct mg_connection *> conns;
+    std::vector<std::vector<size_t>> allBandwidthDataBefore;
 
     // Connect to each server (roughly in parallel)
     std::unique_lock<std::mutex> lck(sync.mtx);
@@ -880,6 +937,8 @@ void distribute_tallied_scores(
                 std::cerr << "Trouble giving full re-encryption to server at " << serverIPs[i] << ":" << serverPorts[i] << std::endl;
         }
 
+        allBandwidthDataBefore.push_back(get_log_data(mg_get_context(currConn)));
+
         // Send the relevant data
         mg_websocket_client_write(currConn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
         mg_websocket_client_write(currConn, MG_WEBSOCKET_OPCODE_DATACOMPLETE, "", 0);
@@ -893,7 +952,14 @@ void distribute_tallied_scores(
 
     // Close connections
     for (size_t i = 0; i < conns.size(); i++)
+    {
+        std::vector<size_t> currBandwidthDataAfter = get_log_data(mg_get_context(conns[i]));
+
+        bandwidthData[0] += currBandwidthDataAfter[0] - allBandwidthDataBefore[i][0];
+        bandwidthData[1] += currBandwidthDataAfter[1] - allBandwidthDataBefore[i][1];
+
         mg_close_connection(conns[i]);
+    }
 }
 
 /*
@@ -2053,9 +2119,11 @@ void PrsonaServerWebSocketHandler::add_new_client(
     Twistpoint shortTermPublicKey, empty;
     file >> shortTermPublicKey;
 
+    std::vector<size_t> bandwidthData(2);
+
     // Obtain global update lock
     std::unique_lock<std::mutex> updateLock(updateMtx, std::defer_lock);
-    obtain_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);
+    obtain_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort, bandwidthData);
 
     // Add new client to server object
     std::vector<Proof> proofOfValidAddition;
@@ -2070,7 +2138,7 @@ void PrsonaServerWebSocketHandler::add_new_client(
     distribute_new_user_updates(proofOfValidAddition, previousVoteTallies, currentPseudonyms, currentUserEncryptedTallies, voteMatrix);
 
     // Release global update lock
-    release_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);
+    release_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort, bandwidthData);
 
     // Serialize response
     std::stringstream buffer;
@@ -2130,12 +2198,14 @@ void PrsonaServerWebSocketHandler::receive_vote(
     // If we're dealing this update to the other servers, obtain global update lock
     std::unique_lock<std::mutex> updateLock(updateMtx, std::defer_lock);
 
+    std::vector<size_t> bandwidthData(2);
+
     std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer->getContext());
     std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
     clock_t cpuTimeBefore = clock();
 
     if (shouldDeal.val())
-        obtain_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);
+        obtain_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort, bandwidthData);
 
     // Load votes into server object
     prsonaServer->receive_vote(pi, newVotes, shortTermPublicKey);
@@ -2143,10 +2213,10 @@ void PrsonaServerWebSocketHandler::receive_vote(
     // If we're dealing this update to the other servers, actually do that
     if (shouldDeal.val())
     {
-        distribute_new_vote(pi, newVotes, shortTermPublicKey);
+        distribute_new_vote(pi, newVotes, shortTermPublicKey, bandwidthData);
 
         // Then release the global update lock
-        release_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort);
+        release_update_locks(updateLock, serverIPs, serverPorts, selfIP, selfPort, bandwidthData);
     }
 
     clock_t cpuTimeAfter = clock();
@@ -2157,9 +2227,8 @@ void PrsonaServerWebSocketHandler::receive_vote(
     timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
     timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;
 
-    std::vector<size_t> bandwidthData(2);
-    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
-    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];
+    bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
+    bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
 
     write_log_data(voteOutputMtx, voteOutputFilename, timingData, bandwidthData);
 
@@ -2256,7 +2325,8 @@ void PrsonaServerWebSocketHandler::distribute_new_user_updates(
 void PrsonaServerWebSocketHandler::distribute_new_vote(
     std::vector<Proof> pi,
     std::vector<TwistBipoint> newVotes,
-    Twistpoint shortTermPublicKey) const
+    Twistpoint shortTermPublicKey,
+    std::vector<size_t>& bandwidthData) const
 {
     struct synchronization_tool sync;
 
@@ -2304,6 +2374,8 @@ void PrsonaServerWebSocketHandler::distribute_new_vote(
             if (!conn)
                 std::cerr << "Couldn't give server " << i << " new vote data" << std::endl;
         }
+
+        std::vector<size_t> bandwidthDataBefore = get_log_data(mg_get_context(conn));
         
         // Send the server the new vote data
         mg_websocket_client_write(conn, MG_WEBSOCKET_OPCODE_BINARY, data.c_str(), data.length());
@@ -2313,6 +2385,11 @@ void PrsonaServerWebSocketHandler::distribute_new_vote(
         while (!sync.val2)
             sync.cv.wait(syncLock);
 
+        std::vector<size_t> bandwidthDataAfter = get_log_data(mg_get_context(conn));
+
+        bandwidthData[0] += bandwidthDataAfter[0] - bandwidthDataBefore[0];
+        bandwidthData[1] += bandwidthDataAfter[1] - bandwidthDataBefore[1];
+
         // Close connection
         mg_close_connection(conn);
 
@@ -2495,8 +2572,10 @@ void PrsonaServerWebSocketHandler::build_up_midway_pseudonyms(
     std::vector<std::vector<std::vector<CurveBipoint>>> serverTallyCommits;
     std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> partwayVoteMatrixCommits;
     std::vector<std::vector<std::vector<std::vector<TwistBipoint>>>> finalVoteMatrixCommits;
+    std::vector<size_t> bandwidthData(2);
+    std::vector<std::vector<size_t>> otherBandwidthDataBefore;
 
-    std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer->getContext());
+    std::vector<size_t> serverBandwidthDataBefore = get_log_data(civetServer->getContext());
     std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
     clock_t cpuTimeBefore = clock();
 
@@ -2523,6 +2602,7 @@ void PrsonaServerWebSocketHandler::build_up_midway_pseudonyms(
         struct mg_connection *currConn = distribute_epoch_updates(serverIPs[i], serverPorts[i], data, &sync);
 
         conns.push_back(currConn);
+        otherBandwidthDataBefore.push_back(get_log_data(mg_get_context(currConn)));
     }
 
     // Wait for all to acknowledge receipt of the update data
@@ -2531,19 +2611,25 @@ void PrsonaServerWebSocketHandler::build_up_midway_pseudonyms(
 
     // Close connections
     for (size_t i = 0; i < conns.size(); i++)
+    {
+        std::vector<size_t> currBandwidthDataAfter = get_log_data(mg_get_context(conns[i]));
+
+        bandwidthData[0] += currBandwidthDataAfter[0] - otherBandwidthDataBefore[i][0];
+        bandwidthData[1] += currBandwidthDataAfter[1] - otherBandwidthDataBefore[i][1];
+
         mg_close_connection(conns[i]);
+    }
 
     clock_t cpuTimeAfter = clock();
     std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
-    std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer->getContext());
+    std::vector<size_t> serverBandwidthDataAfter = get_log_data(civetServer->getContext());
 
     std::vector<double> timingData(2);
     timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
     timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;
 
-    std::vector<size_t> bandwidthData(2);
-    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
-    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];
+    bandwidthData[0] += serverBandwidthDataAfter[0] - serverBandwidthDataBefore[0] + bandwidthRcv;
+    bandwidthData[1] += serverBandwidthDataAfter[1] - serverBandwidthDataBefore[1];
 
     write_log_data(buildUpOutputMtx, buildUpOutputFilename, timingData, bandwidthData);
     
@@ -2576,8 +2662,10 @@ void PrsonaServerWebSocketHandler::break_down_midway_pseudonyms(
     std::vector<std::vector<std::vector<Twistpoint>>> userTallyMaskCommits;
     std::vector<std::vector<std::vector<Twistpoint>>> userTallyMessageCommits;
     std::vector<std::vector<std::vector<Twistpoint>>> userTallySeedCommits;
+    std::vector<size_t> bandwidthData(2);
+    std::vector<std::vector<size_t>> otherBandwidthDataBefore;
 
-    std::vector<size_t> bandwidthDataBefore = get_log_data(civetServer->getContext());
+    std::vector<size_t> serverBandwidthDataBefore = get_log_data(civetServer->getContext());
     std::chrono::high_resolution_clock::time_point wallTimeBefore = std::chrono::high_resolution_clock::now();
     clock_t cpuTimeBefore = clock();
 
@@ -2602,6 +2690,7 @@ void PrsonaServerWebSocketHandler::break_down_midway_pseudonyms(
         struct mg_connection *currConn = distribute_epoch_updates(serverIPs[i], serverPorts[i], data, &sync);
 
         conns.push_back(currConn);
+        otherBandwidthDataBefore.push_back(get_log_data(mg_get_context(currConn)));
     }
 
     // Wait for all to acknowledge receipt of the update data
@@ -2610,19 +2699,25 @@ void PrsonaServerWebSocketHandler::break_down_midway_pseudonyms(
 
     // Close connections
     for (size_t i = 0; i < conns.size(); i++)
+    {
+        std::vector<size_t> currBandwidthDataAfter = get_log_data(mg_get_context(conns[i]));
+
+        bandwidthData[0] += currBandwidthDataAfter[0] - otherBandwidthDataBefore[i][0];
+        bandwidthData[1] += currBandwidthDataAfter[1] - otherBandwidthDataBefore[i][1];
+
         mg_close_connection(conns[i]);
+    }
 
     clock_t cpuTimeAfter = clock();
     std::chrono::high_resolution_clock::time_point wallTimeAfter = std::chrono::high_resolution_clock::now();
-    std::vector<size_t> bandwidthDataAfter = get_log_data(civetServer->getContext());
+    std::vector<size_t> serverBandwidthDataAfter = get_log_data(civetServer->getContext());
 
     std::vector<double> timingData(2);
     timingData[0] = std::chrono::duration_cast<std::chrono::duration<double>>(wallTimeAfter - wallTimeBefore).count();
     timingData[1] = ((double)(cpuTimeAfter - cpuTimeBefore)) / CLOCKS_PER_SEC;
 
-    std::vector<size_t> bandwidthData(2);
-    bandwidthData[0] = bandwidthDataAfter[0] - bandwidthDataBefore[0] + bandwidthRcv;
-    bandwidthData[1] = bandwidthDataAfter[1] - bandwidthDataBefore[1];
+    bandwidthData[0] += serverBandwidthDataAfter[0] - serverBandwidthDataBefore[0] + bandwidthRcv;
+    bandwidthData[1] += serverBandwidthDataAfter[1] - serverBandwidthDataBefore[1];
 
     write_log_data(breakDownOutputMtx, breakDownOutputFilename, timingData, bandwidthData);
 

prsona/src/networking.cpp

@@ -116,8 +116,6 @@ std::vector<size_t> get_log_data(
 
     mg_get_context_info(ctx, buffer, 4096);
 
-    std::cout << "Log data: \n" << buffer << std::endl;
-
     retval.push_back(parse_log_for_data(buffer, "read"));
     retval.push_back(parse_log_for_data(buffer, "written"));