Experiment log parser tweaks

logs_to_csv.py

@@ -6,7 +6,7 @@ import sys
 import math
 import numpy as np
 
-FILE_PARSE_STATE = ["ROUND1", "ESP", "PADDING", "EPOCH"]
+FILE_PARSE_STATE = ["ROUND1", "ESP", "BYTES", "PADDING", "EPOCH"]
 
 if __name__ == "__main__":
 
@@ -19,8 +19,8 @@ if __name__ == "__main__":
     LOGS_FOLDER = sys.argv[1]
     OUTPUT_FILE = sys.argv[2]
     op_file = open(OUTPUT_FILE, 'w+')
-    op_file.write("N, M, T, B, E, epoch_total_time, epoch_stddev,\
- route_time, route_stddev, storage_time, storage_stddev\n")
+    op_file.write("N,M,T,B,E,epoch_total_time,epoch_stddev,\
+route_time,route_stddev,storage_time,storage_stddev,bytes_sent\n")
 
     # Iterate over each folder in LOGS_FOLDER
     for name in os.listdir(LOGS_FOLDER):
@@ -36,6 +36,7 @@ if __name__ == "__main__":
         print(name)
         epoch_time = []
         storage_time = []
+        bytes_sent = 0
         state = FILE_PARSE_STATE[0]
         for m in range(1,M+1):
             EOF = False
@@ -54,14 +55,13 @@ if __name__ == "__main__":
                         #print("R1: " + str(line_cnt))
 
                 elif(state == "ESP"):
-                    if("end storage_received" in line):
+                    if("end storage processing" in line):
                         value = line.split(' ')[1]
                         value = value.strip('(')
                         value = value.strip(')')
                         stg_time = float(value)
                         storage_time.append(stg_time)
-                        #print("ESP: " + str(line_cnt))
-                        state = "EPOCH"
+                        state = "BYTES"
 
                 elif(state == "PADDING"):
                     if('padding' in line):
@@ -73,6 +73,12 @@ if __name__ == "__main__":
                         else:
                             state = "ROUND1"
 
+                elif(state == "BYTES"):
+                    if('bytes_sent' in line):
+                        words = line.split(' ')
+                        bytes_sent = int(words[-1])
+                        state = "EPOCH"
+
                 elif(state == "EPOCH"):
                     if('Epoch' in line and 'time' in line):
                         #print("EPOCH: " + str(line_cnt))
@@ -97,10 +103,11 @@ if __name__ == "__main__":
             print("Storage time = %f +/- %f" %(storage_mean, storage_stddev))
 
             # Insert it into the output csv file
-            op_line = str(n) + ", " + str(M) + ", " + str(t) + ", " + str(b) + ", " + str(epochs) + ", "
-            op_line+= "{:.4f}".format(epoch_mean) + (", ") + "{:.4f}".format(epoch_stddev) + ", "
-            op_line+= "{:.4f}".format(route_mean) + (", ") + "{:.4f}".format(route_stddev) + ", "
-            op_line+= "{:.4f}".format(storage_mean) + (", ") + "{:.4f}".format(storage_stddev) + "\n"
+            op_line = str(n) + "," + str(M) + "," + str(t) + "," + str(b) + "," + str(epochs) + ","
+            op_line+= "{:.4f}".format(epoch_mean) + (",") + "{:.4f}".format(epoch_stddev) + ","
+            op_line+= "{:.4f}".format(route_mean) + (",") + "{:.4f}".format(route_stddev) + ","
+            op_line+= "{:.4f}".format(storage_mean) + (",") + "{:.4f}".format(storage_stddev) + ","
+            op_line+= str(bytes_sent) + "\n"
             op_file.write(op_line)
         else:
             print("No valid logs for %s" % name)