|
|
@@ -0,0 +1,106 @@
|
|
|
+#!/usr/bin/python3
|
|
|
+
|
|
|
+import subprocess
|
|
|
+import os
|
|
|
+import sys
|
|
|
+import math
|
|
|
+import numpy as np
|
|
|
+
|
|
|
+FILE_PARSE_STATE = ["ROUND1", "ESP", "PADDING", "EPOCH"]
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+
|
|
|
+ if(len(sys.argv)!=3):
|
|
|
+ print("Incorrect usage!\n")
|
|
|
+ print("./logs_to_csv.py expects 2 parameters.")
|
|
|
+ print("Usage: ./logs_to_csv.py <Path to logs folder> <output csv file name>")
|
|
|
+ exit()
|
|
|
+
|
|
|
+ LOGS_FOLDER = sys.argv[1]
|
|
|
+ OUTPUT_FILE = sys.argv[2]
|
|
|
+ op_file = open(OUTPUT_FILE, 'w+')
|
|
|
+ op_file.write("N, M, T, B, E, epoch_total_time, epoch_stddev,\
|
|
|
+ route_time, route_stddev, storage_time, storage_stddev\n")
|
|
|
+
|
|
|
+ # Iterate over each folder in LOGS_FOLDER
|
|
|
+ for name in os.listdir(LOGS_FOLDER):
|
|
|
+ #print(name)
|
|
|
+ params = name.split('_')
|
|
|
+ print (params)
|
|
|
+ n = int(params[0])
|
|
|
+ M = int(params[1])
|
|
|
+ t = int(params[2])
|
|
|
+ b = int(params[3])
|
|
|
+
|
|
|
+ expected_real = math.floor(n/M)
|
|
|
+ print(name)
|
|
|
+ epoch_time = []
|
|
|
+ storage_time = []
|
|
|
+ state = FILE_PARSE_STATE[0]
|
|
|
+ for m in range(1,M+1):
|
|
|
+ EOF = False
|
|
|
+ f = open(os.path.join(LOGS_FOLDER, name, 's'+str(m)+'.log'),'r')
|
|
|
+ print(os.path.join(LOGS_FOLDER, name, 's'+str(m)+'.log'))
|
|
|
+ line_cnt = 0
|
|
|
+ while(1):
|
|
|
+ line = f.readline()
|
|
|
+ line_cnt+=1
|
|
|
+ if(line == ""):
|
|
|
+ break
|
|
|
+
|
|
|
+ elif(state == "ROUND1"):
|
|
|
+ if("Round 1" in line):
|
|
|
+ state = "PADDING"
|
|
|
+ #print("R1: " + str(line_cnt))
|
|
|
+
|
|
|
+ elif(state == "ESP"):
|
|
|
+ if("end storage_received" in line):
|
|
|
+ value = line.split(' ')[1]
|
|
|
+ value = value.strip('(')
|
|
|
+ value = value.strip(')')
|
|
|
+ stg_time = float(value)
|
|
|
+ storage_time.append(stg_time)
|
|
|
+ #print("ESP: " + str(line_cnt))
|
|
|
+ state = "EPOCH"
|
|
|
+
|
|
|
+ elif(state == "PADDING"):
|
|
|
+ if('padding' in line):
|
|
|
+ #print("PADDING: " + str(line_cnt))
|
|
|
+ words = line.split(' ')
|
|
|
+ log_real = int(words[0])
|
|
|
+ if(log_real >= expected_real):
|
|
|
+ state = "ESP"
|
|
|
+ else:
|
|
|
+ state = "ROUND1"
|
|
|
+
|
|
|
+ elif(state == "EPOCH"):
|
|
|
+ if('Epoch' in line and 'time' in line):
|
|
|
+ #print("EPOCH: " + str(line_cnt))
|
|
|
+ nwords = line.split(' ')
|
|
|
+ epoch_time.append(float(nwords[-2]))
|
|
|
+ state = "ROUND1"
|
|
|
+
|
|
|
+ if(len(epoch_time)!=0):
|
|
|
+ route_time = []
|
|
|
+ for i in range(len(epoch_time)):
|
|
|
+ route_time.append(epoch_time[i] - storage_time[i])
|
|
|
+ epoch_mean = np.mean(epoch_time)
|
|
|
+ route_mean = np.mean(route_time)
|
|
|
+ storage_mean = np.mean(storage_time)
|
|
|
+ epoch_stddev = np.std(epoch_time)
|
|
|
+ route_stddev = np.std(route_time)
|
|
|
+ storage_stddev = np.std(storage_time)
|
|
|
+ epochs = int(len(epoch_time)/M)
|
|
|
+ print("Num epochs = %d" % epochs);
|
|
|
+ print("Epoch time = %f +/- %f" %(epoch_mean, epoch_stddev))
|
|
|
+ print("Route time = %f +/- %f" %(route_mean, route_stddev))
|
|
|
+ print("Storage time = %f +/- %f" %(storage_mean, storage_stddev))
|
|
|
+
|
|
|
+ # Insert it into the output csv file
|
|
|
+ op_line = str(n) + ", " + str(M) + ", " + str(t) + ", " + str(b) + ", " + str(epochs) + ", "
|
|
|
+ op_line+= "{:.4f}".format(epoch_mean) + (", ") + "{:.4f}".format(epoch_stddev) + ", "
|
|
|
+ op_line+= "{:.4f}".format(route_mean) + (", ") + "{:.4f}".format(route_stddev) + ", "
|
|
|
+ op_line+= "{:.4f}".format(storage_mean) + (", ") + "{:.4f}".format(storage_stddev) + "\n"
|
|
|
+ op_file.write(op_line)
|
|
|
+ else:
|
|
|
+ print("No valid logs for %s" % name)
|
