teems/logs_to_csv.py

#!/usr/bin/python3

import subprocess
import os
import sys
import math
import numpy as np

FILE_PARSE_STATE = ["ROUND1", "ESP", "BYTES", "PADDING", "EPOCH"]

if __name__ == "__main__":

    if(len(sys.argv)!=3):
        print("Incorrect usage!\n")
        print("./logs_to_csv.py expects 2 parameters.")
        print("Usage: ./logs_to_csv.py <Path to logs folder> <output csv file name>")
        exit()

    LOGS_FOLDER = sys.argv[1]
    OUTPUT_FILE = sys.argv[2]
    op_file = open(OUTPUT_FILE, 'w+')
    op_file.write("N,M,T,B,E,epoch_total_time,epoch_stddev,\
route_time,route_stddev,storage_time,storage_stddev,bytes_sent\n")

    # Iterate over each folder in LOGS_FOLDER
    for name in os.listdir(LOGS_FOLDER):
        #print(name)
        params = name.split('_')
        print (params)
        n = int(params[0])
        M = int(params[1])
        t = int(params[2])
        b = int(params[3])

        expected_real = math.floor(n/M)
        print(name)
        epoch_time = []
        storage_time = []
        bytes_sent = 0
        state = FILE_PARSE_STATE[0]
        for m in range(1,M+1):
            EOF = False
            f = open(os.path.join(LOGS_FOLDER, name, 's'+str(m)+'.log'),'r')
            print(os.path.join(LOGS_FOLDER, name, 's'+str(m)+'.log'))
            line_cnt = 0
            while(1):
                line = f.readline()
                line_cnt+=1
                if(line == ""):
                    break

                elif(state == "ROUND1"):
                    if("Round 1" in line):
                        state = "PADDING"
                        #print("R1: " + str(line_cnt))

                elif(state == "ESP"):
                    if("end storage processing" in line):
                        value = line.split(' ')[1]
                        value = value.strip('(')
                        value = value.strip(')')
                        stg_time = float(value)
                        storage_time.append(stg_time)
                        state = "BYTES"

                elif(state == "PADDING"):
                    if('padding' in line):
                        #print("PADDING: " + str(line_cnt))
                        words = line.split(' ')
                        log_real = int(words[0])
                        if(log_real >= expected_real):
                            state = "ESP"
                        else:
                            state = "ROUND1"

                elif(state == "BYTES"):
                    if('bytes_sent' in line):
                        words = line.split(' ')
                        bytes_sent = int(words[-1])
                        state = "EPOCH"

                elif(state == "EPOCH"):
                    if('Epoch' in line and 'time' in line):
                        #print("EPOCH: " + str(line_cnt))
                        nwords = line.split(' ')
                        epoch_time.append(float(nwords[-2]))
                        state = "ROUND1"

        if(len(epoch_time)!=0):
            route_time = []
            for i in range(len(epoch_time)):
                route_time.append(epoch_time[i] - storage_time[i])
            epoch_mean = np.mean(epoch_time)
            route_mean = np.mean(route_time)
            storage_mean = np.mean(storage_time)
            epoch_stddev = np.std(epoch_time)
            route_stddev = np.std(route_time)
            storage_stddev = np.std(storage_time)
            epochs = int(len(epoch_time)/M)
            print("Num epochs = %d" % epochs);
            print("Epoch time = %f +/- %f" %(epoch_mean, epoch_stddev))
            print("Route time = %f +/- %f" %(route_mean, route_stddev))
            print("Storage time = %f +/- %f" %(storage_mean, storage_stddev))

            # Insert it into the output csv file
            op_line = str(n) + "," + str(M) + "," + str(t) + "," + str(b) + "," + str(epochs) + ","
            op_line+= "{:.4f}".format(epoch_mean) + (",") + "{:.4f}".format(epoch_stddev) + ","
            op_line+= "{:.4f}".format(route_mean) + (",") + "{:.4f}".format(route_stddev) + ","
            op_line+= "{:.4f}".format(storage_mean) + (",") + "{:.4f}".format(storage_stddev) + ","
            op_line+= str(bytes_sent) + "\n"
            op_file.write(op_line)
        else:
            print("No valid logs for %s" % name)