teems/logs_to_csv.py

#!/usr/bin/python3

import subprocess
import os
import sys
import math
import numpy as np

FILE_PARSE_STATE = ["ROUND1", "ESP", "BYTES", "PADDING", "EPOCH", "SCMT"]

'''
parse_output_logs is used in two ways:
  (i) For logs_to_csv.py script which produces the output csv given the
  LOGS_FOLDER with multiple experiment log folders within it. This is done when
  this script is run directly

  (ii) For run_experiments, which invokes parse_output_logs with (LOGS_FOLDER,
  experiment_name), takes the LOGS_FOLDER pointing to a diagnostic output
  folder to extract num_sizes of waksman networks, the WN precomputation time,
  epoch time, and send_client_mailbox time.

'''
def parse_output_logs(LOGS_FOLDER, experiment_name, generate_csv = False, op_file = None):

    params = experiment_name.split('_')
    print (params)
    n = int(params[0])
    M = int(params[1])
    t = int(params[2])
    b = int(params[3].strip('/'))

    expected_real = math.floor(n/M)
    epoch_time = []
    # scm = send_client_mailbox
    scm_time = []
    storage_time = []
    # pwn = precompute_Waksman_Network
    pwn_time = []
    bytes_sent = 0
    num_sizes = 0
    state = FILE_PARSE_STATE[0]
    for m in range(1,M+1):
        EOF = False
        f = open(os.path.join(LOGS_FOLDER, experiment_name, 's'+str(m)+'.log'),'r')
        print(os.path.join(LOGS_FOLDER, experiment_name, 's'+str(m)+'.log'))
        line_cnt = 0
        while(1):
            line = f.readline()
            line_cnt+=1
            if(line == ""):
                break

            if('end precompute Waksman Network' in line):
                value = line.split(' ')[1]
                value = value.strip('(')
                value = value.strip(')')
                pwn_time.append(float(value))

            if('Precompute num_sizes' in line):
                value = line.split(' ')[-1]
                num_sizes = int(value)

            elif(state == "ROUND1"):
                if("Round 1 " in line):
                    state = "PADDING"
                    #print("R1: " + str(line_cnt))

            elif(state == "ESP"):
                if("end storage processing" in line):
                    value = line.split(' ')[1]
                    value = value.strip('(')
                    value = value.strip(')')
                    stg_time = float(value)
                    storage_time.append(stg_time)
                    state = "BYTES"

            elif(state == "PADDING"):
                if('padding' in line):
                    #print("PADDING: " + str(line_cnt))
                    words = line.split(' ')
                    log_real = int(words[0])
                    if(log_real >= expected_real):
                        state = "ESP"
                    else:
                        state = "ROUND1"

            elif(state == "BYTES"):
                if('bytes_sent' in line):
                    words = line.split(' ')
                    bytes_sent = int(words[-1])
                    state = "EPOCH"

            elif(state == "EPOCH"):
                if('Epoch' in line and 'time' in line):
                    #print("EPOCH: " + str(line_cnt))
                    nwords = line.split(' ')
                    epoch_time.append(float(nwords[-2]))
                    state = "SCMT"

            elif(state == "SCMT"):
                if('send_client_mailbox time' in line):
                    scm_time = float(line.split(' ')[-2])
                    state = "ROUND1"


    if(len(epoch_time)!=0):
        route_time = []
        for i in range(len(epoch_time)):
            route_time.append(epoch_time[i] - storage_time[i])
        epoch_mean = np.mean(epoch_time)
        route_mean = np.mean(route_time)
        storage_mean = np.mean(storage_time)
        scm_mean = np.mean(scm_time)
        pwn_mean = np.mean(pwn_time)

        epoch_max = np.max(epoch_time)
        scm_max = np.max(scm_time)
        pwn_max = np.max(pwn_time)

        epoch_stddev = np.std(epoch_time)
        route_stddev = np.std(route_time)
        storage_stddev = np.std(storage_time)
        scm_stddev = np.std(scm_time)
        pwn_stddev = np.mean(pwn_time)

        epochs = int(len(epoch_time)/M)
        print("Num epochs = %d" % epochs);
        print("Route time = %f +/- %f" %(route_mean, route_stddev))
        print("Storage time = %f +/- %f" %(storage_mean, storage_stddev))
        print("Epoch time = %f +/- %f (Max epoch_time = %f)" %(epoch_mean, epoch_stddev, epoch_max))
        print("PWN time = %f +/- %f (Max PWN = %f)" %(pwn_mean, pwn_stddev, pwn_max))
        print("SCM time = %f +/- %f (Max SCM = %f)" %(scm_mean, scm_stddev, scm_max))

        if(generate_csv):
            # Insert it into the output csv file
            op_line = str(n) + "," + str(M) + "," + str(t) + "," + str(b) + "," + str(epochs) + ","
            op_line+= "{:.4f}".format(epoch_mean) + (",") + "{:.4f}".format(epoch_stddev) + ","
            op_line+= "{:.4f}".format(route_mean) + (",") + "{:.4f}".format(route_stddev) + ","
            op_line+= "{:.4f}".format(storage_mean) + (",") + "{:.4f}".format(storage_stddev) + ","
            op_line+= str(bytes_sent) + "\n"
            op_file.write(op_line)
        else:
            return(num_sizes, pwn_max, epoch_max, scm_max)
    else:
        print("No valid logs for %s" % LOGS_FOLDER)


if __name__ == "__main__":

    if(len(sys.argv)!=3):
        print("Incorrect usage!\n")
        print("./logs_to_csv.py expects 2 parameters.")
        print("Usage: ./logs_to_csv.py <Path to logs folder> <output csv file name>")
        exit()

    LOGS_FOLDER = sys.argv[1]
    OUTPUT_FILE = sys.argv[2]
    op_file = open(OUTPUT_FILE, 'w+')
    op_file.write("N,M,T,B,E,epoch_total_time,epoch_stddev,\
route_time,route_stddev,storage_time,storage_stddev,bytes_sent\n")

    for exp_name in os.listdir(LOGS_FOLDER):
        parse_output_logs(LOGS_FOLDER, exp_name, True, op_file)

    op_file.close()