lox/Parsing-results/raw_to_csv.py

from pathlib import Path

standard_check_file = open("standard_check"+".csv", "w")
standard_check_file.write("Percent,Bridges,RequestS,Rsstdev,RequestT,Rtstdev,ResponseS,Restdev,ResponseT,ReTstdev,ResponseHT,RHTstdev\n")

for p in Path('.').glob('*.log'):
    print(f"Parsing: {p.name.strip('.log')}\n")
    with p.open() as log_file:
        test_file = open(p.name.strip('.log')+".csv", "w")
        test_file.write("Percent,Bridges,RequestS,Rsstdev,RequestT,Rtstdev,ResponseS,Restdev,ResponseT,ReTstdev,ResponseHT,RHTstdev\n")
        bridges = 0
        c=False
        red = 0
        check = 0
        level = 0
        protocol = 0
        num = 0
        req_size = 0
        req_size_std = 0
        req_time = 0
        req_time_std = 0
        resp_size = 0
        resp_size_std = 0
        resp_time = 0
        resp_time_std = 0
        resp_handle_time = 0
        resp_handle_std = 0
        endline = 0

        # Loop over the remaining lines in the file
        for line in log_file:
            if "***START" in line:
                bridges = line.split()[1].split("*")[0]
            if "CHECK-BLOCKAGE" in line:
                protocol = 1
                num = line.split("-")[6].strip('-')
                if int(bridges) == 900:
                    check =1
                if not c:
                    check_b = open("check_blockage"+str(num)+".csv", "w")
                    check_b.write("Percent,Bridges,RequestS,Rsstdev,RequestT,Rtstdev,ResponseS,Restdev,ResponseT,ReTstdev,ResponseHT,RHTstdev\n")
                    c=True
            elif "BLOCKAGE-MIGRATION" in line:
                protocol = 2
                num = line.split("-")[6].strip('-')
            elif "REDEEM" in line:
                protocol = 3
                if not red:
                    redeem = open("redeem_invites.csv", "w")
                    redeem.write("Percent,Bridges,RequestS,Rsstdev,RequestT,Rtstdev,ResponseS,Restdev,ResponseT,ReTstdev,ResponseHT,RHTstdev\n")
                    red = 1
            elif "ISSUE" in line:
                protocol = 4
            elif "OPEN" in line:
                protocol = 5
            elif "TRUST-PROMOTION" in line:
                protocol = 6
                if not level:
                    trust_promo = open("trust_promo.csv", "w")
                    trust_promo.write("Percent,Bridges,RequestS,Rsstdev,RequestT,Rtstdev,ResponseS,Restdev,ResponseT,ReTstdev,ResponseHT,RHTstdev\n")
            elif "TRUST-MIGRATION" in line:
                protocol = 7
                if not level:
                    mig_file = open("trust_mig.csv", "w")
                    mig_file.write("Percent,Bridges,RequestS,Rsstdev,RequestT,Rtstdev,ResponseS,Restdev,ResponseT,ReTstdev,ResponseHT,RHTstdev\n")
            elif "LEVEL-UP-2" in line:
                protocol = 8
                if not level:
                    level_file = open("level2.csv", "w")
                    level_file.write("Percent,Bridges,RequestS,Rsstdev,RequestT,Rtstdev,ResponseS,Restdev,ResponseT,ReTstdev,ResponseHT,RHTstdev\n")
            elif "LEVEL-UP-3" in line:
                protocol = 9
                if not level:
                    level_file_t = open("level3.csv", "w")
                    level_file_t.write("Percent,Bridges,RequestS,Rsstdev,RequestT,Rtstdev,ResponseS,Restdev,ResponseT,ReTstdev,ResponseHT,RHTstdev\n")
            elif "LEVEL-UP-4" in line:
                protocol = 10
                if not level:
                    level_file_f = open("level4.csv", "w")
                    level_file_f.write("Percent,Bridges,RequestS,Rsstdev,RequestT,Rtstdev,ResponseS,Restdev,ResponseT,ReTstdev,ResponseHT,RHTstdev\n")
                    level = 1
            elif protocol:
                value = line.split(" = ")
                if value[0].startswith("Average"):
                    if "request" in value[0]:
                        if "size" in value[0]:
                            raw_size = value[1].split(" ")
                            req_size = raw_size[0]
                        else:
                            if "µ" in value[1]:
                                micro_sec = value[1].split("µ")
                                raw_size = float(micro_sec[0])*0.001
                            elif "m" not in value[1]:
                                sec = value[1][:-3]
                                raw_size = float(sec)*1000
                            else:
                                raw_size = value[1][:-3]
                            req_time = raw_size
                    else:
                        if "size" in value[0]:
                            raw_size = value[1].split(" ")
                            resp_size = raw_size[0]
                        else:
                            if "µ" in value[1]:
                                micro_sec = value[1].split("µ")
                                raw_size = float(micro_sec[0])*0.001
                            elif "m" not in value[1]:
                                sec = value[1][:-3]
                                raw_size = float(sec)*1000
                            else:
                                raw_size = value[1][:-3]
                            if "handling" in value[0]:
                                resp_handle_time = raw_size
                            else:
                                resp_time = raw_size
                elif value[0].startswith("Request"):
                    if "size" in value[0]:
                        raw_size = value[1].split(" ")
                        req_size_std = raw_size[0]
                    else:
                        if "µ" in value[1]:
                            micro_sec = value[1].split("µ")
                            to_sec = float(micro_sec[0])*0.001
                        else:
                            to_sec = value[1][:-3]
                        req_time_std = to_sec
                elif value[0].startswith("Response"):
                    if "size" in value[0]:
                        raw_size = value[1].split(" ")
                        resp_size_std = raw_size[0]
                    else:
                        if "µ" in value[1]:
                            micro_sec = value[1].split("µ")
                            to_sec = float(micro_sec[0])*0.001
                        else:
                            to_sec = value[1][:-3]
                        if "handling" in value[0]:
                            resp_handle_time_std = to_sec
                            endline = 1
                        else:
                            resp_time_std = to_sec

            if endline == 1:
                if check == 1:
                    standard_check_file.write(str(num) + "," + str(bridges)+"," + str(req_size) + "," + str(req_size_std) + ","  + str(req_time) + "," + str(req_time_std) + ","  + str(resp_size) + "," + str(resp_size_std) + "," + str(resp_time) + "," + str(resp_time_std) + "," + str(resp_handle_time) + "," + str(resp_handle_time_std) + "\n")
                    check = 0
                if protocol == 1:
                    check_b.write(str(num) + "," + str(bridges)+"," + str(req_size) + "," + str(req_size_std) + ","  + str(req_time) + "," + str(req_time_std) + ","  + str(resp_size) + "," + str(resp_size_std) + "," + str(resp_time) + "," + str(resp_time_std) + "," + str(resp_handle_time) + "," + str(resp_handle_time_std) + "\n")
                elif protocol == 3:
                    redeem.write(str(num) + "," + str(bridges)+"," + str(req_size) + "," + str(req_size_std) + ","  + str(req_time) + "," + str(req_time_std) + ","  + str(resp_size) + "," + str(resp_size_std) + "," + str(resp_time) + "," + str(resp_time_std) + "," + str(resp_handle_time) + "," + str(resp_handle_time_std) + "\n")
                elif protocol<6:
                    test_file.write(str(num) + "," + str(bridges)+"," + str(req_size) + "," + str(req_size_std) + ","  + str(req_time) + "," + str(req_time_std) + ","  + str(resp_size) + "," + str(resp_size_std) + "," + str(resp_time) + "," + str(resp_time_std) + "," + str(resp_handle_time) + "," + str(resp_handle_time_std) + "\n")
                else:
                    if protocol == 6:
                        trust_promo.write(str(num) + "," + str(bridges)+"," + str(req_size) + "," + str(req_size_std) + ","  + str(req_time) + "," + str(req_time_std) + ","  + str(resp_size) + "," + str(resp_size_std) + "," + str(resp_time) + "," + str(resp_time_std) + "," + str(resp_handle_time) + "," + str(resp_handle_time_std) + "\n")
                    if protocol == 7:
                        mig_file.write(str(num) + "," + str(bridges)+"," + str(req_size) + "," + str(req_size_std) + ","  + str(req_time) + "," + str(req_time_std) + ","  + str(resp_size) + "," + str(resp_size_std) + "," + str(resp_time) + "," + str(resp_time_std) + "," + str(resp_handle_time) + "," + str(resp_handle_time_std) + "\n")
                    if protocol == 8:
                        level_file.write(str(num) + "," + str(bridges)+"," + str(req_size) + "," + str(req_size_std) + ","  + str(req_time) + "," + str(req_time_std) + ","  + str(resp_size) + "," + str(resp_size_std) + "," + str(resp_time) + "," + str(resp_time_std) + "," + str(resp_handle_time) + "," + str(resp_handle_time_std) + "\n")
                    elif protocol == 9:
                        level_file_t.write(str(num) + "," + str(bridges)+"," + str(req_size) + "," + str(req_size_std) + ","  + str(req_time) + "," + str(req_time_std) + ","  + str(resp_size) + "," + str(resp_size_std) + "," + str(resp_time) + "," + str(resp_time_std) + "," + str(resp_handle_time) + "," + str(resp_handle_time_std) + "\n")
                    elif protocol == 10:
                        level_file_f.write(str(num) + "," + str(bridges)+"," + str(req_size) + "," + str(req_size_std) + ","  + str(req_time) + "," + str(req_time_std) + ","  + str(resp_size) + "," + str(resp_size_std) + "," + str(resp_time) + "," + str(resp_time_std) + "," + str(resp_handle_time) + "," + str(resp_handle_time_std) + "\n")
                endline = 0
                protocol = 0
        if level:
            level_file.close()
            level_file_t.close()
            level_file_f.close()
            trust_promo.close()
            mig_file.close()
        if red:
            redeem.close()
        test_file.close()
standard_check_file.close()
print("Done.")