#!/usr/bin/python3 ''' NOTE: This script is tailored for a machine with 2 40-core processors. Currently assumes 8 threads (C36-39,76-79) are set aside for client simulator; change this by setting "prefix" in Client/clientlaunch and the -t option to clientlaunch below. Similarly, the use of cores C0-35,40-75 for the TEEMS servers can be changed in the gen_manifest.py program. If you have a different number of cores available for servers, also change M_MAX below. ''' import subprocess import os import sys import math import time from gen_manifest import generate_manifest from gen_enclave_config import generate_config from logs_to_csv import parse_output_logs ############################################################################### # CONFIGS TO SET: MANIFEST_FILE = "App/manifest.yaml" LOG_FOLDER = "190824_Fig8/" NUM_EPOCHS = 5 PRIVATE_ROUTE = False PRIV_OUT = 1 PRIV_IN = 1 PUB_OUT = 1 PUB_IN = 1 # N = number of clients # M = number of servers # T = threads per server instance ## A large grid of many combinations # N = [1<<16, 1<<17, 1<<18, 1<<19, 1<<20] # M = [72, 64, 48, 36, 32, 24, 16, 8, 4, 2, 1] # T = [16, 8, 4, 2, 1] ## Figure 7 #N = [1<<15, 1<<16, 1<<17, 1<<18, 1<<19, 1<<20] #M = [4] #T = [4] ## Figure 8 N = [1<<20] M = [72, 64, 48, 36, 32, 24, 16, 8, 4, 2, 1] T = [1] # Max servers depending on number of threads (assuming 72 available # cores for servers) M_MAX = { 1:72, 2:36, 4:18, 8:9, 16:4 } # B = message size (bytes) B = 256 NUM_DIAGNOSTIC_EPOCHS = 4 ############################################################################### def epoch_time_estimate(n, m, t, b): # Base epoch time is 5 sec etime_base = 5 clients_per_server = math.ceil(n/m) # Using 8 sec for 2^20 clients in route_compute time as the base for calculations below etime_route_compute = 0.8 * math.ceil(clients_per_server/100000) etime_precompute = 1 * math.ceil(clients_per_server/100000) # Costs for Waksman network precompute # Public routing needs 7 WN, private routing needs 3 WNs etime_precompute *=5 # Client time: # Takes about 30 sec for handling 2^20 clients etime_client = 3 * math.ceil(clients_per_server/100000) etime = etime_base + etime_precompute + etime_route_compute + etime_client return int(etime) if __name__ == "__main__": if not os.path.exists(LOG_FOLDER): os.mkdir(LOG_FOLDER) DIAGNOSTIC_FOLDER = LOG_FOLDER + "diagnostic/" if not os.path.exists(DIAGNOSTIC_FOLDER): os.mkdir(DIAGNOSTIC_FOLDER) for t in T: b = B m_start = 1 # Set M_MAX depending on t m_end = M_MAX[t] for m in M: if(m <= m_end): for n in N: #for run in ["diagnostic", "experiment"]: for run in ["experiment"]: num_WN_to_precompute = 0 if(PRIVATE_ROUTE): num_WN_to_precompute = 2 * 3 else: num_WN_to_precompute = 2 * 8 # Make the correct output folder for diagnostic/experiment experiment_name = str(n) + "_" + str(m) + "_" + str(t) + "_" + str(b) + "/" if(run == "diagnostic"): log_subfolder = DIAGNOSTIC_FOLDER elif(run == "experiment"): log_subfolder = LOG_FOLDER log_subfolder = log_subfolder + experiment_name if not os.path.exists(log_subfolder): os.mkdir(log_subfolder) if(run == "diagnostic"): print("\n\n Running DIAGNOSTIC t = %d, m = %d, n = %d \n\n" % (t, m, n)) # Manifest generated by diagnostic can be reused by the actual experiment generate_manifest(n, m, t, b, PRIVATE_ROUTE, PRIV_OUT, PRIV_IN, PUB_OUT, PUB_IN) generate_config(n, m, t, b, PRIVATE_ROUTE, PRIV_OUT, PRIV_IN, PUB_OUT, PUB_IN, num_WN_to_precompute) epoch_param = epoch_time_estimate(n, m, t, b) elif(run == "experiment"): #print("Waiting for 2 mins to reset sockets") #time.sleep(120) #num_sizes, pwn_max, epoch_max, scm_max = parse_output_logs(DIAGNOSTIC_FOLDER, experiment_name) #print("From logs_to_csv: num_sizes = %d, pwn_max = %f, epoch_max = %f, scm_max = %f" #% (num_sizes, pwn_max, epoch_max, scm_max)) print("\n\n Running EXPERIMENT t = %d, m = %d, n = %d \n\n" % (t, m, n)) #num_WN_to_precompute = math.ceil((num_sizes * pwn_max)/epoch_max) #print("num_WN_to_precompute = %d" %(num_WN_to_precompute)) #if(num_WN_to_precompute < 2 * num_sizes): # num_WN_to_precompute = 2 * num_sizes #print("num_WN_to_precompute (pushed up to min 2 sets) = %d" %(num_WN_to_precompute)) #epoch_param = math.ceil(epoch_max + 10 * m * scm_max) generate_manifest(n, m, t, b, PRIVATE_ROUTE, PRIV_OUT, PRIV_IN, PUB_OUT, PUB_IN) epoch_param = epoch_time_estimate(n, m, t, b) generate_config(n, m, t, b, PRIVATE_ROUTE, PRIV_IN, PUB_OUT, PUB_IN, num_WN_to_precompute) # Either estimate from epoch_time_estimate for diagnostic # or the one we got from diagnostic run epoch_duration = math.ceil(epoch_param) print("Epoch_duration = %d" % epoch_duration) # Since launch is invoked from App/ ; we add a ../ to subfolder before # passing it to slaunch log_subfolder = "../" + log_subfolder slaunch = [] slaunch.append("./launch") slaunch.append("route_clients") slaunch.append("-l") slaunch.append(log_subfolder) slaunch.append("-n") nodes_to_include = ["-n"] for i in range(1, m+1): nodes_to_include.append("s"+str(i)) slaunch.append("s"+str(i)) slaunch.append("-d") slaunch.append(str(epoch_duration)) slaunch.append("-e") if(run == "experiment"): slaunch.append(str(NUM_EPOCHS)) else: slaunch.append(str(NUM_DIAGNOSTIC_EPOCHS)) if(run == "experiment"): slaunch.append("-w") slaunch.append(str(num_WN_to_precompute)) os.chdir("./App") # Server outputs are captured by log files provided to each of the server # launch calls made by App/launch make = subprocess.call(["make", "-C", "..", "-j"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) try: os.mkdir("keys") except: # It's OK if it already exists pass pubkeys = subprocess.call(["./getpubkeys"]) server_process = subprocess.run(slaunch) claunch = [] claunch.append("./clientlaunch") claunch.append("-t") claunch.append("8") claunch.append("-l") claunch.append(log_subfolder+"clients.log") os.chdir("./../Client/") client_process = subprocess.call(claunch) os.chdir("./../")