Improved experiment scripts and updated them for the new server

src/relay_working_experiment.py

@@ -33,11 +33,13 @@ import useful
 #remote_name = None
 #
 class CustomExperiment(experiment.Experiment):
-	def __init__(self, use_helgrind, target_tor, num_additional_eventloops, remote_name, *args, **kwargs):
+	def __init__(self, use_helgrind, target_tor, target_ld_preload, num_additional_eventloops, remote_name, remote_options, *args, **kwargs):
 		self.use_helgrind = use_helgrind
 		self.target_tor = target_tor
+		self.target_ld_preload = target_ld_preload
 		self.num_additional_eventloops = num_additional_eventloops
 		self.remote_name = remote_name
+		self.remote_options = remote_options
 		super().__init__(*args, **kwargs)
 		#
 		self.chutney_path = '/home/sengler/code/working/chutney'
@@ -55,6 +57,7 @@ class CustomExperiment(experiment.Experiment):
 		#target_tor_path = '/home/sengler/code/working/tor/src/app/tor'
 		#target_tor_path = '/home/sengler/code/releases/tor-0.4.2.5/src/app/tor'
 
+		'''
 		if self.remote_name == 'cluck2':
 			local_ip = '172.19.156.16'
 			target_ip = '172.19.156.136'
@@ -62,11 +65,27 @@ class CustomExperiment(experiment.Experiment):
 			#target_ip = '129.97.119.226'
 			target_hostname = 'cluck2'
 			target_dir = '/tmp/chutney-net'
+			#
+		elif self.remote_name == 'cluck15':
+			local_ip = '172.19.156.16'
+			target_ip = '172.19.156.188'
+			target_hostname = '129.97.119.239'
+			target_dir = '/tmp/chutney-net'
+			#
+		elif self.remote_name == 'grunt3':
+			#local_ip = '172.19.156.16'
+			#target_ip = '172.19.156.44'
+			#local_ip = '129.97.119.196'
+			#target_ip = '129.97.119.203'
+			target_hostname = '129.97.119.203'
+			target_dir = '/tmp/chutney-net'
+			#
 		elif self.remote_name == 'sengler-rpi':
 			local_ip = '129.97.119.196'
 			target_ip = '129.97.169.9'
-			target_hostname = 'sengler-rpi'
+			target_hostname = target_ip
 			target_dir = '/tmp/chutney-net'
+			#
 		elif self.remote_name is None:
 			local_ip = None
 			target_ip = None
@@ -75,6 +94,7 @@ class CustomExperiment(experiment.Experiment):
 		else:
 			raise Exception('hostname not known')
 		#
+		'''
 
 		target_optional_args = {}
 		if self.target_tor is not None:
@@ -82,23 +102,34 @@ class CustomExperiment(experiment.Experiment):
 		if self.use_helgrind:
 			target_optional_args['valgrind_settings'] = ['--tool=helgrind', '-v', '--suppressions=libevent.supp', '--read-var-info=yes']
 
+		if self.target_ld_preload is not None:
+			target_optional_args['add_environ_vars'] = {'LD_PRELOAD': self.target_ld_preload}
+
 		#target_optional_args['add_environ_vars'] = {'LD_PRELOAD': '/usr/lib/libprofiler.so.0'}
 		#target_optional_args['add_environ_vars'] = {'LD_PRELOAD': '/usr/lib/libtcmalloc_and_profiler.so.4'}
 		#target_optional_args['add_environ_vars'] = {'LD_PRELOAD': '/home/sengler/build/lib/libtcmalloc_and_profiler.so'}
 		#target_optional_args['add_environ_vars'] = {'EVENT_NOEPOLL': '', 'EVENT_SHOW_METHOD': ''}
-		if target_ip is not None:
-			target_optional_args['ip'] = target_ip
-		if target_hostname is not None:
-			target_optional_args['remote_hostname'] = target_hostname
-		if target_dir is not None:
-			target_optional_args['remote_net_dir'] = target_dir
+		if self.remote_options['target_ip'] is not None:
+			target_optional_args['ip'] = self.remote_options['target_ip']
+		if self.remote_options['target_ssh'] is not None:
+			target_optional_args['remote_hostname'] = self.remote_options['target_ssh']
+		if self.remote_options['target_chutney_net_dir'] is not None:
+			target_optional_args['remote_net_dir'] = self.remote_options['target_chutney_net_dir']
+
+		if self.remote_options.get('target_numa_settings', None) is not None:
+			target_optional_args['numa_settings'] = self.remote_options['target_numa_settings']
+		#
 
 		target_optional_args['num_cpus'] = 2 # make sure it can process onion skins fast enough, and keep it consistent between computers
 		# tor actually uses one more worker thread than what you ask for
 		target_optional_args['num_additional_eventloops'] = self.num_additional_eventloops
 		target_optional_args['dircache'] = False
 		# the voting interval is 40 seconds which puts an unrealistic workload on the target, so we disable it
-		target_cpu_prof = False #True
+		if 'target_cpu_prof' in self.remote_options:
+			target_cpu_prof = self.remote_options['target_cpu_prof']
+		else:
+			target_cpu_prof = False
+		#
 		target_daemon = False
 		target_log_throughput = True
 		target_logs = ['notice']
@@ -121,8 +152,8 @@ class CustomExperiment(experiment.Experiment):
 			if not 'num_cpus' in node.options:
 				node.options['num_cpus'] = 2
 			#
-			if not 'ip' in node.options and local_ip is not None:
-				node.options['ip'] = local_ip
+			if not 'ip' in node.options and self.remote_options['local_ip'] is not None:
+				node.options['ip'] = self.remote_options['local_ip']
 			#
 		#
 		#numa_remaining = numa.get_numa_overview()
@@ -169,11 +200,13 @@ class CustomExperiment(experiment.Experiment):
 			if p.poll() != None:
 				raise Exception('Remote CPU monitoring script exited immediately')
 			#
-			try:
-				subprocess.check_output(['ssh', self.remote_name, 'sudo renice -n -10 -g "$(pgrep --full --exact "{}")"'.format(command)], stderr=subprocess.STDOUT)
-				# need to set the niceness for the process group, not just the process in order to also apply to threads
-			except:
-				logging.warn('Could not set the nice value for the remote python script, ignoring...')
+			if self.remote_options.get('has_sudo', False) is True:
+				try:
+					subprocess.check_output(['ssh', self.remote_name, 'sudo renice -n -10 -g "$(pgrep --full --exact "{}")"'.format(command)], stderr=subprocess.STDOUT)
+					# need to set the niceness for the process group, not just the process in order to also apply to threads
+				except:
+					logging.warn('Could not set the nice value for the remote python script, ignoring...')
+				#
 			#
 			if next_action is not None:
 				next_action()
@@ -272,9 +305,12 @@ if __name__ == '__main__':
 	                    help='log helgrind data')
 	args = parser.parse_args()
 	#
+	#experiment_dir = '/var/ssd-raid/sengler/data/experiments'
+	experiment_dir = '/home/sengler/data/experiments'
+	#
 	experiment_time = time.time()
 	#base_save_data_path = os.path.join('/home/sengler/data/experiments', str(int(experiment_time)))
-	base_save_data_path = os.path.join('/var/ssd-raid/sengler/data/experiments', str(int(experiment_time)))
+	base_save_data_path = os.path.join(experiment_dir, str(int(experiment_time)))
 	os.mkdir(base_save_data_path)
 	#
 	measureme_log_path = None
@@ -283,17 +319,72 @@ if __name__ == '__main__':
 	start_time = time.time()
 	#
 	#tors = {'working':'/home/sengler/code/working/tor/src/app/tor', 'working-without':'/home/sengler/code/working/tor-without-tcmalloc/src/app/tor', 'dev-without':'/home/sengler/code/dev/tor-throughput-log-0.4.2.6-without-tcmalloc/src/app/tor', 'dev-with':'/home/sengler/code/dev/tor-throughput-log-0.4.2.6-with-tcmalloc/src/app/tor'}
+
+
+	#####tors = collections.OrderedDict()
+	#####tors['working'] = '/home/sengler/code/working/tor/src/app/tor'
+	#####tors['working-without'] = '/home/sengler/code/working/tor-without-tcmalloc/src/app/tor'
+	#####tors['dev-with'] = '/home/sengler/code/dev/tor-throughput-log-0.4.2.6-with-tcmalloc/src/app/tor'
+	#####tors['dev-without'] = '/home/sengler/code/dev/tor-throughput-log-0.4.2.6-without-tcmalloc/src/app/tor'
+	######hosts = ['sengler-rpi', 'cluck2']
+	#####hosts = ['grunt3']
+	######hosts = ['sengler-rpi']
+	#####num_repetitions = 15
+	#####nums_additional_eventloops_options = [0, 1, 2, 3]
+	######nums_additional_eventloops_options = [3, 2, 1, 0]
+
+	#tcmalloc_ld_preload = '/home/sengler/build/lib/libtcmalloc_and_profiler.so'
+	tcmalloc_ld_preload = '/home/sengler/build/lib/libtcmalloc.so'
+	jemalloc_ld_preload = '/home/sengler/build/lib/libjemalloc.so'
+
 	tors = collections.OrderedDict()
-	tors['working'] = '/home/sengler/code/working/tor/src/app/tor'
-	tors['working-without'] = '/home/sengler/code/working/tor-without-tcmalloc/src/app/tor'
-	tors['dev-with'] = '/home/sengler/code/dev/tor-throughput-log-0.4.2.6-with-tcmalloc/src/app/tor'
-	tors['dev-without'] = '/home/sengler/code/dev/tor-throughput-log-0.4.2.6-without-tcmalloc/src/app/tor'
-	hosts = ['sengler-rpi', 'cluck2']
-	###hosts = ['cluck2']
-	###hosts = ['sengler-rpi']
-	num_repetitions = 15
+	tors['multi-tcmalloc'] = ('/home/sengler/code/working/tor/src/app/tor', tcmalloc_ld_preload)
+	tors['multi-jemalloc'] = ('/home/sengler/code/working/tor/src/app/tor', jemalloc_ld_preload)
+	tors['multi-none'] =     ('/home/sengler/code/working/tor/src/app/tor', None)
+	tors['vanilla-tcmalloc'] = ('/home/sengler/code/dev/tor-0.4.2.6-throughput-log/src/app/tor', tcmalloc_ld_preload)
+	tors['vanilla-jemalloc'] = ('/home/sengler/code/dev/tor-0.4.2.6-throughput-log/src/app/tor', jemalloc_ld_preload)
+	tors['vanilla-none'] =     ('/home/sengler/code/dev/tor-0.4.2.6-throughput-log/src/app/tor', None)
+
+	configurations = {}
+	configurations['broken'] = {'num_clients': 150,
+	                            'num_guards': 30,      # number of relays (including guards)
+	                            'num_authorities': 2,  # will also act as a relay or guard
+	                            'num_exits': 30,       # will be used only as an exit
+	                            'num_streams_per_client': 10,
+	                            'num_bytes': 20*(2**20)}
+	configurations['full-server'] = {'num_clients': 150,
+	                                 'num_guards': 300,      # number of relays (including guards)
+	                                 'num_authorities': 3,  # will also act as a relay or guard
+	                                 'num_exits': 300,       # will be used only as an exit
+	                                 'num_streams_per_client': 10,
+	                                 'num_bytes': 10*(2**20)}
+	configurations['small-server'] = {'num_clients': 100,
+						              'num_guards': 300,     # number of relays (including guards)
+						              'num_authorities': 3,  # will also act as a relay or guard
+						              'num_exits': 300,      # will be used only as an exit
+						              'num_streams_per_client': 6,
+						              'num_bytes': 5*(2**20)}
+
+	remotes = collections.OrderedDict()
+	remotes['clack1'] = {'local_ip': '192.168.1.203',
+	                     'target_ip': '192.168.1.102',
+	                     'target_ssh': '192.168.1.102',
+	                     'target_chutney_net_dir': '/tmp/chutney-net',
+	                     'target_numa_settings': (0, [0, 32, 2, 34, 4, 36])}
+    #                     'target_cpu_prof': True}
+	remotes['sengler-rpi'] = {'local_ip': '129.97.119.248',
+	                          'target_ip': '129.97.169.9',
+	                          'target_ssh': '129.97.169.9',
+	                          'target_chutney_net_dir': '/tmp/chutney-net',
+	                          'has_sudo': True}
+
+	experiments = [('clack1', 'full-server'), ('sengler-rpi', 'small-server')]
+
+	num_repetitions = 10
 	nums_additional_eventloops_options = [0, 1, 2, 3]
 	#nums_additional_eventloops_options = [3, 2, 1, 0]
+
+
 	#
 	#tors = {'working':'/home/sengler/code/working/tor/src/app/tor', 'dev-without':'/home/sengler/code/dev/tor-throughput-log-0.4.2.6-without-tcmalloc/src/app/tor'}
 	#hosts = ['cluck2']
@@ -309,8 +400,18 @@ if __name__ == '__main__':
 	#
 	try:
 		for repeat in range(num_repetitions):
-			for host in hosts:
-				for (tor_name, tor_path) in tors.items():
+			for (remote_name, configuration_name) in experiments:
+				remote_options = remotes[remote_name]
+				configuration_options = configurations[configuration_name]
+				#
+				for (tor_name, (tor_path, tor_ld_preload)) in tors.items():
+					num_clients = configuration_options['num_clients']
+					num_guards = configuration_options['num_guards']
+					num_authorities = configuration_options['num_authorities']
+					num_exits = configuration_options['num_exits']
+					num_streams_per_client = configuration_options['num_streams_per_client']
+					num_bytes = configuration_options['num_bytes']
+					#
 					#num_clients = 4
 					#num_guards = 6 # number of relays (including guards)
 					#num_authorities = 2 # will also act as a relay or guard
@@ -323,21 +424,36 @@ if __name__ == '__main__':
 					#	num_exits = 12 # will be used only as an exit
 					#	num_streams_per_client = 3
 					#	num_bytes = 20*(2**20)
-					if host == 'cluck2':
+					'''
+					if remote_name == 'cluck2':
 						num_clients = 150
 						num_guards = 30 # number of relays (including guards)
 						num_authorities = 2 # will also act as a relay or guard
 						num_exits = 30 # will be used only as an exit
 						num_streams_per_client = 10
 						num_bytes = 20*(2**20)
-					elif host == 'sengler-rpi':
+					elif remote_name == 'cluck15':
+						num_clients = 150
+						num_guards = 30 # number of relays (including guards)
+						num_authorities = 2 # will also act as a relay or guard
+						num_exits = 30 # will be used only as an exit
+						num_streams_per_client = 10
+						num_bytes = 20*(2**20)
+					elif remote_name == 'grunt3':
+						num_clients = 150
+						num_guards = 30 # number of relays (including guards)
+						num_authorities = 2 # will also act as a relay or guard
+						num_exits = 30 # will be used only as an exit
+						num_streams_per_client = 10
+						num_bytes = 20*(2**20)
+					elif remote_name == 'sengler-rpi':
 						num_clients = 100
 						num_guards = 300 # number of relays (including guards)
 						num_authorities = 3 # will also act as a relay or guard
 						num_exits = 300 # will be used only as an exit
 						num_streams_per_client = 6
 						num_bytes = 5*(2**20)
-					elif host is None:
+					elif remote_name is None:
 						num_clients = 10
 						num_guards = 10 # number of relays (including guards)
 						num_authorities = 2 # will also act as a relay or guard
@@ -345,22 +461,23 @@ if __name__ == '__main__':
 						num_streams_per_client = 5
 						num_bytes = 20*(2**20)
 					else:
-						raise Exception('host not known')
+						raise Exception('remote not known')
 					#
+					'''
 					nums_additional_eventloops = [0]
-					if tor_name == 'working' or tor_name == 'working-without':
+					if tor_name.startswith('multi'):
 						nums_additional_eventloops = nums_additional_eventloops_options
 					#
 					for num_additional_eventloops in nums_additional_eventloops:
 						attempt = 0
 						while True:
 							attempt_str = '' if attempt == 0 else '_attempt-{}'.format(attempt)
-							save_data_path = os.path.join(base_save_data_path, '{}_{}_{}_{}{}'.format(host, tor_name, num_additional_eventloops, repeat, attempt_str))
+							save_data_path = os.path.join(base_save_data_path, '{}_{}_{}_{}{}'.format(remote_name, tor_name, num_additional_eventloops, repeat, attempt_str))
 							os.mkdir(save_data_path)
-							logging.info('Starting on {} using {}-{} ({}), repeat {}, attempt {}'.format(host, tor_name, num_additional_eventloops, tor_path, repeat, attempt))
+							logging.info('Starting on {} using {}-{} ({}, {}), repeat {}, attempt {}'.format(remote_name, tor_name, num_additional_eventloops, tor_path, tor_ld_preload, repeat, attempt))
 							#
 							#exp = CustomExperiment(args.helgrind, args.target_tor, save_data_path, measureme_log_path, args.num_bytes,
-							exp = CustomExperiment(args.helgrind, tor_path, num_additional_eventloops, host, save_data_path,
+							exp = CustomExperiment(args.helgrind, tor_path, tor_ld_preload, num_additional_eventloops, remote_name, remote_options, save_data_path,
 							                       measureme_log_path, num_bytes,
 							                       num_streams_per_client, num_clients, num_guards, num_authorities, num_exits,
 							                       build_circuit_generator, args.buffer_len, args.wait_range, measureme, test_network=False)
@@ -386,13 +503,16 @@ if __name__ == '__main__':
 									raise
 								#
 							#
-							os.system('rm -rf /run/user/3271/chutney-net/nodes/*/diff-cache')
-							os.system('rm -rf /run/user/3271/chutney-net/nodes/*/keys')
-							os.system('rm -f /run/user/3271/chutney-net/nodes/*/cached-*')
-							os.system('rm -f /run/user/3271/chutney-net/nodes/*/v3-status-votes')
-							shutil.copytree('/run/user/3271/chutney-net/nodes', os.path.join(save_data_path, 'nodes'))
+							chutney_data_dir = os.getenv('CHUTNEY_DATA_DIR')
+							assert chutney_data_dir is not None
+							#
+							os.system('rm -rf {}/nodes/*/diff-cache'.format(chutney_data_dir))
+							os.system('rm -rf {}/nodes/*/keys'.format(chutney_data_dir))
+							os.system('rm -f {}/nodes/*/cached-*'.format(chutney_data_dir))
+							os.system('rm -f {}/nodes/*/v3-status-votes'.format(chutney_data_dir))
+							shutil.copytree('{}/nodes'.format(chutney_data_dir), os.path.join(save_data_path, 'nodes'))
 							os.system("ps u | grep 'tor'")
-							os.system('rm -rf /run/user/3271/chutney-net/*')
+							os.system('rm -rf {}/*'.format(chutney_data_dir))
 							break
 						#
 						exp = None