relay_working_experiment.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536
  1. #!/usr/bin/python3
  2. #
  3. import argparse
  4. import shutil
  5. import logging
  6. import random
  7. import os
  8. import subprocess
  9. import multiprocessing
  10. import threading
  11. import time
  12. import json
  13. import gzip
  14. import pickle
  15. import tempfile
  16. import collections
  17. import gc
  18. #
  19. import stem.control
  20. import stem.descriptor.remote
  21. import stem.process
  22. #
  23. import numa
  24. import log_system_usage
  25. import chutney_manager
  26. import throughput_server
  27. import experiment_client
  28. import experiment
  29. import useful
  30. #
  31. #remote_name = 'sengler-rpi'
  32. #remote_name = 'cluck2'
  33. #remote_name = None
  34. #
  35. class CustomExperiment(experiment.Experiment):
  36. def __init__(self, use_helgrind, target_tor, target_ld_preload, num_additional_eventloops, remote_name, remote_options, *args, **kwargs):
  37. self.use_helgrind = use_helgrind
  38. self.target_tor = target_tor
  39. self.target_ld_preload = target_ld_preload
  40. self.num_additional_eventloops = num_additional_eventloops
  41. self.remote_name = remote_name
  42. self.remote_options = remote_options
  43. super().__init__(*args, **kwargs)
  44. #
  45. self.chutney_path = '/root/code/working/chutney'
  46. self.tor_path = '/root/code/dev/tor-0.4.2.6-fixed-controller'
  47. #
  48. def configure_chutney(self):
  49. #self.nodes = [chutney_manager.Node(tag='a', relay=1, authority=1, torrc='authority.tmpl') for _ in range(self.num_authorities)] + \
  50. # [chutney_manager.Node(tag='r', relay=1, torrc='relay-non-exit.tmpl') for _ in range(self.num_guards)] + \
  51. # [chutney_manager.Node(tag='e', exit=1, torrc='relay.tmpl') for _ in range(self.num_exits)] + \
  52. # [chutney_manager.Node(tag='c', client=1, torrc='client.tmpl') for _ in range(self.num_clients)]
  53. #
  54. #target_tor_path = '/root/code/working/tor/src/app/tor'
  55. #target_tor_path = '/root/code/releases/tor-0.4.2.5/src/app/tor'
  56. '''
  57. if self.remote_name == 'cluck2':
  58. local_ip = '172.19.156.16'
  59. target_ip = '172.19.156.136'
  60. #local_ip = '129.97.119.196'
  61. #target_ip = '129.97.119.226'
  62. target_hostname = 'cluck2'
  63. target_dir = '/tmp/chutney-net'
  64. #
  65. elif self.remote_name == 'cluck15':
  66. local_ip = '172.19.156.16'
  67. target_ip = '172.19.156.188'
  68. target_hostname = '129.97.119.239'
  69. target_dir = '/tmp/chutney-net'
  70. #
  71. elif self.remote_name == 'grunt3':
  72. #local_ip = '172.19.156.16'
  73. #target_ip = '172.19.156.44'
  74. #local_ip = '129.97.119.196'
  75. #target_ip = '129.97.119.203'
  76. target_hostname = '129.97.119.203'
  77. target_dir = '/tmp/chutney-net'
  78. #
  79. elif self.remote_name == 'sengler-rpi':
  80. local_ip = '129.97.119.196'
  81. target_ip = '129.97.169.9'
  82. target_hostname = target_ip
  83. target_dir = '/tmp/chutney-net'
  84. #
  85. elif self.remote_name is None:
  86. local_ip = None
  87. target_ip = None
  88. target_hostname = None
  89. target_dir = None
  90. else:
  91. raise Exception('hostname not known')
  92. #
  93. '''
  94. target_optional_args = {}
  95. if self.target_tor is not None:
  96. target_optional_args['tor'] = self.target_tor
  97. if self.use_helgrind:
  98. target_optional_args['valgrind_settings'] = ['--tool=helgrind', '-v', '--suppressions=libevent.supp', '--read-var-info=yes']
  99. if self.target_ld_preload is not None:
  100. target_optional_args['add_environ_vars'] = {'LD_PRELOAD': self.target_ld_preload}
  101. #target_optional_args['add_environ_vars'] = {'LD_PRELOAD': '/usr/lib/libprofiler.so.0'}
  102. #target_optional_args['add_environ_vars'] = {'LD_PRELOAD': '/usr/lib/libtcmalloc_and_profiler.so.4'}
  103. #target_optional_args['add_environ_vars'] = {'LD_PRELOAD': '/root/build/lib/libtcmalloc_and_profiler.so'}
  104. #target_optional_args['add_environ_vars'] = {'EVENT_NOEPOLL': '', 'EVENT_SHOW_METHOD': ''}
  105. if self.remote_options['target_ip'] is not None:
  106. target_optional_args['ip'] = self.remote_options['target_ip']
  107. if self.remote_options['target_ssh'] is not None:
  108. target_optional_args['remote_hostname'] = self.remote_options['target_ssh']
  109. if self.remote_options['target_chutney_net_dir'] is not None:
  110. target_optional_args['remote_net_dir'] = self.remote_options['target_chutney_net_dir']
  111. if self.remote_options.get('target_numa_settings', None) is not None:
  112. target_optional_args['numa_settings'] = self.remote_options['target_numa_settings']
  113. #
  114. target_optional_args['num_cpus'] = 2 # make sure it can process onion skins fast enough, and keep it consistent between computers
  115. # tor actually uses one more worker thread than what you ask for
  116. target_optional_args['num_additional_eventloops'] = self.num_additional_eventloops
  117. target_optional_args['dircache'] = False
  118. # the voting interval is 40 seconds which puts an unrealistic workload on the target, so we disable it
  119. if 'target_cpu_prof' in self.remote_options:
  120. target_cpu_prof = self.remote_options['target_cpu_prof']
  121. else:
  122. target_cpu_prof = False
  123. #
  124. target_daemon = False
  125. target_log_throughput = True
  126. target_logs = ['notice']
  127. #other_logs = ['info', 'notice']
  128. other_logs = ['notice']
  129. #if self.use_helgrind:
  130. # valgrind_settings = ['--tool=helgrind', '-v', '--suppressions=libevent.supp', '--read-var-info=yes']
  131. #else:
  132. # valgrind_settings = None
  133. #
  134. self.nodes = [chutney_manager.Node(tag='a', relay=1, authority=1, torrc='authority.tmpl', log_files=other_logs) for _ in range(self.num_authorities)] + \
  135. [chutney_manager.Node(tag='r', relay=1, torrc='relay-non-exit.tmpl', log_files=other_logs) for _ in range(self.num_guards)] + \
  136. [chutney_manager.Node(tag='target', relay=1, torrc='relay-non-exit.tmpl', log_throughput=target_log_throughput,
  137. daemon=target_daemon, log_files=target_logs, sandbox=0, google_cpu_profiler=target_cpu_prof, **target_optional_args)] + \
  138. [chutney_manager.Node(tag='e', exit=1, torrc='relay.tmpl', log_files=other_logs) for _ in range(self.num_exits)] + \
  139. [chutney_manager.Node(tag='c', client=1, torrc='client.tmpl', log_files=other_logs) for _ in range(self.num_clients)]
  140. #
  141. for node in self.nodes:
  142. if not 'num_cpus' in node.options:
  143. node.options['num_cpus'] = 2
  144. #
  145. if not 'ip' in node.options and self.remote_options['local_ip'] is not None:
  146. node.options['ip'] = self.remote_options['local_ip']
  147. #
  148. #
  149. #numa_remaining = numa.get_numa_overview()
  150. #for (node, index) in zip(self.nodes, range(len(self.nodes))):
  151. # num_cpus = node.options['num_cpus']
  152. # if num_cpus%2 != 0:
  153. # num_cpus += 1
  154. # #
  155. # if node.options['tag'] == 'target':
  156. # num_cpus = max(num_cpus, 6)
  157. # #
  158. # #if node.options['tag'] != 'target':
  159. # # (numa_node, processors) = chutney_manager.numa_scheduler(num_cpus, numa_remaining)
  160. # # node.options['numa_settings'] = (numa_node, processors)
  161. # #
  162. ##
  163. self.proxy_control_ports = [self.nodes[x].guess_control_port(x) for x in range(len(self.nodes)) if ('client', 1) in self.nodes[x].options.items()]
  164. # TODO: ^^ improve this
  165. #
  166. def start_remote_logging(self, next_action=None):
  167. if self.remote_name is None:
  168. # running locally
  169. if next_action is not None:
  170. next_action()
  171. #
  172. return
  173. #
  174. local_script_path = 'log_system_usage.py'
  175. remote_script_path = '/tmp/log_system_usage.py'
  176. remote_save_path = '/tmp/cpu-usage.pickle.gz'
  177. local_save_path = os.path.join(self.save_data_path, 'remote-cpu-usage.pickle.gz')
  178. #
  179. tor_pids = subprocess.check_output(['ssh', self.remote_name, 'pgrep tor']).decode('utf-8').split()
  180. tor_pids = [pid for pid in tor_pids]
  181. logging.info('Logging the following pids on {}: {}'.format(self.remote_name, tor_pids))
  182. command = 'python3 {} --interval 0.2 --pids {} {}'.format(remote_script_path, ','.join(tor_pids), remote_save_path)
  183. #
  184. try:
  185. subprocess.check_output(['scp', local_script_path, '{}:{}'.format(self.remote_name, remote_script_path)], stderr=subprocess.STDOUT)
  186. p = subprocess.Popen(['ssh', self.remote_name, command])
  187. #
  188. time.sleep(5)
  189. # wait a few seconds to make sure it doesn't exit immediately
  190. if p.poll() != None:
  191. raise Exception('Remote CPU monitoring script exited immediately')
  192. #
  193. if self.remote_options.get('has_sudo', False) is True:
  194. try:
  195. subprocess.check_output(['ssh', self.remote_name, 'sudo renice -n -10 -g "$(pgrep --full --exact "{}")"'.format(command)], stderr=subprocess.STDOUT)
  196. # need to set the niceness for the process group, not just the process in order to also apply to threads
  197. except:
  198. logging.warn('Could not set the nice value for the remote python script, ignoring...')
  199. #
  200. #
  201. if next_action is not None:
  202. next_action()
  203. time.sleep(5)
  204. # wait a few seconds so that we have extra data
  205. # this may be useful if we need to do averaging
  206. #
  207. if p.poll() != None:
  208. raise Exception('Remote CPU monitoring script exited before it was supposed to')
  209. #
  210. finally:
  211. try:
  212. subprocess.check_output(['ssh', self.remote_name, 'pkill --full --exact --signal sigint \'{}\''.format(command)], stderr=subprocess.STDOUT)
  213. except:
  214. logging.warn('Could not kill remote python script')
  215. #
  216. try:
  217. p.wait(timeout=30)
  218. except subprocess.TimeoutExpired:
  219. p.kill()
  220. logging.warn('Process did not end as expected, so sent a SIGKILL')
  221. except:
  222. logging.warn('Could not kill')
  223. #
  224. try:
  225. subprocess.check_output(['scp', '{}:{}'.format(self.remote_name, remote_save_path), local_save_path], stderr=subprocess.STDOUT)
  226. except:
  227. logging.warn('Failed to get remote \'{}\' data file'.format(remote_save_path))
  228. #
  229. try:
  230. subprocess.check_output(['ssh', self.remote_name, 'rm', remote_save_path], stderr=subprocess.STDOUT)
  231. except:
  232. logging.warn('Failed to delete remote \'{}\' data file'.format(remote_save_path))
  233. #
  234. try:
  235. subprocess.check_output(['ssh', self.remote_name, 'rm', remote_script_path], stderr=subprocess.STDOUT)
  236. except:
  237. logging.warn('Failed to delete remote \'{}\' script file'.format(remote_script_path))
  238. #
  239. #
  240. #
  241. #
  242. def build_circuit_generator(consensus, server_address):
  243. fingerprints = [desc.nickname for desc in consensus]
  244. exit_fingerprints = [desc.nickname for desc in consensus if desc.exit_policy.can_exit_to(*server_address)]
  245. authority_fingerprints = [desc.nickname for desc in consensus if desc.nickname.endswith('a')]
  246. #
  247. target_fingerprints = [desc.nickname for desc in consensus if desc.nickname.endswith('target')]
  248. assert len(target_fingerprints) >= 1, 'No target relay in the consensus'
  249. non_exit_fingerprints = list(set(fingerprints)-set(exit_fingerprints)-set(target_fingerprints)-set(authority_fingerprints))
  250. #
  251. assert len(exit_fingerprints) >= 1, 'Need at least one exit relay'
  252. assert len(non_exit_fingerprints) >= 1, 'Need at least one non-exit relay'
  253. #
  254. non_exit_fingerprints = sorted(non_exit_fingerprints)
  255. target_fingerprints = sorted(target_fingerprints)
  256. exit_fingerprints = sorted(exit_fingerprints)
  257. # try to get reproducible behavior
  258. #
  259. #return lambda gen_id=None: [random.choice(non_exit_fingerprints), target_fingerprint, random.choice(exit_fingerprints)]
  260. return lambda gen_id: [non_exit_fingerprints[gen_id%len(non_exit_fingerprints)], target_fingerprints[gen_id%len(target_fingerprints)], exit_fingerprints[gen_id%len(exit_fingerprints)]]
  261. '''
  262. fingerprints = [desc.fingerprint for desc in consensus]
  263. exit_fingerprints = [desc.fingerprint for desc in consensus if desc.exit_policy.can_exit_to(*server_address)]
  264. #
  265. target_fingerprint = [desc.fingerprint for desc in consensus if desc.nickname.endswith('target')][0]
  266. non_exit_fingerprints = list(set(fingerprints)-set(exit_fingerprints)-set([target_fingerprint]))
  267. #
  268. assert len(exit_fingerprints) >= 1, 'Need at least one exit relay'
  269. assert len(non_exit_fingerprints) >= 1, 'Need at least one non-exit relay'
  270. #
  271. #return lambda gen_id=None: [random.choice(non_exit_fingerprints), target_fingerprint, random.choice(exit_fingerprints)]
  272. return lambda gen_id: [non_exit_fingerprints[gen_id%len(non_exit_fingerprints)], target_fingerprint, exit_fingerprints[gen_id%len(exit_fingerprints)]]
  273. '''
  274. #
  275. def existing_file(path):
  276. if not os.path.isfile(path):
  277. raise argparse.ArgumentTypeError('The file path is not valid')
  278. return path
  279. #
  280. if __name__ == '__main__':
  281. #
  282. logging.basicConfig(level=logging.DEBUG)
  283. logging.getLogger('stem').setLevel(logging.WARNING)
  284. #
  285. parser = argparse.ArgumentParser(description='Test the network throughput.')
  286. parser.add_argument('num_bytes', type=useful.parse_bytes,
  287. help='number of bytes to send per connection (can also end with \'B\', \'KiB\', \'MiB\', or \'GiB\')', metavar='num-bytes')
  288. parser.add_argument('--buffer-len', type=useful.parse_bytes,
  289. help='size of the send and receive buffers (can also end with \'B\', \'KiB\', \'MiB\', or \'GiB\')', metavar='bytes')
  290. parser.add_argument('--wait-range', type=int, default=0,
  291. help='add a random wait time to each connection so that they don\'t all start at the same time (default is 0)', metavar='time')
  292. parser.add_argument('--target-tor', type=existing_file, default=None,
  293. help='use a different tor binary for the target', metavar='tor-path')
  294. parser.add_argument('--helgrind', action='store_true',
  295. help='log helgrind data')
  296. args = parser.parse_args()
  297. #
  298. #experiment_dir = '/var/ssd-raid/sengler/data/experiments'
  299. experiment_dir = '/results'
  300. #
  301. experiment_time = time.time()
  302. #base_save_data_path = os.path.join('/root/data/experiments', str(int(experiment_time)))
  303. base_save_data_path = os.path.join(experiment_dir, str(int(experiment_time)))
  304. os.mkdir(base_save_data_path)
  305. #
  306. measureme_log_path = None
  307. measureme = False
  308. #
  309. start_time = time.time()
  310. #
  311. #tors = {'working':'/root/code/working/tor/src/app/tor', 'working-without':'/root/code/working/tor-without-tcmalloc/src/app/tor', 'dev-without':'/root/code/dev/tor-throughput-log-0.4.2.6-without-tcmalloc/src/app/tor', 'dev-with':'/root/code/dev/tor-throughput-log-0.4.2.6-with-tcmalloc/src/app/tor'}
  312. #####tors = collections.OrderedDict()
  313. #####tors['working'] = '/root/code/working/tor/src/app/tor'
  314. #####tors['working-without'] = '/root/code/working/tor-without-tcmalloc/src/app/tor'
  315. #####tors['dev-with'] = '/root/code/dev/tor-throughput-log-0.4.2.6-with-tcmalloc/src/app/tor'
  316. #####tors['dev-without'] = '/root/code/dev/tor-throughput-log-0.4.2.6-without-tcmalloc/src/app/tor'
  317. ######hosts = ['sengler-rpi', 'cluck2']
  318. #####hosts = ['grunt3']
  319. ######hosts = ['sengler-rpi']
  320. #####num_repetitions = 15
  321. #####nums_additional_eventloops_options = [0, 1, 2, 3]
  322. ######nums_additional_eventloops_options = [3, 2, 1, 0]
  323. #tcmalloc_ld_preload = '/root/build/lib/libtcmalloc_and_profiler.so'
  324. tcmalloc_ld_preload = '/root/build/lib/libtcmalloc.so'
  325. jemalloc_ld_preload = '/root/build/lib/libjemalloc.so'
  326. tors = collections.OrderedDict()
  327. tors['multi-tcmalloc'] = ('/root/code/working/tor/src/app/tor', tcmalloc_ld_preload)
  328. tors['multi-jemalloc'] = ('/root/code/working/tor/src/app/tor', jemalloc_ld_preload)
  329. tors['multi-none'] = ('/root/code/working/tor/src/app/tor', None)
  330. tors['vanilla-tcmalloc'] = ('/root/code/dev/tor-0.4.2.6-throughput-log/src/app/tor', tcmalloc_ld_preload)
  331. tors['vanilla-jemalloc'] = ('/root/code/dev/tor-0.4.2.6-throughput-log/src/app/tor', jemalloc_ld_preload)
  332. tors['vanilla-none'] = ('/root/code/dev/tor-0.4.2.6-throughput-log/src/app/tor', None)
  333. configurations = {}
  334. configurations['broken'] = {'num_clients': 150,
  335. 'num_guards': 30, # number of relays (including guards)
  336. 'num_authorities': 2, # will also act as a relay or guard
  337. 'num_exits': 30, # will be used only as an exit
  338. 'num_streams_per_client': 10,
  339. 'num_bytes': 20*(2**20)}
  340. configurations['full-server'] = {'num_clients': 150,
  341. 'num_guards': 300, # number of relays (including guards)
  342. 'num_authorities': 3, # will also act as a relay or guard
  343. 'num_exits': 300, # will be used only as an exit
  344. 'num_streams_per_client': 10,
  345. 'num_bytes': 10*(2**20)}
  346. configurations['small-server'] = {'num_clients': 100,
  347. 'num_guards': 300, # number of relays (including guards)
  348. 'num_authorities': 3, # will also act as a relay or guard
  349. 'num_exits': 300, # will be used only as an exit
  350. 'num_streams_per_client': 6,
  351. 'num_bytes': 5*(2**20)}
  352. configurations['tiny'] = {'num_clients': 10,
  353. 'num_guards': 10, # number of relays (including guards)
  354. 'num_authorities': 3, # will also act as a relay or guard
  355. 'num_exits': 10, # will be used only as an exit
  356. 'num_streams_per_client': 1,
  357. 'num_bytes': 5*(2**20)}
  358. remotes = collections.OrderedDict()
  359. remotes['clack1'] = {'local_ip': '192.168.1.203',
  360. 'target_ip': '192.168.1.102',
  361. 'target_ssh': '192.168.1.102',
  362. 'target_chutney_net_dir': '/tmp/chutney-net',
  363. 'target_numa_settings': (0, [0, 32, 2, 34, 4, 36])}
  364. # 'target_cpu_prof': True}
  365. remotes['sengler-rpi'] = {'local_ip': '129.97.119.248',
  366. 'target_ip': '129.97.169.9',
  367. 'target_ssh': '129.97.169.9',
  368. 'target_chutney_net_dir': '/tmp/chutney-net',
  369. 'has_sudo': True}
  370. remotes['localhost'] = {'local_ip': '127.0.0.1',
  371. 'target_ip': '127.0.0.1',
  372. 'target_ssh': '127.0.0.1',
  373. 'target_chutney_net_dir': '/tmp/chutney-net'}
  374. experiments = [('clack1', 'full-server'), ('sengler-rpi', 'small-server')]
  375. num_repetitions = 10
  376. nums_additional_eventloops_options = [0, 1, 2, 3]
  377. #nums_additional_eventloops_options = [3, 2, 1, 0]
  378. #
  379. #tors = {'working':'/root/code/working/tor/src/app/tor', 'dev-without':'/root/code/dev/tor-throughput-log-0.4.2.6-without-tcmalloc/src/app/tor'}
  380. #hosts = ['cluck2']
  381. #num_repetitions = 1
  382. #nums_additional_eventloops_options = [0, 1, 2, 3]
  383. #
  384. #tors = {'dev-debug-stall':'/root/code/dev/tor-throughput-log-0.4.2.6-debug-stall/src/app/tor'}
  385. #tors = {'dev-without':'/root/code/dev/tor-throughput-log-0.4.2.6-test-kist-changes/src/app/tor'}
  386. #tors = {'dev-without':'/root/code/dev/tor-throughput-log-0.4.2.6-without-tcmalloc/src/app/tor'}
  387. #hosts = ['cluck2']
  388. #num_repetitions = 5
  389. #nums_additional_eventloops_options = [3, 2, 1, 0]
  390. #
  391. try:
  392. for repeat in range(num_repetitions):
  393. for (remote_name, configuration_name) in experiments:
  394. remote_options = remotes[remote_name]
  395. configuration_options = configurations[configuration_name]
  396. #
  397. for (tor_name, (tor_path, tor_ld_preload)) in tors.items():
  398. num_clients = configuration_options['num_clients']
  399. num_guards = configuration_options['num_guards']
  400. num_authorities = configuration_options['num_authorities']
  401. num_exits = configuration_options['num_exits']
  402. num_streams_per_client = configuration_options['num_streams_per_client']
  403. num_bytes = configuration_options['num_bytes']
  404. #
  405. #num_clients = 4
  406. #num_guards = 6 # number of relays (including guards)
  407. #num_authorities = 2 # will also act as a relay or guard
  408. #num_exits = 8 # will be used only as an exit
  409. #num_streams_per_client = 1
  410. #if True:
  411. # num_clients = 4
  412. # num_guards = 10 # number of relays (including guards)
  413. # num_authorities = 2 # will also act as a relay or guard
  414. # num_exits = 12 # will be used only as an exit
  415. # num_streams_per_client = 3
  416. # num_bytes = 20*(2**20)
  417. '''
  418. if remote_name == 'cluck2':
  419. num_clients = 150
  420. num_guards = 30 # number of relays (including guards)
  421. num_authorities = 2 # will also act as a relay or guard
  422. num_exits = 30 # will be used only as an exit
  423. num_streams_per_client = 10
  424. num_bytes = 20*(2**20)
  425. elif remote_name == 'cluck15':
  426. num_clients = 150
  427. num_guards = 30 # number of relays (including guards)
  428. num_authorities = 2 # will also act as a relay or guard
  429. num_exits = 30 # will be used only as an exit
  430. num_streams_per_client = 10
  431. num_bytes = 20*(2**20)
  432. elif remote_name == 'grunt3':
  433. num_clients = 150
  434. num_guards = 30 # number of relays (including guards)
  435. num_authorities = 2 # will also act as a relay or guard
  436. num_exits = 30 # will be used only as an exit
  437. num_streams_per_client = 10
  438. num_bytes = 20*(2**20)
  439. elif remote_name == 'sengler-rpi':
  440. num_clients = 100
  441. num_guards = 300 # number of relays (including guards)
  442. num_authorities = 3 # will also act as a relay or guard
  443. num_exits = 300 # will be used only as an exit
  444. num_streams_per_client = 6
  445. num_bytes = 5*(2**20)
  446. elif remote_name is None:
  447. num_clients = 10
  448. num_guards = 10 # number of relays (including guards)
  449. num_authorities = 2 # will also act as a relay or guard
  450. num_exits = 12 # will be used only as an exit
  451. num_streams_per_client = 5
  452. num_bytes = 20*(2**20)
  453. else:
  454. raise Exception('remote not known')
  455. #
  456. '''
  457. nums_additional_eventloops = [0]
  458. if tor_name.startswith('multi'):
  459. nums_additional_eventloops = nums_additional_eventloops_options
  460. #
  461. for num_additional_eventloops in nums_additional_eventloops:
  462. attempt = 0
  463. while True:
  464. attempt_str = '' if attempt == 0 else '_attempt-{}'.format(attempt)
  465. save_data_path = os.path.join(base_save_data_path, '{}_{}_{}_{}{}'.format(remote_name, tor_name, num_additional_eventloops, repeat, attempt_str))
  466. os.mkdir(save_data_path)
  467. logging.info('Starting on {} using {}-{} ({}, {}), repeat {}, attempt {}'.format(remote_name, tor_name, num_additional_eventloops, tor_path, tor_ld_preload, repeat, attempt))
  468. #
  469. #exp = CustomExperiment(args.helgrind, args.target_tor, save_data_path, measureme_log_path, args.num_bytes,
  470. exp = CustomExperiment(args.helgrind, tor_path, tor_ld_preload, num_additional_eventloops, remote_name, remote_options, save_data_path,
  471. measureme_log_path, num_bytes,
  472. num_streams_per_client, num_clients, num_guards, num_authorities, num_exits,
  473. build_circuit_generator, args.buffer_len, args.wait_range, measureme, test_network=False)
  474. #
  475. def sleep_then_run(duration, func):
  476. logging.info('Sleeping for {} seconds before running \'{}\''.format(duration, func.__name__))
  477. time.sleep(duration)
  478. logging.info('Done sleeping')
  479. return func()
  480. #
  481. #import subprocess
  482. #p = subprocess.Popen(['ssh', '-t', 'sengler-rpi', 'python3 /tmp/log_system_usage.py /tmp/usage.gz'])
  483. #
  484. try:
  485. exp.start_chutney(lambda: exp.start_throughput_server(lambda: sleep_then_run(120, lambda: exp.start_system_logging(lambda: exp.start_remote_logging(exp.start_throughput_clients)))))
  486. except (stem.Timeout, stem.CircuitExtensionFailed, experiment.RepeatExperimentError):
  487. tries = 9
  488. attempt += 1
  489. if attempt < tries:
  490. logging.exception('Experiment run failed, trying again ({} tries remaining)'.format(tries-attempt))
  491. continue
  492. else:
  493. raise
  494. #
  495. #
  496. chutney_data_dir = os.getenv('CHUTNEY_DATA_DIR')
  497. assert chutney_data_dir is not None
  498. #
  499. os.system('rm -rf {}/nodes/*/diff-cache'.format(chutney_data_dir))
  500. os.system('rm -rf {}/nodes/*/keys'.format(chutney_data_dir))
  501. os.system('rm -f {}/nodes/*/cached-*'.format(chutney_data_dir))
  502. os.system('rm -f {}/nodes/*/v3-status-votes'.format(chutney_data_dir))
  503. shutil.copytree('{}/nodes'.format(chutney_data_dir), os.path.join(save_data_path, 'nodes'))
  504. os.system("ps u | grep 'tor'")
  505. os.system('rm -rf {}/*'.format(chutney_data_dir))
  506. break
  507. #
  508. exp = None
  509. gc.collect()
  510. # not sure if this is actually useful, but hopefully it reduces memory usage for when we need to fork
  511. #
  512. #
  513. #
  514. #
  515. except KeyboardInterrupt:
  516. logging.info('Stopped (KeyboardInterrupt)')
  517. #
  518. logging.info('Total time: {:.2f} minutes'.format((time.time()-start_time)/60))
  519. #