plot_streams.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278
  1. #!/usr/bin/env python3
  2. #
  3. import gzip
  4. import sys
  5. import pickle
  6. import matplotlib.pylab as plt
  7. import numpy as np
  8. #
  9. import log_system_usage
  10. #
  11. def plot_cells(stream, ax, time_offset=None, clickable=False, label=None, color=None):
  12. num_bytes = np.cumsum(stream['length'])
  13. timestamps = np.asarray(stream['timestamp'])
  14. #
  15. if time_offset is not None:
  16. timestamps = timestamps-time_offset
  17. #
  18. return ax.step(timestamps, num_bytes/(1024**2), where='post', label=label, color=color, picker=(5 if clickable else None))[0]
  19. #
  20. def onresize(event):
  21. # matplotlib axes size only scales based on fractions (even with tight_layout), so we manually calculate a fixed padding size
  22. w = event.width/event.canvas.figure.dpi
  23. h = event.height/event.canvas.figure.dpi
  24. w_padding = 0.8 # in inches
  25. h_padding = 0.6 # in inches
  26. #
  27. for ax in event.canvas.figure.axes:
  28. for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] + ax.get_xticklabels() + ax.get_yticklabels()):
  29. item.set_fontsize(11+0.5*(w*h/(5**2)))
  30. #
  31. #
  32. event.canvas.figure.subplots_adjust(left=w_padding/w+0.01, right=1-((w_padding/2)/w), top=1-(h_padding/h), bottom=h_padding/h+0.01)
  33. #
  34. def zoom_axes_cb(event):
  35. scale = 2**(-event.step)
  36. ax = event.inaxes
  37. mouse_x = event.xdata
  38. mouse_y = event.ydata
  39. old_xlim = ax.get_xlim()
  40. old_ylim = ax.get_ylim()
  41. #
  42. ax.set_xlim([old_xlim[0]*scale+mouse_x*(1-scale), old_xlim[1]*scale+mouse_x*(1-scale)])
  43. ax.set_ylim([old_ylim[0]*scale+mouse_y*(1-scale), old_ylim[1]*scale+mouse_y*(1-scale)])
  44. #
  45. event.canvas.draw()
  46. #
  47. def pick_measureme_line_cb(event, lines, target_axes=None):
  48. if target_axes != None and event.mouseevent.inaxes not in target_axes:
  49. return
  50. #
  51. this_line = event.artist
  52. if event.mouseevent.button == 1 and not event.mouseevent.dblclick and event.mouseevent.key is None:
  53. # if the mouse is single-clicked and no keyboard key is held down
  54. if this_line.get_visible() and this_line in lines.keys():
  55. this_info = [x[1] for x in lines.items() if x[0] == this_line][0]
  56. #
  57. for_legend = []
  58. for (line, info) in lines.items():
  59. if info['measureme_id'] != this_info['measureme_id']:
  60. line.set_visible(False)
  61. else:
  62. line.set_visible(True)
  63. for_legend.append(line)
  64. #
  65. #
  66. event.mouseevent.inaxes.legend(for_legend, [x.get_label() for x in for_legend], loc='lower right')
  67. event.canvas.draw_idle()
  68. #
  69. #
  70. #
  71. def reset_measureme_lines_cb(event, lines, target_axes=None):
  72. if target_axes != None and event.inaxes not in target_axes:
  73. return
  74. #
  75. if event.button == 1 and event.dblclick and event.key is None:
  76. # if the mouse is double-clicked and no keyboard key is held down
  77. for (line, info) in lines.items():
  78. if info['is_main_plot']:
  79. line.set_visible(True)
  80. else:
  81. line.set_visible(False)
  82. #
  83. #
  84. legend = event.inaxes.get_legend()
  85. if legend != None:
  86. legend.remove()
  87. #
  88. event.canvas.draw_idle()
  89. #
  90. #
  91. def mouse_pan_cb(event, pan_settings):
  92. if event.inaxes is not None and event.key == 'control' and event.button == 1:
  93. ax = event.inaxes
  94. pixel_to_data = ax.transData.inverted()
  95. current_pos = pixel_to_data.transform_point((event.x, event.y))
  96. last_pos = pixel_to_data.transform_point((pan_settings['start_x'], pan_settings['start_y']))
  97. #
  98. old_xlim = ax.get_xlim()
  99. old_ylim = ax.get_ylim()
  100. #
  101. ax.set_xlim([old_xlim[0]+(last_pos[0]-current_pos[0]), old_xlim[1]+(last_pos[0]-current_pos[0])])
  102. ax.set_ylim([old_ylim[0]+(last_pos[1]-current_pos[1]), old_ylim[1]+(last_pos[1]-current_pos[1])])
  103. #
  104. event.canvas.draw_idle()
  105. #
  106. pan_settings['start_x'] = event.x
  107. pan_settings['start_y'] = event.y
  108. #
  109. def subsample_plot_cb(event, lines, target_axes=None):
  110. if target_axes != None and event.inaxes not in target_axes:
  111. return
  112. #
  113. if event.key == 'control':
  114. num_points = 0
  115. range_x = event.inaxes.xaxis.get_view_interval()
  116. range_y = event.inaxes.yaxis.get_view_interval()
  117. #
  118. for line in [l for l in lines if l.get_visible()]:
  119. data_x = line.get_xdata(orig=True)
  120. data_y = line.get_ydata(orig=True)
  121. #
  122. num_points += ((data_x>=range_x[0]) & (data_x<=range_x[1]) & (data_y>=range_y[0]) & (data_y<=range_y[1])).sum()
  123. # how many points are being rendered
  124. #
  125. for (line, info) in lines.items():
  126. data_x = line.get_xdata(orig=True)
  127. data_y = line.get_ydata(orig=True)
  128. line.orig_x = data_x
  129. line.orig_y = data_y
  130. line.orig_drawstyle = line.get_drawstyle()
  131. #
  132. subsample_spacing = max(1, int(num_points/10000))
  133. # the constant can be decreased to speed up plotting on slower computers
  134. #
  135. mask = np.ones(len(data_x))
  136. mask[::subsample_spacing] = 0
  137. line.set_xdata(data_x[mask==0])
  138. line.set_ydata(data_y[mask==0])
  139. line.set_drawstyle('default')
  140. #
  141. event.canvas.draw_idle()
  142. #
  143. #
  144. def undo_subsampling_cb(event, lines, target_axes=None):
  145. if target_axes != None and event.inaxes not in target_axes:
  146. return
  147. #
  148. if event.key == 'control':
  149. for (line, info) in lines.items():
  150. line.set_xdata(line.orig_x)
  151. line.set_ydata(line.orig_y)
  152. line.set_drawstyle(line.orig_drawstyle)
  153. #
  154. event.canvas.draw_idle()
  155. #
  156. #
  157. def get_complimentary_color(color_index):
  158. return (color_index+1 if color_index%2==0 else color_index-1)
  159. #
  160. if __name__ == '__main__':
  161. with gzip.GzipFile(sys.argv[1], 'rb') as f:
  162. streams = pickle.load(f)
  163. #
  164. with gzip.GzipFile(sys.argv[2], 'rb') as f:
  165. system_usage = pickle.load(f)
  166. system_usage['timestamps'] = np.array(system_usage['timestamps'])
  167. #
  168. with gzip.GzipFile(sys.argv[3], 'rb') as f:
  169. numa_data = pickle.load(f)
  170. #
  171. with gzip.GzipFile(sys.argv[4], 'rb') as f:
  172. fingerprints = pickle.load(f)
  173. #
  174. #fig, ax = plt.subplots()#constrained_layout=True
  175. fig, (ax, ax_cpu_usage) = plt.subplots(2, 1, sharex=True)
  176. #
  177. start_time = min([hop[t]['timestamp'][0] for m in streams for s in streams[m] for d in streams[m][s]
  178. for hop in streams[m][s][d] for t in ('received','sent')])
  179. #
  180. lines = {}
  181. assigned_colors = []
  182. colormap = plt.get_cmap('tab20') #'tab10'
  183. direction_shortforms = {'forward':'fwd', 'backward':'bwd'}
  184. transmission_shortforms = {'received':'recv', 'sent':'sent'}
  185. #
  186. for measureme_id in streams:
  187. # for each circuit
  188. for stream_id in streams[measureme_id]:
  189. # for each stream
  190. direction = 'forward'
  191. for hop_index in range(len(streams[measureme_id][stream_id][direction])):
  192. # for each hop in the circuit (including the OP)
  193. for transmission in ('received', 'sent'):
  194. data = streams[measureme_id][stream_id][direction][hop_index][transmission]
  195. #
  196. if hop_index == len(streams[measureme_id][stream_id][direction])-1:
  197. guard_fingerprint = streams[measureme_id][stream_id][direction][1]['fingerprint']
  198. if guard_fingerprint not in assigned_colors:
  199. assigned_colors.append(guard_fingerprint)
  200. #
  201. if transmission == 'sent':
  202. color_index = assigned_colors.index(guard_fingerprint)
  203. else:
  204. color_index = get_complimentary_color(assigned_colors.index(guard_fingerprint))
  205. #
  206. else:
  207. color_index = hop_index*2 + ('received', 'sent').index(transmission)
  208. #
  209. is_main_plot = (hop_index == len(streams[measureme_id][stream_id][direction])-1 and transmission == 'sent')
  210. direction_label = direction_shortforms[direction]
  211. transmission_label = transmission_shortforms[transmission]
  212. fingerprint = str(streams[measureme_id][stream_id][direction][hop_index]['fingerprint'])
  213. label = 'hop={}, {:.4}, {:.4}, mid={}, sid={}, fprnt={:.6s}'.format(hop_index, direction_label, transmission_label,
  214. measureme_id, stream_id, fingerprint)
  215. line = plot_cells(data, ax, time_offset=start_time, clickable=is_main_plot, label=label, color=colormap(color_index))
  216. if not is_main_plot:
  217. line.set_visible(False)
  218. #
  219. lines[line] = {'measureme_id':measureme_id, 'stream_id':stream_id, 'direction':direction,
  220. 'hop_index':hop_index, 'transmission':transmission, 'is_main_plot':is_main_plot}
  221. #
  222. #
  223. #
  224. #
  225. guard_counter = {}
  226. for measureme_id in streams:
  227. for stream_id in streams[measureme_id]:
  228. direction = 'forward'
  229. fingerprint = streams[measureme_id][stream_id][direction][1]['fingerprint']
  230. if fingerprint not in guard_counter:
  231. guard_counter[fingerprint] = 0
  232. #
  233. guard_counter[fingerprint] += 1
  234. #
  235. #
  236. system_usage_timestamps = (system_usage['timestamps'][1:]+system_usage['timestamps'][:-1])/2 - start_time
  237. cpu_usages = {int(cpu): np.array(log_system_usage.calculate_cpu_usage_continuous(system_usage['stats']['cpus'][cpu])) for cpu in system_usage['stats']['cpus']}
  238. tor_cpu_usages = {nick: np.sum([cpu_usages[cpu] for cpu in numa_data[nick][1]], axis=0) for nick in numa_data}
  239. #
  240. mask = system_usage_timestamps > 0
  241. system_usage_timestamps = system_usage_timestamps[mask]
  242. tor_cpu_usages = {nick: tor_cpu_usages[nick][mask] for nick in tor_cpu_usages}
  243. #
  244. for nick in sorted(fingerprints.keys()):
  245. #if fingerprints[nick] != None:
  246. # is not an OP
  247. guard_circuit_count = 0
  248. if fingerprints[nick] in guard_counter:
  249. guard_circuit_count = guard_counter[fingerprints[nick]]
  250. elif fingerprints[nick] is None:
  251. guard_circuit_count = '?'
  252. #
  253. ax_cpu_usage.plot(system_usage_timestamps, tor_cpu_usages[nick]*100, label='{} / {:.6s} (guard for {} circuits)'.format(nick, str(fingerprints[nick]),
  254. guard_circuit_count))
  255. #
  256. #
  257. ax_cpu_usage.set_xlabel('Time (s)')
  258. ax_cpu_usage.set_ylabel('CPU Usage (%)')
  259. ax_cpu_usage.legend()
  260. ax_cpu_usage.grid(linestyle=':')
  261. #
  262. ax.set_xlabel('Time (s)')
  263. ax.set_ylabel('Data (MiB)')
  264. ax.set_title(sys.argv[1])
  265. ax.grid(linestyle=':')
  266. fig.tight_layout(pad=0)
  267. #ax.set_ylim(0, None)
  268. #
  269. fig.canvas.mpl_connect('resize_event', onresize)
  270. fig.canvas.mpl_connect('scroll_event', zoom_axes_cb)
  271. fig.canvas.mpl_connect('pick_event', lambda event,lines=lines,axes=[ax]: pick_measureme_line_cb(event, lines, target_axes=axes))
  272. fig.canvas.mpl_connect('button_press_event', lambda event,lines=lines,axes=[ax]: reset_measureme_lines_cb(event, lines, target_axes=axes))
  273. fig.canvas.mpl_connect('motion_notify_event', lambda event,pan_settings={'start_x':0,'start_y':0}: mouse_pan_cb(event, pan_settings))
  274. fig.canvas.mpl_connect('key_press_event', lambda event,lines=lines,axes=[ax]: subsample_plot_cb(event, lines, target_axes=axes))
  275. fig.canvas.mpl_connect('key_release_event', lambda event,lines=lines,axes=[ax]: undo_subsampling_cb(event, lines, target_axes=axes))
  276. #
  277. plt.show(fig)
  278. #