plot_combined.py 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162
  1. import sys
  2. import os
  3. import time
  4. #
  5. import numpy as np
  6. import matplotlib.pylab as plt
  7. #
  8. import data_helpers
  9. import log_system_usage
  10. #
  11. if __name__ == '__main__':
  12. initial_time = time.time()
  13. #
  14. title = sys.argv[1]
  15. (_, throughput_data) = data_helpers.read_relay_throughput(sys.argv[2])
  16. client_data = data_helpers.read_client_info(sys.argv[3])
  17. server_data = data_helpers.read_server_results(sys.argv[4])
  18. try:
  19. host_system_usage = log_system_usage.load_cpu_stats(sys.argv[5])
  20. remote_system_usage = log_system_usage.load_cpu_stats(sys.argv[6])
  21. except FileNotFoundError:
  22. print('The system usage logs weren\'t found, so skipping them...')
  23. plot_cpu_usage = False
  24. else:
  25. plot_cpu_usage = True
  26. #
  27. print('Loaded Files: {}'.format(time.time()-initial_time))
  28. #
  29. norm_throughput = data_helpers.normalize_relay_throughput(throughput_data)
  30. start_time = client_data['start_time']
  31. last_byte_times = np.array([x['time_of_last_byte'] for x in server_data])
  32. end_time = np.max(last_byte_times)
  33. #
  34. if plot_cpu_usage:
  35. host_system_usage['timestamps'] = np.array(host_system_usage['timestamps'])
  36. host_cpu_usage = {int(cpu): np.array(log_system_usage.calculate_cpu_usage_continuous(host_system_usage['stats']['cpus'][cpu])) for cpu in host_system_usage['stats']['cpus']}
  37. remote_system_usage['timestamps'] = np.array(remote_system_usage['timestamps'])
  38. remote_cpu_usage = {int(cpu): np.array(log_system_usage.calculate_cpu_usage_continuous(remote_system_usage['stats']['cpus'][cpu])) for cpu in remote_system_usage['stats']['cpus']}
  39. #
  40. plot_processes = ('processes' in remote_system_usage)
  41. plot_processes = False
  42. #
  43. if plot_processes:
  44. target_tor_proc_states = [[y['state'] for y in x if 'target/torrc' in y['args']] for x in remote_system_usage['processes']]
  45. host_tor_proc_states = [[y['state'] for y in x if '/torrc' in y['args'] and 'target/torrc' not in y['args']] for x in host_system_usage['processes']]
  46. host_all_proc_states = [[y['state'] for y in x] for x in host_system_usage['processes']]
  47. #print(remote_system_usage['processes'][700])
  48. any_host_tor_proc_states_D = ['D' in x for x in host_tor_proc_states]
  49. any_host_proc_states_D = ['D' in x for x in host_all_proc_states]
  50. #any_proc_states_D = ['D' in x for x in [[y['state'] for y in z] for z in remote_system_usage['processes']]]
  51. assert set([len(x) for x in target_tor_proc_states]) == {0,1}
  52. for x in target_tor_proc_states:
  53. if len(x) == 0:
  54. x.append(None)
  55. #
  56. #
  57. print(set([y['args'] for x in host_system_usage['processes'] for y in x if y['state'] == 'D']))
  58. #
  59. target_tor_proc_states = [x[0] for x in target_tor_proc_states]
  60. #
  61. #
  62. throughput_start_index = np.argmax(norm_throughput['timestamps'] > start_time)-5
  63. throughput_end_index = np.argmax(norm_throughput['timestamps'] > end_time)+5
  64. if plot_cpu_usage:
  65. host_cpu_start_index = np.argmax(host_system_usage['timestamps'] > start_time)-20
  66. host_cpu_end_index = np.argmax(host_system_usage['timestamps'] > end_time)+20
  67. remote_cpu_start_index = np.argmax(remote_system_usage['timestamps'] > start_time)-20
  68. remote_cpu_end_index = np.argmax(remote_system_usage['timestamps'] > end_time)+20
  69. #
  70. #start_time = 0
  71. #
  72. print('Processed Data: {}'.format(time.time()-initial_time))
  73. #
  74. fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True, figsize=(20,13))
  75. fig.suptitle('{}\n\n{}'.format(title, os.path.basename(sys.argv[2])))
  76. #
  77. ax1_colors = plt.get_cmap('tab20').colors[0:2]
  78. for x in range(2):
  79. ax1.step(norm_throughput['timestamps'][throughput_start_index:throughput_end_index]-start_time,
  80. 0.5*np.sum(norm_throughput['threads'][throughput_start_index:throughput_end_index,x::2],
  81. axis=1)/2**20, where='post', color=ax1_colors[x])
  82. #for x in range(int(norm_throughput['threads'].shape[1]/2)):
  83. # ax1.step(norm_throughput['timestamps'][throughput_start_index:throughput_end_index]-start_time,
  84. # 0.5*np.sum(norm_throughput['threads'][throughput_start_index:throughput_end_index,x*2:x*2+2],
  85. # axis=1)/2**20, where='post', color=ax1_colors[0])
  86. #
  87. ax1.step(norm_throughput['timestamps'][throughput_start_index:throughput_end_index]-start_time,
  88. 0.5*np.sum(norm_throughput['threads'][throughput_start_index:throughput_end_index,:],
  89. axis=1)/2**20, where='post', color='grey')
  90. ax1.set_ylabel('Throughput (MiB/s)', color=ax1_colors[0])
  91. #
  92. ax1_twin = ax1.twinx()
  93. ax1_twin_color = plt.get_cmap('tab20').colors[4]
  94. ax1_twin.plot(np.sort(last_byte_times)-start_time, np.arange(len(last_byte_times)), color=ax1_twin_color)
  95. #ax1_twin.set_ylim([0, None])
  96. ax1_twin.set_ylabel('Number of completed streams', color=ax1_twin_color)
  97. #
  98. print('Finished plotting ax1: {}'.format(time.time()-initial_time))
  99. #
  100. colormap = plt.get_cmap('tab20').colors #'tab10'
  101. assigned_colors = []
  102. #
  103. for transfer in server_data:
  104. color_selector = transfer['custom_data']['circuit'][1][-1]
  105. if color_selector in assigned_colors:
  106. color_index = assigned_colors.index(color_selector)
  107. else:
  108. color_index = len(assigned_colors)
  109. assigned_colors.append(color_selector)
  110. #
  111. #bins = np.arange(start_time, transfer['deltas']['timestamps'][-1], 0.1)
  112. #binned_indexes = np.digitize(transfer['deltas']['timestamps'], bins)
  113. #binned_deltas = np.zeros(bins.shape)
  114. #for x in range(len(binned_indexes)):
  115. # binned_deltas[binned_indexes[x]-1] += transfer['deltas']['bytes'][x]
  116. #
  117. #zeros = (binned_deltas == 0).nonzero()[0]
  118. #bins = np.delete(bins, zeros)
  119. #binned_deltas = np.delete(binned_deltas, zeros)
  120. #ax2.step(bins-start_time, np.cumsum(binned_deltas), color=colormap[color_index%len(colormap)], where='post')
  121. ax2.step(transfer['deltas']['timestamps']-start_time, np.cumsum(transfer['deltas']['bytes']),
  122. color=colormap[color_index%len(colormap)], where='post')
  123. #
  124. ax2.set_ylabel('Bytes')
  125. ax2_twin = ax2.twinx()
  126. ax2_twin.set_ylim([x/(2**20) for x in ax2.get_ylim()])
  127. ax2_twin.set_ylabel('MiB')
  128. #
  129. print('Finished plotting ax2: {}'.format(time.time()-initial_time))
  130. #
  131. if plot_cpu_usage:
  132. for cpu in remote_cpu_usage:
  133. ax3.step(remote_system_usage['timestamps'][remote_cpu_start_index:remote_cpu_end_index]-start_time,
  134. 100*remote_cpu_usage[cpu][remote_cpu_start_index:remote_cpu_end_index],
  135. label='CPU {}'.format(cpu))
  136. #
  137. if plot_processes:
  138. vals = list(set(target_tor_proc_states))
  139. vals.remove(None)
  140. vals = sorted(vals) + [None]
  141. print(vals)
  142. ax3.step(remote_system_usage['timestamps'][remote_cpu_start_index:remote_cpu_end_index]-start_time,
  143. [10*vals.index(x)+120 for x in target_tor_proc_states[remote_cpu_start_index:remote_cpu_end_index]])
  144. ax3.step(host_system_usage['timestamps'][host_cpu_start_index:host_cpu_end_index]-start_time,
  145. [int(x)*20+160 for x in any_host_proc_states_D[host_cpu_start_index:host_cpu_end_index]])
  146. #
  147. #
  148. import matplotlib
  149. ax3.grid()
  150. ax3.xaxis.set_major_formatter(matplotlib.ticker.FormatStrFormatter('%.3f'))
  151. ax3.set_xlabel('Time (s)')
  152. ax3.set_ylabel('Per-Core CPU Usage (%)')
  153. ax3.legend()
  154. #
  155. print('Finished plotting ax3: {}'.format(time.time()-initial_time))
  156. #
  157. fig.tight_layout()
  158. plt.subplots_adjust(top=0.92)
  159. output_filename = title.lower().replace(' ','-').replace('/','').replace('(','').replace(')','').replace(',','')
  160. #plt.savefig(os.path.join('/tmp', output_filename))
  161. plt.show(fig)
  162. #