Browse Source

Log parser

Ian Goldberg 11 months ago
parent
commit
bd2912052c
1 changed files with 35 additions and 0 deletions
  1. 35 0
      docker/parse_logs

+ 35 - 0
docker/parse_logs

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+import numpy as np
+import re
+import sys
+
+# Print the mean and stddev of the floats in the list of data
+def printstats(data):
+    mean = np.mean(data)
+    stddev = np.std(data)
+    print(f",{mean:.3f},{stddev:.3f}", end='')
+
+if __name__ == "__main__":
+    times = {}
+    for line in sys.stdin.readlines():
+        if matches := \
+            re.match(r'(\d+) (\d+) \d+ (\d+\.?\d*) (\d+\.?\d*) (\d+\.?\d*)',
+                line):
+            groups = matches.groups()
+            label, send_time, fetch_time, tot_time = \
+                (int(groups[0]), int(groups[1])), float(groups[2]), \
+                float(groups[3]), float(groups[4])
+            if label not in times:
+                times[label] = [ [], [], [] ]
+            times[label][0].append(send_time)
+            times[label][1].append(fetch_time)
+            times[label][2].append(tot_time)
+
+    print("users,batches,send_mean,send_stddev,fetch_mean,fetch_stddev,tot_mean,tot_stddev")
+    for k in sorted(times.keys()):
+        print(k[0], k[1], sep=',', end='')
+        for data in times[k]:
+            printstats(data)
+        print('')
+