regression.py 3.5 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192
  1. import sys, os, subprocess, re, time, signal
  2. class Result:
  3. def __init__(self, out, log, code):
  4. self.out = out.split('\n')
  5. self.log = log.split('\n')
  6. self.code = code
  7. class Regression:
  8. def __init__(self, loader = None, executable = '', prepare = None, timeout = 0):
  9. self.loader = loader
  10. self.executable = executable
  11. self.prepare = prepare
  12. self.runs = dict()
  13. default_timeout = int(os.getenv('TIMEOUT', '10000'))
  14. if default_timeout > timeout:
  15. self.timeout = default_timeout
  16. else:
  17. self.timeout = timeout
  18. self.keep_log = (os.getenv('KEEP_LOG', '0') == '1')
  19. def add_check(self, name, check, times = 1, ignore_failure=0, args = []):
  20. combined_args = ' '.join(args)
  21. if not combined_args in self.runs:
  22. self.runs[combined_args] = []
  23. self.runs[combined_args].append((name, check, ignore_failure, times))
  24. def run_checks(self):
  25. something_failed = 0
  26. for combined_args in self.runs:
  27. needed_times = 1
  28. for (name, check, ignore_failure, times) in self.runs[combined_args]:
  29. if needed_times < times:
  30. needed_times = times
  31. run_times = 0
  32. outputs = []
  33. timed_out = False
  34. while run_times < needed_times:
  35. args = []
  36. if self.loader:
  37. args.append(self.loader)
  38. if self.executable:
  39. args.append(self.executable)
  40. if combined_args:
  41. args += combined_args.split(' ')
  42. if self.prepare:
  43. self.prepare(args)
  44. p = subprocess.Popen(args,
  45. stdout=subprocess.PIPE,
  46. stderr=subprocess.PIPE,
  47. preexec_fn=os.setpgrp)
  48. try:
  49. out, log = p.communicate(timeout=self.timeout * 0.001)
  50. except subprocess.TimeoutExpired:
  51. timed_out = True
  52. os.killpg(p.pid, signal.SIGKILL)
  53. out, log = p.communicate()
  54. out = out.decode('utf-8')
  55. log = log.decode('utf-8')
  56. outputs.append(Result(out, log, p.returncode))
  57. run_times = run_times + 1
  58. keep_log = False
  59. for (name, check, ignore_failure, times) in self.runs[combined_args]:
  60. if run_times == times:
  61. result = check(outputs)
  62. if not timed_out and result:
  63. print('\033[92m[Success ]\033[0m', name)
  64. else:
  65. if ignore_failure:
  66. print('[Fail (Ignored)]', name)
  67. else:
  68. print('\033[93m[Fail ]\033[0m', name)
  69. something_failed = 1
  70. if timed_out : print('Test timed out!')
  71. keep_log = True
  72. if self.keep_log and keep_log:
  73. sargs = [re.sub(r"\W", '_', a).strip('_') for a in args]
  74. filename = 'log-' + '_'.join(sargs) + '_' + time.strftime("%Y%m%d_%H%M%S")
  75. with open(filename, 'w') as f:
  76. f.write(log + out)
  77. print('keep log to %s' % (filename))
  78. if something_failed:
  79. return -1
  80. else:
  81. return 0