regression.py 3.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104
  1. #!/usr/bin/python
  2. import sys, os, subprocess, re, time
  3. class Result:
  4. def __init__(self, out, log, code):
  5. self.out = out.split('\n')
  6. self.log = log.split('\n')
  7. self.code = code
  8. class Regression:
  9. def __init__(self, loader = None, executable = '', prepare = None, timeout = 0):
  10. self.loader = loader
  11. self.executable = executable
  12. self.prepare = prepare
  13. self.runs = dict()
  14. default_timeout = int(os.getenv('TIMEOUT', '10000'))
  15. if default_timeout > timeout:
  16. self.timeout = default_timeout
  17. else:
  18. self.timeout = timeout
  19. self.keep_log = (os.getenv('KEEP_LOG', '0') == '1')
  20. def add_check(self, name, check, times = 1, flaky=0, args = []):
  21. combined_args = ' '.join(args)
  22. if not combined_args in self.runs:
  23. self.runs[combined_args] = []
  24. self.runs[combined_args].append((name, check, flaky, times))
  25. def run_checks(self):
  26. something_failed = 0
  27. for combined_args in self.runs:
  28. needed_times = 1
  29. for (name, check, flaky, times) in self.runs[combined_args]:
  30. if needed_times < times:
  31. needed_times = times
  32. run_times = 0
  33. outputs = []
  34. timed_out = False
  35. while run_times < needed_times:
  36. args = []
  37. if self.loader:
  38. args.append(self.loader)
  39. if self.executable:
  40. args.append(self.executable)
  41. if combined_args:
  42. args += combined_args.split(' ')
  43. if self.prepare:
  44. self.prepare(args)
  45. time.sleep(0.1)
  46. p = subprocess.Popen(args,
  47. stdout=subprocess.PIPE,
  48. stderr=subprocess.PIPE)
  49. sleep_time = 0
  50. finish = False
  51. while sleep_time < self.timeout:
  52. time.sleep(0.001)
  53. if p.poll() is not None:
  54. finish = True
  55. break
  56. sleep_time += 1
  57. if not finish and p.poll() is None:
  58. timed_out = True
  59. p.kill()
  60. time.sleep(0.1)
  61. out = p.stdout.read()
  62. log = p.stderr.read()
  63. outputs.append(Result(out, log, p.returncode))
  64. run_times = run_times + 1
  65. keep_log = False
  66. for (name, check, flaky, times) in self.runs[combined_args]:
  67. if run_times == times:
  68. result = check(outputs)
  69. if result:
  70. print '\033[92m[Success]\033[0m', name
  71. else:
  72. print '\033[93m[Fail ]\033[0m', name
  73. if timed_out : print 'Test timed out!'
  74. keep_log = True
  75. if flaky:
  76. print ' This test is known not to work, but should be fixed'
  77. else:
  78. something_failed = 1
  79. if self.keep_log and keep_log:
  80. sargs = [re.sub(r"\W", '_', a).strip('_') for a in args]
  81. filename = 'log-' + '_'.join(sargs) + '_' + time.strftime("%Y%m%d_%H%M%S")
  82. with open(filename, 'w') as f:
  83. f.write(log + out)
  84. print 'keep log to %s' % (filename)
  85. if something_failed:
  86. return -1
  87. else:
  88. return 0