Browse Source

[Scripts/regression] Rename 'flaky' to 'ignore_failure'

There are different reasons why a test failure should be ignored, so
rename the option to avoid confusion if the test isn't flaky.
Simon Gaiser 5 years ago
parent
commit
9d1b70bd57
2 changed files with 8 additions and 8 deletions
  1. 2 2
      Pal/regression/02_Memory.py
  2. 6 6
      Scripts/regression.py

+ 2 - 2
Pal/regression/02_Memory.py

@@ -21,11 +21,11 @@ regression.add_check(name="Memory Allocation with Address",
 # enclave init. Therefore the memory protection and deallocation tests will
 # fail. By utilizing SGX2 it's possibile to fix this.
 
-regression.add_check(name="Memory Protection", flaky = sgx,
+regression.add_check(name="Memory Protection", ignore_failure = sgx,
     check=lambda res: "Memory Allocation Protection (RW) OK" in res[0].log and
                       "Memory Protection (R) OK" in res[0].log)
 
-regression.add_check(name="Memory Deallocation", flaky = sgx,
+regression.add_check(name="Memory Deallocation", ignore_failure = sgx,
     check=lambda res: "Memory Deallocation OK" in res[0].log)
 
 def check_quota(res):

+ 6 - 6
Scripts/regression.py

@@ -21,17 +21,17 @@ class Regression:
             self.timeout = timeout
         self.keep_log = (os.getenv('KEEP_LOG', '0') == '1')
 
-    def add_check(self, name, check, times = 1, flaky=0, args = []):
+    def add_check(self, name, check, times = 1, ignore_failure=0, args = []):
         combined_args = ' '.join(args)
         if not combined_args in self.runs:
             self.runs[combined_args] = []
-        self.runs[combined_args].append((name, check, flaky, times))
+        self.runs[combined_args].append((name, check, ignore_failure, times))
 
     def run_checks(self):
         something_failed = 0
         for combined_args in self.runs:
             needed_times = 1
-            for (name, check, flaky, times) in self.runs[combined_args]:
+            for (name, check, ignore_failure, times) in self.runs[combined_args]:
                 if needed_times < times:
                     needed_times = times
 
@@ -78,7 +78,7 @@ class Regression:
 
                 run_times = run_times + 1
                 keep_log = False
-                for (name, check, flaky, times) in self.runs[combined_args]:
+                for (name, check, ignore_failure, times) in self.runs[combined_args]:
                     if run_times == times:
                         result = check(outputs)
                         if result:
@@ -87,8 +87,8 @@ class Regression:
                             print '\033[93m[Fail   ]\033[0m', name
                             if timed_out : print 'Test timed out!'
                             keep_log = True
-                            if flaky:
-                                print '   This test is known not to work, but should be fixed'
+                            if ignore_failure:
+                                print '   Ignoring failure.'
                             else:
                                 something_failed = 1