Adding the possibility for a program to fail

docs/userconfig.rst

@@ -152,6 +152,11 @@ verify [boolean]
     True if the extraction program compares the benchmark and test
     outputs directly.  See :ref:`verification` for more details.  Default:
     False.
+can_fail [boolean]
+    If True, the exit code of the executable is not checked. Otherwise,
+    the test will fail if the error code is not zero. Useful to be set
+    if you want to check an expected failure, e.g. to see that the code
+    crashes if the provided input is invalid. Default: False.
 vcs [string]
     Version control system used for the source code.  This is used to
     label the benchmarks.  The program binary is assumed to be in the same
@@ -172,7 +177,7 @@ for all tests of this type:
 * min_nprocs (default: 0)
 * max_nprocs (default: 2^31-1 or 2^63-1)
 * output (no default)
-* run_concurrent (defailt: false)
+* run_concurrent (default: false)
 * submit_template
 
 See :ref:`jobconfig` for more details.

lib/testcode2/__init__.py

@@ -82,6 +82,9 @@ def __init__(self, name, exe, test_id, benchmark, **kwargs):
         self.skip_args = ''
         self.verify = False
         self.extract_fn = None
+        # By default, the job is expected to exit with error code 0.
+        # Setting it to True will discard the exit status/error code.
+        self.can_fail = False
 
         # Info
         self.vcs = None
@@ -312,10 +315,11 @@ def run_test(self, verbose=1, cluster_queue=None, rundir=None):
                             err.append(sys.exc_info()[1])
                     status = validation.Status()
                     if job.returncode != 0:
-                        err.insert(0, 'Error running job.  Return code: %i'
-                                        % job.returncode)
-                        (status, msg) = self.skip_job(test_input, test_arg,
-                                                      verbose)
+                        if not self.test_program.can_fail:
+                            err.insert(0, 'Error running job.  Return code: %i'
+                                            % job.returncode)
+                            (status, msg) = self.skip_job(test_input, test_arg,
+                                                          verbose)
                     if status.skipped():
                         self._update_status(status, (test_input, test_arg))
                         if verbose > 0 and verbose < 3:

lib/testcode2/config.py

@@ -105,7 +105,8 @@ def parse_userconfig(config_file, executables=None, test_id=None,
     test_program_options = ('run_cmd_template',
         'launch_parallel', 'ignore_fields', 'data_tag', 'extract_cmd_template',
         'extract_fn', 'extract_program', 'extract_args', 'extract_fmt',
-        'verify', 'vcs', 'skip_program', 'skip_args', 'skip_cmd_template')
+        'verify', 'vcs', 'skip_program', 'skip_args', 'skip_cmd_template',
+        'can_fail')
     default_test_options = ('inputs_args', 'output', 'nprocs',
         'min_nprocs', 'max_nprocs', 'submit_template',)
     test_programs = {}
@@ -171,6 +172,10 @@ def parse_userconfig(config_file, executables=None, test_id=None,
         if 'vcs' in tp_dict:
             tp_dict['vcs'] = vcs.VCSRepository(tp_dict['vcs'],
                     os.path.dirname(exe))
+        if 'can_fail' in tp_dict:
+            tp_dict['can_fail'] = \
+                    userconfig.getboolean(section, 'can_fail')
+
         program = testcode2.TestProgram(section, exe, test_id,
             user_options['benchmark'], **tp_dict)
         test_programs[section] = program