bugfix: input and output should support List[str], not just a single str

code_eval_stdio.py

@@ -26,7 +26,7 @@ import numpy as np
 
 import evaluate
 
-from .execute import check_correctness
+.from execute import check_correctness
 
 
 _CITATION = """\
@@ -145,8 +145,8 @@ class CodeEval(evaluate.Metric):
                     "predictions": datasets.Sequence(datasets.Value("string")),
                     "references": datasets.Features(
                         {
-                            "input": datasets.Value("string"),
-                            "reference_output": datasets.Value("string"),
+                            "inputs": datasets.Sequence(datasets.Value("string")),
+                            "reference_outputs": datasets.Sequence(datasets.Value("string")),
                         }
                     ),
                 }
@@ -178,7 +178,7 @@ class CodeEval(evaluate.Metric):
 
             for task_id, (candidates, reference) in enumerate(zip(predictions, references)):
                 for candidate in candidates:
-                    args = (candidate, reference['input'], reference['reference_output'], timeout, task_id, completion_id[task_id])
+                    args = (candidate, reference['inputs'], reference['reference_outputs'], timeout, task_id, completion_id[task_id])
                     future = executor.submit(check_correctness, *args)
                     futures.append(future)
                     completion_id[task_id] += 1

execute.py

@@ -25,49 +25,47 @@ import signal
 import tempfile
 
 
-def check_correctness(program, test_input, test_output, timeout, task_id, completion_id):
+def check_correctness(program, test_inputs, test_outputs, timeout, task_id, completion_id):
     """
     Evaluates the functional correctness of a completion by running the test
     suite provided in the problem.
-
     :param program: The program string to evaluate.
-    :param test_input: The input string to provide to the program via STDIN.
-    :param test_output: The expected output string from the program via STDOUT.
+    :param test_inputs: A list of input strings to provide to the program via STDIN.
+    :param test_outputs: A list of expected output strings from the program via STDOUT.
     :param timeout: Maximum execution time in seconds.
     :param task_id: ID of the task being evaluated.
     :param completion_id: Completion ID to match results later.
     """
     manager = multiprocessing.Manager()
-    result = manager.list()
+    results = manager.list()
 
-    process = multiprocessing.Process(target=unsafe_execute, args=(program, test_input, test_output, result, timeout))
+    process = multiprocessing.Process(target=unsafe_execute, args=(program, test_inputs, test_outputs, results, timeout))
     process.start()
     process.join(timeout=timeout + 1)
 
     if process.is_alive():
         process.kill()
-        result.append("timed out")
+        results.append("timed out")
 
     return dict(
         task_id=task_id,
-        passed=result[0] == "passed",
-        result=result[0],
+        passed=all(result.get("status") == "passed" for result in results),
+        results=list(results),
         completion_id=completion_id,
+        program=program,
     )
 
 
-def unsafe_execute(program, test_input, test_output, result, timeout):
+def unsafe_execute(program, test_inputs, test_outputs, results, timeout):
     """
     Executes the program with redirected STDIN and compares its STDOUT to the expected output.
-
     :param program: The program string to execute.
-    :param test_input: Input to provide to the program via STDIN.
-    :param test_output: Expected output to compare with STDOUT.
-    :param result: A multiprocessing.Manager().list() to store the execution result.
+    :param test_inputs: List of inputs to provide to the program via STDIN.
+    :param test_outputs: List of expected outputs to compare with STDOUT.
+    :param results: A multiprocessing.Manager().list() to store the execution results.
     :param timeout: Maximum execution time in seconds.
     """
     with create_tempdir():
-
         # These system calls are needed when cleaning up tempdir.
         import os
         import shutil
@@ -79,36 +77,37 @@ def unsafe_execute(program, test_input, test_output, result, timeout):
         # Disable functionalities that can make destructive changes to the test.
         reliability_guard()
 
-        # Run program.
+        # Run the program for each input-output pair.
         try:
             exec_globals = {}
-            actual_output = None
-
-            # Redirect I/O and execute
-            input_stream = io.StringIO(test_input)
-            output_stream = io.StringIO()
-
-            with swallow_io(input_stream, output_stream):
-                with time_limit(timeout):
-                    exec(program, exec_globals)
-
-            actual_output = output_stream.getvalue().strip()
-            expected_output = test_output.strip()
-
-            if actual_output == expected_output:
-                result.append("passed")
-            else:
-                result.append(f"failed: got '{actual_output}', expected '{expected_output}'")
-
-        except TimeoutException:
-            result.append("timed out")
-        except BaseException as e:
-            result.append(f"failed: {e}")
 
-        # Needed for cleaning up.
-        shutil.rmtree = rmtree
-        os.rmdir = rmdir
-        os.chdir = chdir
+            for test_input, test_output in zip(test_inputs, test_outputs):
+                input_stream = io.StringIO(test_input)
+                output_stream = io.StringIO()
+
+                try:
+                    with swallow_io(input_stream, output_stream):
+                        with time_limit(timeout):
+                            exec(program, exec_globals)
+
+                    actual_output = output_stream.getvalue().strip()
+                    expected_output = test_output.strip()
+
+                    if actual_output == expected_output:
+                        results.append({"status": "passed", "input": test_input, "expected": test_output, "actual": actual_output})
+                    else:
+                        results.append({"status": "failed", "input": test_input, "expected": test_output, "actual": actual_output})
+
+                except TimeoutException:
+                    results.append({"status": "timed out", "input": test_input})
+                except BaseException as e:
+                    results.append({"status": "failed", "input": test_input, "error": str(e)})
+
+        finally:
+            # Restore system calls for cleaning up
+            shutil.rmtree = rmtree
+            os.rmdir = rmdir
+            os.chdir = chdir
 
 
 @contextlib.contextmanager