merge inputs into references

code_eval_stdio.py

@@ -145,8 +145,8 @@ class CodeEval(evaluate.Metric):
                     "predictions": datasets.Sequence(datasets.Value("string")),
                     "references": datasets.Sequence(
                         {
-                            "inputs": datasets.Value("string"),
-                            "expected_output": datasets.Value("string"),
+                            "input": datasets.Value("string"),
+                            "reference_output": datasets.Value("string"),
                         }
                     ),
                 }
@@ -161,7 +161,7 @@ class CodeEval(evaluate.Metric):
         """
         Returns the scores
         predictions: List[List[str]] the python program
-        references: List[Tuple[str, str]] test inputs and expected outputs
+        references: List[Dict[str, str]] test inputs and reference outputs
         """
 
         if os.getenv("HF_ALLOW_CODE_EVAL", 0) != "1":
@@ -176,9 +176,9 @@ class CodeEval(evaluate.Metric):
             n_samples = 0
             results = defaultdict(list)
 
-            for task_id, (candidates, (input_data, expected_output)) in enumerate(zip(predictions, references)):
+            for task_id, (candidates, reference) in enumerate(zip(predictions, references)):
                 for candidate in candidates:
-                    args = (candidate, input_data, expected_output, timeout, task_id, completion_id[task_id])
+                    args = (candidate, reference['input'], reference['reference_output'], timeout, task_id, completion_id[task_id])
                     future = executor.submit(check_correctness, *args)
                     futures.append(future)
                     completion_id[task_id] += 1