Update README.md

README.md

@@ -43,7 +43,7 @@ The Code Eval metric calculates how good are predictions given a set of referenc
 
 `predictions`: a list of candidates to evaluate. Each candidate should be a list of strings with several code candidates to solve the problem.
 
-`references`: a list of tuple of [str, str], corresponding of input and expected output for each prediction.
+`references`: a list of Dict of [str, str], each dict has two keys "input" and "reference_output". e.g. [{"input": "1 2", "reference_output": "3"}]
 
 `k`: number of code candidates to consider in the evaluation. The default value is `[1, 10, 100]`.
 
@@ -54,7 +54,7 @@ The Code Eval metric calculates how good are predictions given a set of referenc
 ```python
 from evaluate import load
 code_eval_stdio = load("hage2000/code_eval_stdio")
-references = [("2 3", "5")]
+references = [{"input":"2 3", "reference_output":"5})]
 candidates = [[ "nums = list(map(int, input().split()))\nprint(sum(nums))"]]
 pass_at_k, results = code_eval_stdio.compute(references=references, predictions=candidates, k=[1, 2])
 ```
@@ -86,7 +86,7 @@ Full match at `k=1`:
 ```python
 from evaluate import load
 code_eval_stdio = load("hage2000/code_eval_stdio")
-references = [("2 3", "5")]
+references = [{"input":"2 3", "reference_output":"5"}]
 candidates = [[ "nums = list(map(int, input().split()))\nprint(sum(nums))"]]
 pass_at_k, results = code_eval_stdio.compute(references=references, predictions=candidates, k=[1, 2])
 print(pass_at_k)
@@ -98,7 +98,7 @@ No match for k = 1:
 ```python
 from evaluate import load
 code_eval_stdio = load("hage2000/code_eval_stdio")
-references = [("2 3", "5")]
+references = [{"input":"2 3", "reference_output":"5"}]
 candidates = [[ "nums = list(map(int, input().split()))\nprint(nums[0]*nums[1])"]]
 pass_at_k, results = code_eval_stdio.compute(references=references, predictions=candidates, k=[1, 2])
 print(pass_at_k)
@@ -110,7 +110,7 @@ Partial match at k=1, full match at k=2:
 ```python
 from evaluate import load
 code_eval_stdio = load("hage2000/code_eval_stdio")
-references = [("2 3", "5")]
+references = [{"input":, "reference_output":"5"}]
 candidates = [[ "nums = list(map(int, input().split()))\nprint(sum(nums))", "nums = list(map(int, input().split()))\nprint(nums[0]*nums[1])"]]
 pass_at_k, results = code_eval_stdio.compute(references=references, predictions=candidates, k=[1, 2])
 print(pass_at_k)