update space

app.py

@@ -36,7 +36,7 @@ from src.submission.submit import add_new_eval
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-## Space initialisation
+# Space initialisation
 try:
     print(EVAL_REQUESTS_PATH)
     snapshot_download(
@@ -54,6 +54,7 @@ except Exception:
 
 
 LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+# print("Before calling init_leaderboard:", LEADERBOARD_DF)
 
 (
     finished_eval_queue_df,
@@ -61,6 +62,7 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
+
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
@@ -74,20 +76,20 @@ def init_leaderboard(dataframe):
         ),
         search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
+        # filter_columns=[
+        #     ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+        #     ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+        #     ColumnFilter(
+        #         AutoEvalColumn.params.name,
+        #         type="slider",
+        #         min=0.01,
+        #         max=150,
+        #         label="Select the number of parameters (B)",
+        #     ),
+        #     ColumnFilter(
+        #         AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+        #     ),
+        # ],
         bool_checkboxgroup_label="Hide models",
         interactive=False,
     )
@@ -97,7 +99,7 @@ demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-    gr.Markdown(INTRODUCTION_TEXT_ZH, elem_classes="markdown-text")
+    # gr.Markdown(INTRODUCTION_TEXT_ZH, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
@@ -106,16 +108,16 @@ with demo:
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             with gr.TabItem("EN", elem_id="llm-benchmark-tab-table", id=1):
                 gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-            with gr.TabItem("ZH", elem_id="llm-benchmark-tab-table", id=2):
-                gr.Markdown(LLM_BENCHMARKS_TEXT_ZH, elem_classes="markdown-text")
+            # with gr.TabItem("ZH", elem_id="llm-benchmark-tab-table", id=2):
+            #     gr.Markdown(LLM_BENCHMARKS_TEXT_ZH, elem_classes="markdown-text")
 
         with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
             with gr.Column():
                 with gr.Row():
                     with gr.TabItem("EN", elem_id="llm-benchmark-tab-table", id=1):
                         gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-                    with gr.TabItem("ZH", elem_id="llm-benchmark-tab-table", id=2):
-                        gr.Markdown(EVALUATION_QUEUE_TEXT_ZH, elem_classes="markdown-text")
+                    # with gr.TabItem("ZH", elem_id="llm-benchmark-tab-table", id=2):
+                    #     gr.Markdown(EVALUATION_QUEUE_TEXT_ZH, elem_classes="markdown-text")
 
                 with gr.Column():
                     with gr.Accordion(
@@ -221,4 +223,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch(share=True)

src/about.py

@@ -13,8 +13,8 @@ class Task:
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
     task0 = Task("mmmlu", "acc", "MMMLU")
-    task1 = Task("mmlu", "acc", "MMLU")
-    task2 = Task("cmmlu", "acc", "CMMLU")
+    # task1 = Task("mmlu", "acc", "MMLU")
+    # task2 = Task("cmmlu", "acc", "CMMLU")
     task3 = Task("mmmlu_ar", "acc", "MMMLU_AR")
     task4 = Task("mmmlu_bn", "acc", "MMMLU_BN")
     task5 = Task("mmmlu_de", "acc", "MMMLU_DE")

src/leaderboard/read_evals.py

@@ -96,7 +96,7 @@ class EvalResult:
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
         request_file = get_request_file_for_model(requests_path, self.full_model.split("/")[-1], self.precision.value.name)
-        # print("########",request_file)
+        # print("########",requests_path,self.full_model.split("/")[-1])
 
         try:
             with open(request_file, "r") as f:
@@ -112,9 +112,10 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        keys_to_average = ['mmmlu', 'mmlu', 'cmmlu']
-        average = sum([self.results[key] for key in keys_to_average if self.results.get(key) is not None]) / len(
-            keys_to_average)
+        # keys_to_average = ['mmmlu', 'mmlu', 'cmmlu']
+        # average = sum([self.results[key] for key in keys_to_average if self.results.get(key) is not None]) / len(
+        #     keys_to_average)
+        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -182,6 +183,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        print(results_path)
         eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
@@ -198,5 +200,4 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             results.append(v)
         except KeyError:  # not all eval values present
             continue
-
     return results