add meralion2

app.py

@@ -19,6 +19,8 @@ pages = {
     'ASR-English'        : asr_english,
     'ASR-Mandarin'       : asr_mandarin,
     'ASR-Singlish'       : asr_singlish,
+    'ASR-SEA'            : asr_sea,
+    'ASR-Private'        : asr_private,
     'Speech Translation' : speech_translation,
     'SQA-English'        : speech_question_answering_english,
     'SQA-Singlish'       : speech_question_answering_singlish,
@@ -47,6 +49,8 @@ menu_items = [
                      sac.MenuItem(label='ASR-English', icon='mic'),
                      sac.MenuItem(label='ASR-Mandarin', icon='mic'),
                      sac.MenuItem(label='ASR-Singlish', icon='mic'),
+                     sac.MenuItem(label='ASR-SEA', icon='mic'),
+                     sac.MenuItem(label='ASR-Private', icon='mic'),
                  ]
                  ),
 

app/content.py

@@ -1,5 +1,157 @@
+asr_english_datasets = [
+    'LibriSpeech-Clean', 
+    'LibriSpeech-Other', 
+    'CommonVoice-15-EN', 
+    'Peoples-Speech', 
+    'GigaSpeech-1', 
+    'Earnings-21', 
+    'Earnings-22', 
+    'TED-LIUM-3', 
+    'TED-LIUM-3-LongForm', 
+]
 
-displayname2datasetname = {
+
+asr_singlish_datasets = [
+    'MNSC-PART1-ASR', 
+    'MNSC-PART2-ASR',
+    'MNSC-PART3-ASR',
+    'MNSC-PART4-ASR',
+    'MNSC-PART5-ASR',
+    'MNSC-PART6-ASR',
+]
+
+
+asr_mandarin_datasets = [
+    'AISHELL-ASR-ZH', 
+    'CommonVoice-ZH'
+]
+
+
+asr_sea_datasets = [
+    'CommonVoice-17-Indonesian', 
+    'CommonVoice-17-Tamil',
+    # 'CommonVoice-17-Thai', 
+    'CommonVoice-17-Vietnamese',
+    'GigaSpeech-2-Indonesain',
+    'GigaSpeech-2-Thai',
+    'GigaSpeech-2-Vietnamese',
+    'Fleurs-Tamil',
+    'Lotus-Thai'
+]
+
+
+asr_private_datasets =  [
+    'CNA',
+    'IDPC',
+    'Parliament',
+    'UKUS-News',
+    'Mediacorp',
+    'IDPC-Short',
+    'Parliament-Short',
+    'UKUS-News-Short',
+    'Mediacorp-Short',
+    'YouTube ASR: English Singapore Content',
+    'YouTube ASR: English with Strong Emotion',
+    'YouTube ASR: Malay with English Prompt',
+    'YouTube ASR: Chinese with English Prompt',
+    'YouTube ASR: Tamil with English Prompt'
+]
+
+
+speech_translation_datasets = [
+    'CoVoST2-EN-ID', 
+    'CoVoST2-EN-ZH',
+    'CoVoST2-EN-TA', 
+    'CoVoST2-ID-EN', 
+    'CoVoST2-ZH-EN', 
+    'CoVoST2-TA-EN'
+]
+
+
+speech_qa_english_datasets = [
+    'CN-College-Listen-MCQ',
+    'DREAM-TTS-MCQ',
+    'SLUE-P2-SQA5', 
+    'Public-SG-Speech-QA', 
+    'Spoken-SQuAD',
+    'MMAU-mini'
+]
+
+
+speech_qa_singlish_datasets = [
+    'MNSC-PART3-SQA', 
+    'MNSC-PART4-SQA',
+    'MNSC-PART5-SQA',
+    'MNSC-PART6-SQA',
+]
+
+
+sds_datasets = [
+    'MNSC-PART3-SDS', 
+    'MNSC-PART4-SDS',
+    'MNSC-PART5-SDS',
+    'MNSC-PART6-SDS',
+]
+
+
+si_datasets = [
+    'OpenHermes-Audio', 
+    'ALPACA-Audio',
+]
+
+
+ac_datasets = [
+    'WavCaps', 
+    'AudioCaps',
+]
+
+
+asqa_datasets = [
+    'Clotho-AQA', 
+    'WavCaps-QA', 
+    'AudioCaps-QA'
+]
+
+
+er_datasets = [
+    'IEMOCAP-Emotion', 
+    'MELD-Sentiment', 
+    'MELD-Emotion',
+]
+
+
+ar_datasets = [
+    'VoxCeleb-Accent',
+    'MNSC-AR-Sentence',
+    'MNSC-AR-Dialogue',
+]
+
+
+gr_datasets = [
+    'VoxCeleb-Gender', 
+    'IEMOCAP-Gender'
+]
+
+
+music_datasets = ['MuChoMusic']
+
+
+wer_development_datasets = [
+    'YouTube ASR: Malay with Malay Prompt',
+    'YouTube ASR: Chinese with Chinese Prompt',
+    'SEAME-Dev-Mandarin',
+    'SEAME-Dev-Singlish',
+]
+
+
+non_wer_development_datasets = [
+    'YouTube SQA: English with Singapore Content',
+    'YouTube SDS: English with Singapore Content',
+    'YouTube PQA: English with Singapore Content',
+]
+
+
+wer_displayname2datasetname = {
     'LibriSpeech-Clean'    : 'librispeech_test_clean',
     'LibriSpeech-Other'    : 'librispeech_test_other',
     'CommonVoice-15-EN'    : 'common_voice_15_en_test',
@@ -9,65 +161,102 @@ displayname2datasetname = {
     'Earnings-22'          : 'earnings22_test',
     'TED-LIUM-3'           : 'tedlium3_test',
     'TED-LIUM-3-LongForm'  : 'tedlium3_long_form_test',
+
+    'MNSC-PART1-ASR'       : 'imda_part1_asr_test',
+    'MNSC-PART2-ASR'       : 'imda_part2_asr_test',
+    'MNSC-PART3-ASR'       : 'imda_part3_30s_asr_test',
+    'MNSC-PART4-ASR'       : 'imda_part4_30s_asr_test',
+    'MNSC-PART5-ASR'       : 'imda_part5_30s_asr_test',
+    'MNSC-PART6-ASR'       : 'imda_part6_30s_asr_test',
+
     'AISHELL-ASR-ZH'       : 'aishell_asr_zh_test',
+    'CommonVoice-ZH'       : 'commonvoice_zh_asr',
+
+    'CommonVoice-17-Indonesian'    : 'commonvoice_17_id_asr',
+    'CommonVoice-17-Tamil'         : 'commonvoice_17_ta_asr',
+    'CommonVoice-17-Thai'          : 'commonvoice_17_th_asr',
+    'CommonVoice-17-Vietnamese'    : 'commonvoice_17_vi_asr',
+    'GigaSpeech-2-Indonesain'      : 'gigaspeech2_id_test',
+    'GigaSpeech-2-Thai'            : 'gigaspeech2_th_test',
+    'GigaSpeech-2-Vietnamese'      : 'gigaspeech2_vi_test',
+    'Fleurs-Tamil'                 : 'fleurs_tamil_ta_30_asr',
+    'Lotus-Thai'                   : 'lotus_thai_th_30_asr',
+
+    'CNA'             : 'cna_test',
+    'IDPC'            : 'idpc_test',
+    'Parliament'      : 'parliament_test',
+    'UKUS-News'       : 'ukusnews_test',
+    'Mediacorp'       : 'mediacorp_test',
+    'IDPC-Short'      : 'idpc_short_test',
+    'Parliament-Short': 'parliament_short_test',
+    'UKUS-News-Short' : 'ukusnews_short_test',
+    'Mediacorp-Short' : 'mediacorp_short_test',
+    
+    'YouTube ASR: English Singapore Content': 'ytb_asr_batch1',
+    'YouTube ASR: English with Strong Emotion': 'ytb_asr_batch2',
+    'YouTube ASR: Malay with English Prompt': 'ytb_asr_batch3_malay',
+    'YouTube ASR: Chinese with English Prompt': 'ytb_asr_batch3_chinese',
+    'YouTube ASR: Tamil with English Prompt': 'ytb_asr_batch3_tamil',
+
+    'YouTube ASR: Malay with Malay Prompt': 'ytb_asr_batch3_ms_ms_prompt',
+    'YouTube ASR: Chinese with Chinese Prompt': 'ytb_asr_batch3_zh_zh_prompt',
+    
+    'SEAME-Dev-Mandarin'   : 'seame_dev_man',
+    'SEAME-Dev-Singlish'   : 'seame_dev_sge',
+}
+
+
+non_wer_displayname2datasetname = {
     'CoVoST2-EN-ID'        : 'covost2_en_id_test',
     'CoVoST2-EN-ZH'        : 'covost2_en_zh_test',
     'CoVoST2-EN-TA'        : 'covost2_en_ta_test',
     'CoVoST2-ID-EN'        : 'covost2_id_en_test',
     'CoVoST2-ZH-EN'        : 'covost2_zh_en_test',
     'CoVoST2-TA-EN'        : 'covost2_ta_en_test',
+
     'CN-College-Listen-MCQ': 'cn_college_listen_mcq_test',
     'DREAM-TTS-MCQ'        : 'dream_tts_mcq_test',
     'SLUE-P2-SQA5'         : 'slue_p2_sqa5_test',
     'Public-SG-Speech-QA'  : 'public_sg_speech_qa_test',
     'Spoken-SQuAD'         : 'spoken_squad_test',
+    'MMAU-mini'            : 'mmau_mini',
+
+    'MNSC-PART3-SQA'       : 'imda_part3_30s_sqa_human_test',
+    'MNSC-PART4-SQA'       : 'imda_part4_30s_sqa_human_test',
+    'MNSC-PART5-SQA'       : 'imda_part5_30s_sqa_human_test',
+    'MNSC-PART6-SQA'       : 'imda_part6_30s_sqa_human_test',
+
+    'MNSC-PART3-SDS'       : 'imda_part3_30s_ds_human_test',
+    'MNSC-PART4-SDS'       : 'imda_part4_30s_ds_human_test',
+    'MNSC-PART5-SDS'       : 'imda_part5_30s_ds_human_test',
+    'MNSC-PART6-SDS'       : 'imda_part6_30s_ds_human_test',
+
     'OpenHermes-Audio'     : 'openhermes_audio_test',
     'ALPACA-Audio'         : 'alpaca_audio_test',
+
     'WavCaps'              : 'wavcaps_test',
     'AudioCaps'            : 'audiocaps_test',
+
     'Clotho-AQA'           : 'clotho_aqa_test',
     'WavCaps-QA'           : 'wavcaps_qa_test',
     'AudioCaps-QA'         : 'audiocaps_qa_test',
+
+    'IEMOCAP-Emotion'      : 'iemocap_emotion_test',
+    'MELD-Sentiment'       : 'meld_sentiment_test',
+    'MELD-Emotion'         : 'meld_emotion_test',
+
     'VoxCeleb-Accent'      : 'voxceleb_accent_test',
     'MNSC-AR-Sentence'     : 'imda_ar_sentence',
     'MNSC-AR-Dialogue'     : 'imda_ar_dialogue',
+
     'VoxCeleb-Gender'      : 'voxceleb_gender_test',
     'IEMOCAP-Gender'       : 'iemocap_gender_test',
-    'IEMOCAP-Emotion'      : 'iemocap_emotion_test',
-    'MELD-Sentiment'       : 'meld_sentiment_test',
-    'MELD-Emotion'         : 'meld_emotion_test',
+
     'MuChoMusic'           : 'muchomusic_test',
-    'MNSC-PART1-ASR'       : 'imda_part1_asr_test',
-    'MNSC-PART2-ASR'       : 'imda_part2_asr_test',
-    'MNSC-PART3-ASR'       : 'imda_part3_30s_asr_test',
-    'MNSC-PART4-ASR'       : 'imda_part4_30s_asr_test',
-    'MNSC-PART5-ASR'       : 'imda_part5_30s_asr_test',
-    'MNSC-PART6-ASR'       : 'imda_part6_30s_asr_test',
-    'MNSC-PART3-SQA'       : 'imda_part3_30s_sqa_human_test',
-    'MNSC-PART4-SQA'       : 'imda_part4_30s_sqa_human_test',
-    'MNSC-PART5-SQA'       : 'imda_part5_30s_sqa_human_test',
-    'MNSC-PART6-SQA'       : 'imda_part6_30s_sqa_human_test',
-    'MNSC-PART3-SDS'       : 'imda_part3_30s_ds_human_test',
-    'MNSC-PART4-SDS'       : 'imda_part4_30s_ds_human_test',
-    'MNSC-PART5-SDS'       : 'imda_part5_30s_ds_human_test',
-    'MNSC-PART6-SDS'       : 'imda_part6_30s_ds_human_test',
 
-    'CNA'             : 'cna_test',
-    'IDPC'            : 'idpc_test',
-    'Parliament'      : 'parliament_test',
-    'UKUS-News'       : 'ukusnews_test',
-    'Mediacorp'       : 'mediacorp_test',
-    'IDPC-Short'      : 'idpc_short_test',
-    'Parliament-Short': 'parliament_short_test',
-    'UKUS-News-Short' : 'ukusnews_short_test',
-    'Mediacorp-Short' : 'mediacorp_short_test',
-    
-    'YouTube ASR: English Singapore Content': 'ytb_asr_batch1',
-    'YouTube ASR: English with Strong Emotion': 'ytb_asr_batch2',
-    'YouTube ASR: Malay with English Prompt': 'ytb_asr_batch3_malay',
-    'YouTube ASR: Malay with Malay Prompt': 'ytb_asr_batch3_ms_ms_prompt',
-    'YouTube ASR: Chinese with English Prompt': 'ytb_asr_batch3_chinese',
-    'YouTube ASR: Chinese with Chinese Prompt': 'ytb_asr_batch3_zh_zh_prompt',
+    'YouTube SQA: English with Singapore Content': 'ytb_sqa_batch1',
+    'YouTube SDS: English with Singapore Content': 'ytb_sds_batch1',
+    'YouTube PQA: English with Singapore Content': 'ytb_pqa_batch1',
     
     'YouTube SQA: Malay': 'ytb_sqa_batch3_malay',
     'YouTube SQA: Chinese': 'ytb_sqa_batch3_chinese',
@@ -76,15 +265,13 @@ displayname2datasetname = {
     'YouTube SDS: Malay': 'ytb_sds_batch3_malay',
     'YouTube SDS: Chinese': 'ytb_sds_batch3_chinese',
     'YouTube SDS: Tamil': 'ytb_sds_batch3_tamil',
-    
-    'SEAME-Dev-Mandarin'   : 'seame_dev_man',
-    'SEAME-Dev-Singlish'   : 'seame_dev_sge',
+}
 
-    'YouTube SQA: English with Singapore Content': 'ytb_sqa_batch1',
-    'YouTube SDS: English with Singapore Content': 'ytb_sds_batch1',
-    'YouTube PQA: English with Singapore Content': 'ytb_pqa_batch1',
 
-}
+displayname2datasetname = {}
+displayname2datasetname.update(wer_displayname2datasetname)
+displayname2datasetname.update(non_wer_displayname2datasetname)
+
 
 datasetname2diaplayname = {datasetname: displayname for displayname, datasetname in displayname2datasetname.items()}
 
@@ -152,6 +339,19 @@ dataset_diaplay_information = {
     'Parliament-Short': 'Under Development',
     'UKUS-News-Short' : 'Under Development',
     'Mediacorp-Short' : 'Under Development',
+
+    'CommonVoice-ZH'               : 'Under Development',
+    'CommonVoice-17-Indonesian'    : 'Under Development',
+    'CommonVoice-17-Tamil'         : 'Under Development',
+    'CommonVoice-17-Thai'          : 'Under Development',
+    'CommonVoice-17-Vietnamese'    : 'Under Development',
+    'GigaSpeech-2-Indonesain'      : 'Under Development',
+    'GigaSpeech-2-Thai'            : 'Under Development',
+    'GigaSpeech-2-Vietnamese'      : 'Under Development',
+    'Fleurs-Tamil'                 : 'Under Development',
+    'Lotus-Thai'                   : 'Under Development',
+    'MMAU-mini'                    : 'Under Development',
+
     
     'YouTube ASR: English Singapore Content'  : 'YouTube Evaluation Dataset for ASR Task: <br> This dataset contains English and Singlish audio clips, featuring Singapore-related content. <br> It includes approximately 2.5 hours of audio, with individual clips ranging from 2 seconds to 30 seconds in length.',
     

app/draw_diagram.py

@@ -7,6 +7,7 @@ from app.content import *
 
 import pandas as pd
 
+from app.content import wer_displayname2datasetname
 from model_information import get_dataframe
 info_df = get_dataframe()
 
@@ -81,38 +82,7 @@ def draw(folder_name, category_name, displayname, metrics, cus_sort=True):
                 
                 return df_style
 
-        if cur_dataset_name in [
-                            'LibriSpeech-Clean',
-                            'LibriSpeech-Other',
-                            'CommonVoice-15-EN',
-                            'Peoples-Speech',
-                            'GigaSpeech-1',
-                            'Earnings-21',
-                            'Earnings-22',
-                            'TED-LIUM-3',
-                            'TED-LIUM-3-LongForm',
-                            'AISHELL-ASR-ZH',
-                            'MNSC-PART1-ASR',
-                            'MNSC-PART2-ASR',
-                            'MNSC-PART3-ASR',
-                            'MNSC-PART4-ASR',
-                            'MNSC-PART5-ASR',
-                            'MNSC-PART6-ASR',
-                            'CNA',
-                            'IDPC',
-                            'Parliament',
-                            'UKUS-News',
-                            'Mediacorp',
-                            'IDPC-Short',
-                            'Parliament-Short',
-                            'UKUS-News-Short',
-                            'Mediacorp-Short',
-                            'YTB-ASR-Batch1',
-                            'YTB-ASR-Batch2',
-                            'SEAME-Dev-Man',
-                            'SEAME-Dev-Sge',
-                            ]:
-            
+        if cur_dataset_name in wer_displayname2datasetname:
             chart_data_table = chart_data_table.sort_values(
                     by=chart_data_table.columns[1],
                     ascending=True

app/pages.py

@@ -120,28 +120,12 @@ def dashboard():
                         """)
 
 
-
-
-
-
-
 def asr_english():
     st.title("Task: Automatic Speech Recognition - English")
     
     sum = ['Overall']
-    dataset_lists = [
-                    'LibriSpeech-Clean', 
-                    'LibriSpeech-Other', 
-                    'CommonVoice-15-EN', 
-                    'Peoples-Speech', 
-                    'GigaSpeech-1', 
-                    'Earnings-21', 
-                    'Earnings-22', 
-                    'TED-LIUM-3', 
-                    'TED-LIUM-3-LongForm', 
-                    ]
-
-    filters_levelone = sum + dataset_lists
+
+    filters_levelone = sum + asr_english_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -156,23 +140,12 @@ def asr_english():
             draw('su', 'asr_english', filter_1, 'wer', cus_sort=True)
 
 
-
-
-
 def asr_singlish():
     st.title("Task: Automatic Speech Recognition - Singlish")
 
     sum = ['Overall']
-    dataset_lists = [
-                    'MNSC-PART1-ASR', 
-                    'MNSC-PART2-ASR',
-                    'MNSC-PART3-ASR',
-                    'MNSC-PART4-ASR',
-                    'MNSC-PART5-ASR',
-                    'MNSC-PART6-ASR',
-                    ]
 
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + asr_singlish_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -187,17 +160,12 @@ def asr_singlish():
             draw('su', 'asr_singlish', filter_1, 'wer')
 
 
-
-
 def asr_mandarin():
     st.title("Task: Automatic Speech Recognition - Mandarin")
 
     sum = ['Overall']
-    dataset_lists = [
-                    'AISHELL-ASR-ZH', 
-                    ]
 
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + asr_mandarin_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -211,22 +179,53 @@ def asr_mandarin():
             dataset_contents(dataset_diaplay_information[filter_1], metrics_info['wer'])
             draw('su', 'asr_mandarin', filter_1, 'wer')
 
+
+def asr_sea():
+    st.title("Task: Automatic Speech Recognition - SEA Languages")
+
+    sum = ['Overall']
+
+    filters_levelone = sum + asr_sea_datasets
+    
+    left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Dataset', filters_levelone)
+    
+    if filter_1:
+        if filter_1 in sum:
+            sum_table_mulit_metrix('asr_sea', ['wer'])
+        else:
+            dataset_contents(dataset_diaplay_information[filter_1], metrics_info['wer'])
+            draw('su', 'asr_sea', filter_1, 'wer')
+
+
+def asr_private():
+    st.title("Task: Automatic Speech Recognition - Private Datasets")
+
+    sum = ['Overall']
+
+    filters_levelone = sum + asr_private_datasets
     
+    left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
+    
+    with left:
+        filter_1 = st.selectbox('Dataset', filters_levelone)
+    
+    if filter_1:
+        if filter_1 in sum:
+            sum_table_mulit_metrix('asr_private', ['wer'])
+        else:
+            dataset_contents(dataset_diaplay_information[filter_1], metrics_info['wer'])
+            draw('su', 'asr_private', filter_1, 'wer')
 
 
 def speech_translation():
     st.title("Task: Speech Translation")
     
     sum = ['Overall']
-    dataset_lists = [
-                        'CoVoST2-EN-ID', 
-                        'CoVoST2-EN-ZH',
-                        'CoVoST2-EN-TA', 
-                        'CoVoST2-ID-EN', 
-                        'CoVoST2-ZH-EN', 
-                        'CoVoST2-TA-EN']
 
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + speech_translation_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -241,22 +240,12 @@ def speech_translation():
             draw('su', 'ST', filter_1, 'bleu')
 
 
-
-
 def speech_question_answering_english():
     st.title("Task: Spoken Question Answering - English")
     
     sum = ['Overall']
 
-    dataset_lists = [
-                    'CN-College-Listen-MCQ',
-                    'DREAM-TTS-MCQ',
-                    'SLUE-P2-SQA5', 
-                    'Public-SG-Speech-QA', 
-                    'Spoken-SQuAD',
-                     ]
-
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + speech_qa_english_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -276,22 +265,12 @@ def speech_question_answering_english():
             draw('su', 'sqa_english', filter_1, 'llama3_70b_judge')
 
 
-
-
 def speech_question_answering_singlish():
     st.title("Task: Spoken Question Answering - Singlish")
     
     sum = ['Overall']
 
-    dataset_lists = [
-              'MNSC-PART3-SQA', 
-              'MNSC-PART4-SQA',
-              'MNSC-PART5-SQA',
-              'MNSC-PART6-SQA',
-              ]
-
-
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + speech_qa_singlish_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -312,15 +291,7 @@ def spoken_dialogue_summarization_singlish():
     
     sum = ['Overall']
 
-    dataset_lists = [
-              'MNSC-PART3-SDS', 
-              'MNSC-PART4-SDS',
-              'MNSC-PART5-SDS',
-              'MNSC-PART6-SDS',
-              ]
-
-
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + sds_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -336,18 +307,12 @@ def spoken_dialogue_summarization_singlish():
             draw('su', 'sds_singlish', filter_1, 'llama3_70b_judge')
 
 
-
-
 def speech_instruction():
     st.title("Task: Speech Instruction")
     
     sum = ['Overall']
-
-    dataset_lists = ['OpenHermes-Audio', 
-                     'ALPACA-Audio',
-                     ]
     
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + si_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -362,14 +327,11 @@ def speech_instruction():
             draw('su', 'speech_instruction', filter_1, 'llama3_70b_judge')
 
 
-
-
 def audio_captioning():
     st.title("Task: Audio Captioning")
 
-    filters_levelone = ['WavCaps', 
-                        'AudioCaps',
-                        ]
+    filters_levelone = ac_datasets
+
     filters_leveltwo = ['Llama3-70b-judge', 'Meteor']
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
@@ -384,18 +346,12 @@ def audio_captioning():
         draw('asu', 'audio_captioning', filter_1, metric.lower().replace('-', '_'))
 
 
-
-
 def audio_scene_question_answering():
     st.title("Task: Audio Scene Question Answering")
 
     sum = ['Overall']
-
-    dataset_lists = ['Clotho-AQA', 
-                    'WavCaps-QA', 
-                    'AudioCaps-QA']
     
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + asqa_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -410,20 +366,12 @@ def audio_scene_question_answering():
             draw('asu', 'audio_scene_question_answering', filter_1, 'llama3_70b_judge')
 
 
-
-
 def emotion_recognition():
     st.title("Task: Emotion Recognition")
 
     sum = ['Overall']
 
-    dataset_lists = [
-                    'IEMOCAP-Emotion', 
-                    'MELD-Sentiment', 
-                    'MELD-Emotion',
-                    ]
-
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + er_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -438,20 +386,12 @@ def emotion_recognition():
             draw('vu', 'emotion_recognition', filter_1, 'llama3_70b_judge')
 
 
-
-
 def accent_recognition():
     st.title("Task: Accent Recognition")
 
     sum = ['Overall']
-    dataset_lists = [
-        'VoxCeleb-Accent',
-        'MNSC-AR-Sentence',
-        'MNSC-AR-Dialogue',
-        ]
-
 
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + ar_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -467,19 +407,12 @@ def accent_recognition():
             draw('vu', 'accent_recognition', filter_1, 'llama3_70b_judge')
 
 
-
-
 def gender_recognition():
     st.title("Task: Gender Recognition")
     
     sum = ['Overall']
 
-    dataset_lists =  [
-                        'VoxCeleb-Gender', 
-                        'IEMOCAP-Gender'
-                        ]
-
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + gr_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -494,17 +427,12 @@ def gender_recognition():
             draw('vu', 'gender_recognition', filter_1, 'llama3_70b_judge')
 
 
-
-
 def music_understanding():
     st.title("Task: Music Understanding - MCQ Questions")
     
     sum = ['Overall']
 
-    dataset_lists =  ['MuChoMusic',
-                      ]
-
-    filters_levelone = sum + dataset_lists
+    filters_levelone = sum + music_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -519,43 +447,10 @@ def music_understanding():
             draw('vu', 'music_understanding', filter_1, 'llama3_70b_judge')
 
 
-
-
-
 def under_development():
     st.title("Task: Under Development")
-    
 
-    dataset_lists =  [
-                      'YouTube ASR: English Singapore Content',
-                      'YouTube ASR: English with Strong Emotion',
-                      'YouTube ASR: Malay with English Prompt',
-                      'YouTube ASR: Malay with Malay Prompt',
-                      'YouTube ASR: Chinese with English Prompt',
-                      'YouTube ASR: Chinese with Chinese Prompt',
-                    
-                      'YouTube SQA: English with Singapore Content',
-                      'YouTube SDS: English with Singapore Content',
-                      'YouTube PQA: English with Singapore Content',
-                      
-                      'CNA',
-                      'IDPC',
-                      'Parliament',
-                      'UKUS-News',
-                      'Mediacorp',
-                      'IDPC-Short',
-                      'Parliament-Short',
-                      'UKUS-News-Short',
-                      'Mediacorp-Short',
-                      
-                      'SEAME-Dev-Mandarin',
-                      'SEAME-Dev-Singlish',
-
-                      
-
-                      ]
-
-    filters_levelone = dataset_lists
+    filters_levelone = non_wer_development_datasets + wer_development_datasets
     
     left, center, _, middle, right = st.columns([0.4, 0.2, 0.2, 0.2 ,0.2])
     
@@ -592,39 +487,8 @@ def under_development():
                 st.markdown('To be implemented')
     # = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =    
     
-    if filter_1 in [
-                      'CNA',
-                      'IDPC',
-                      'Parliament',
-                      'UKUS-News',
-                      'Mediacorp',
-                      'IDPC-Short',
-                      'Parliament-Short',
-                      'UKUS-News-Short',
-                      'Mediacorp-Short',
-                      
-                      'YouTube ASR: English Singapore Content',
-                      'YouTube ASR: English with Strong Emotion',
-                      'YouTube ASR: Malay with English Prompt',
-                      'YouTube ASR: Malay with Malay Prompt',
-                     
-                      'YouTube ASR: Chinese with English Prompt',
-                      'YouTube ASR: Chinese with Chinese Prompt',
-                      
-                      'SEAME-Dev-Mandarin',
-                      'SEAME-Dev-Singlish',
-                      ]:
-        
+    if filter_1 in wer_development_datasets:
         draw('vu', 'under_development_wer', filter_1, 'wer')
 
-    elif filter_1 in [
-        'YouTube SQA: English with Singapore Content',
-        'YouTube SDS: English with Singapore Content',
-        'YouTube PQA: English with Singapore Content',
-        ]:
+    elif filter_1 in non_wer_development_datasets:
         draw('vu', 'under_development_llama3_70b_judge', filter_1, 'llama3_70b_judge')
-
-
-
-
-

app/summarization.py

@@ -27,7 +27,6 @@ def sum_table_mulit_metrix(task_name, metrics_lists: List[str]):
             chart_data = one_chart_data
         else:
             chart_data = pd.merge(chart_data, one_chart_data, on='Model', how='outer')
-    
 
     selected_columns = [i for i in chart_data.columns if i != 'Model']
     chart_data['Average'] = chart_data[selected_columns].mean(axis=1)
@@ -79,7 +78,8 @@ def sum_table_mulit_metrix(task_name, metrics_lists: List[str]):
         
 
         # Format numeric columns to 2 decimal places
-        chart_data_table[chart_data_table.columns[1]] = chart_data_table[chart_data_table.columns[1]].apply(lambda x: round(float(x), 3) if isinstance(float(x), (int, float)) else float(x))
+        target_column = chart_data_table.columns[1]
+        chart_data_table.loc[:, target_column] = chart_data_table[target_column].apply(lambda x: round(float(x), 3) if isinstance(float(x), (int, float)) else float(x))
 
         if metrics in ['wer']:
             ascend = True

model_information.py

@@ -36,6 +36,30 @@ data['Original Name'].append('MERaLiON-AudioLLM-Whisper-SEA-LION')
 data['Proper Display Name'].append('Fusion: MERaLiON-AudioLLM-Whisper-SEA-LION')
 data['Link'].append('https://huggingface.co/MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION')
 
+data['Original Name'].append('MERaLiON-AudioLLM-v2-2b')
+data['Proper Display Name'].append('Fusion: MERaLiON-2-3B')
+data['Link'].append('https://huggingface.co/MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION')
+
+data['Original Name'].append('MERaLiON-AudioLLM-v2-9b')
+data['Proper Display Name'].append('Fusion: MERaLiON-2-10B')
+data['Link'].append('https://huggingface.co/MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION')
+
+data['Original Name'].append('MERaLiON-AudioLLM-v2-9b-asr')
+data['Proper Display Name'].append('Fusion: MERaLiON-2-10B-ASR')
+data['Link'].append('https://huggingface.co/MERaLiON/MERaLiON-AudioLLM-Whisper-SEA-LION')
+
+data['Original Name'].append('phi_4_multimodal_instruct')
+data['Proper Display Name'].append('Fusion: Phi-4-multimodal-instruct')
+data['Link'].append('https://huggingface.co/microsoft/Phi-4-multimodal-instruct')
+
+data['Original Name'].append('Qwen2.5-Omni-3B')
+data['Proper Display Name'].append('Fusion: Qwen2.5-Omni-3B')
+data['Link'].append('https://huggingface.co/Qwen/Qwen2.5-Omni-3B')
+
+data['Original Name'].append('Qwen2.5-Omni-7B')
+data['Proper Display Name'].append('Fusion: Qwen2.5-Omni-7B')
+data['Link'].append('https://huggingface.co/Qwen/Qwen2.5-Omni-7B')
+
 data['Original Name'].append('cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct')
 data['Proper Display Name'].append('Cascade: Whisper-Large-v2 / SEA-LIONv3')
 data['Link'].append('https://github.com/aisingapore/sealion')
@@ -44,7 +68,6 @@ data['Original Name'].append('whisper_large_v3')
 data['Proper Display Name'].append('Whisper-large-v3')
 data['Link'].append('https://huggingface.co/openai/whisper-large-v3')
 
-
 data['Original Name'].append('gemini-1.5-flash')
 data['Proper Display Name'].append('Gemini-1.5-Flash')
 data['Link'].append('https://ai.google.dev/gemini-api/docs/models/gemini')

results_organized/bleu/st.csv

@@ -1,12 +1,17 @@
 Model,covost2_en_id_test,covost2_en_zh_test,covost2_en_ta_test,covost2_id_en_test,covost2_zh_en_test,covost2_ta_en_test
-Qwen-Audio-Chat,4.102230932924371,15.330641138043728,0.03451483807236294,0.45648619714728844,9.898238298955656,0.01699144301093184
-MERaLiON-AudioLLM-Whisper-SEA-LION,37.60224687716629,43.941098854450516,14.407399367512914,44.43289180618449,18.76473995941838,5.023057608950299
-hy_whisper_local_cs,1.0869208512565696,0.10573269629215352,0.008950516549431693,22.267131378964944,7.31707791416422,2.8610263518826757
-Qwen2-Audio-7B-Instruct,16.325186897428104,25.765420247070075,0.03245972071872916,6.326113431899141,16.466557744958333,0.04425838146050298
-whisper_large_v3,1.600581653970121,0.16408986541757878,0.02107778621423822,46.01512198258627,14.673689493155793,2.451098639578599
+Qwen-Audio-Chat,4.102230932924371,15.330641138043728,0.0345148380723629,0.4564861971472884,9.898238298955656,0.0169914430109318
+hy_whisper_local_cs,1.0869208512565696,0.1057326962921535,0.0089505165494316,22.267131378964944,7.31707791416422,2.8610263518826757
+Qwen2-Audio-7B-Instruct,16.325186897428104,25.765420247070075,0.0324597207187291,6.326113431899141,16.466557744958333,0.0442583814605029
+whisper_large_v3,1.600581653970121,0.1640898654175787,0.0210777862142382,46.01512198258627,14.673689493155791,2.451098639578599
 old_models,,,,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,27.620150160643625,35.274306071307024,8.433062902024755,46.80524126004861,15.209998552437538,2.8327095799289337
 gemini-1.5-flash,,,,,,
-WavLLM_fairseq,13.841886973016162,31.96381187282953,0.0033159224040994286,5.933522277713613,2.368659001743569,0.1695522548322915
-SALMONN_7B,14.102682915273142,33.88941292215531,0.00046745670226766583,26.89649039333571,5.296039450108202,0.3649023706010388
-cascade_whisper_large_v3_llama_3_8b_instruct,10.930203684508578,5.987143868370054,1.0368044741318085,46.79924664837527,14.154700735606419,2.4245628096245917
+WavLLM_fairseq,13.841886973016162,31.96381187282953,0.0033159224040994,5.933522277713613,2.368659001743569,0.1695522548322915
+MERaLiON-AudioLLM-Whisper-SEA-LION,37.058238343330466,43.96331874536172,13.808713343771569,43.37364836260576,19.55610418584389,4.758175879451736
+MERaLiON-AudioLLM-v2-2b,30.658188021678257,40.02820084309168,5.601731502002274,37.77329494766737,16.777825775562142,1.9423083468131173
+MERaLiON-AudioLLM-v2-9b,36.242124109428445,43.747307981166834,10.885517678613343,47.85937752036512,22.133726547487697,3.4786390367027833
+Qwen2.5-Omni-3B,3.2577143149506815,10.28866767786604,0.020665917336912663,15.00712601210481,8.98152195711894,0.04161842995351044
+Qwen2.5-Omni-7B,2.612412992528698,12.429229982446326,0.05482974047730791,12.471476026200369,9.974234734341179,0.02999794683579762
+SALMONN_7B,14.193483776951359,33.255550227097565,0.0005121531999434492,27.88515689237341,5.175547389931541,0.40577007761551664
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,27.59161630015759,28.71368811388653,7.474730798912167,46.80524126004861,15.209998552437538,2.8327095799289337
+cascade_whisper_large_v3_llama_3_8b_instruct,10.753313930099422,6.089840198985321,1.0029597453865848,46.79744652156276,14.156349261775734,2.4177196689141547
+phi_4_multimodal_instruct,14.553644350540432,45.48015814069248,0.14817117451495013,0.37716244197757426,22.330318273444895,0.07320611681035753

results_organized/llama3_70b_judge/accent_recognition.csv

@@ -1,12 +1,17 @@
 Model,voxceleb_accent_test,imda_ar_sentence,imda_ar_dialogue
 Qwen-Audio-Chat,48.05088223225277,3.933333333333333,0.6666666666666667
-MERaLiON-AudioLLM-Whisper-SEA-LION,47.01682396389003,7.816666666666666,77.83333333333333
 hy_whisper_local_cs,,,
-Qwen2-Audio-7B-Instruct,29.187525646286417,2.55,0.9666666666666667
+Qwen2-Audio-7B-Instruct,29.187525646286417,2.55,0.9666666666666668
 whisper_large_v3,,,
 old_models,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,24.640951990151827,26.016666666666666,7.633333333333334
 gemini-1.5-flash,,,
-WavLLM_fairseq,39.96717275338531,2.6833333333333336,0.23333333333333336
-SALMONN_7B,34.222404595814524,2.5166666666666666,0.06666666666666667
-cascade_whisper_large_v3_llama_3_8b_instruct,39.32704144439885,12.416666666666666,9.666666666666666
+WavLLM_fairseq,39.96717275338531,2.6833333333333336,0.2333333333333333
+MERaLiON-AudioLLM-Whisper-SEA-LION,47.066064833812064,,
+MERaLiON-AudioLLM-v2-2b,66.59827656955272,,
+MERaLiON-AudioLLM-v2-9b,40.78785391875257,,
+Qwen2.5-Omni-3B,0.9027492819039803,,
+Qwen2.5-Omni-7B,1.661879359868691,,
+SALMONN_7B,31.69881001231022,,
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,28.00574476815757,,
+cascade_whisper_large_v3_llama_3_8b_instruct,40.29544521953221,,
+phi_4_multimodal_instruct,2.6261797291752154,,

results_organized/llama3_70b_judge/audio_captioning.csv

@@ -1,12 +1,17 @@
 Model,audiocaps_test,wavcaps_test
 Qwen-Audio-Chat,47.04090909090909,32.9364161849711
-MERaLiON-AudioLLM-Whisper-SEA-LION,38.00454545454545,33.97687861271676
 hy_whisper_local_cs,,
 Qwen2-Audio-7B-Instruct,40.77727272727273,33.78034682080925
 whisper_large_v3,,
 old_models,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,3.0954545454545457,6.3468208092485545
 gemini-1.5-flash,,
 WavLLM_fairseq,5.5,6.901734104046243
-SALMONN_7B,37.445454545454545,23.76878612716763
-cascade_whisper_large_v3_llama_3_8b_instruct,2.4727272727272727,3.445086705202312
+MERaLiON-AudioLLM-Whisper-SEA-LION,39.38636363636363,34.566473988439306
+MERaLiON-AudioLLM-v2-2b,35.07727272727273,31.410404624277458
+MERaLiON-AudioLLM-v2-9b,36.04090909090909,35.16763005780347
+Qwen2.5-Omni-3B,43.69545454545454,34.70520231213873
+Qwen2.5-Omni-7B,37.7,26.09248554913295
+SALMONN_7B,35.24090909090909,22.520231213872833
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,2.4545454545454546,3.8265895953757223
+cascade_whisper_large_v3_llama_3_8b_instruct,2.5136363636363637,3.3179190751445087
+phi_4_multimodal_instruct,33.595454545454544,28.069364161849713

results_organized/llama3_70b_judge/audio_scene_question_answering.csv

@@ -1,12 +1,17 @@
 Model,clotho_aqa_test,audiocaps_qa_test,wavcaps_qa_test
 Qwen-Audio-Chat,61.934856587263,50.22364217252396,42.69736842105263
-MERaLiON-AudioLLM-Whisper-SEA-LION,63.15021876519203,49.77635782747604,46.31578947368421
 hy_whisper_local_cs,,,
 Qwen2-Audio-7B-Instruct,50.919591292758774,45.75079872204473,44.473684210526315
 whisper_large_v3,,,
 old_models,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,24.647544968400585,18.466453674121407,18.88157894736842
 gemini-1.5-flash,,,
 WavLLM_fairseq,43.01199466903598,29.840255591054312,26.25
-SALMONN_7B,57.75401069518716,50.287539936102235,47.30263157894737
-cascade_whisper_large_v3_llama_3_8b_instruct,29.47134606841404,17.380191693290733,16.710526315789473
+MERaLiON-AudioLLM-Whisper-SEA-LION,62.67379679144385,48.81789137380192,45.131578947368425
+MERaLiON-AudioLLM-v2-2b,50.53962080700049,44.79233226837061,43.0921052631579
+MERaLiON-AudioLLM-v2-9b,58.20126397666505,50.35143769968051,44.868421052631575
+Qwen2.5-Omni-3B,52.64948954788527,48.56230031948882,43.15789473684211
+Qwen2.5-Omni-7B,46.592124453087024,50.41533546325879,40.0
+SALMONN_7B,58.19154107924162,50.35143769968051,46.90789473684211
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,22.673796791443852,17.44408945686901,14.013157894736842
+cascade_whisper_large_v3_llama_3_8b_instruct,29.820126397666506,17.06070287539936,18.75
+phi_4_multimodal_instruct,48.37141468157511,40.319488817891376,37.96052631578947

results_organized/llama3_70b_judge/emotion_recognition.csv

@@ -1,12 +1,17 @@
 Model,iemocap_emotion_test,meld_sentiment_test,meld_emotion_test
 Qwen-Audio-Chat,29.382470119521916,44.90421455938697,50.72796934865901
-MERaLiON-AudioLLM-Whisper-SEA-LION,48.505976095617534,46.206896551724135,36.36015325670498
 hy_whisper_local_cs,,,
 Qwen2-Audio-7B-Instruct,53.98406374501992,53.9463601532567,41.60919540229885
 whisper_large_v3,,,
 old_models,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,44.322709163346616,56.59003831417625,47.356321839080465
 gemini-1.5-flash,,,
 WavLLM_fairseq,59.76095617529881,51.072796934865906,41.57088122605364
-SALMONN_7B,23.804780876494025,41.7624521072797,30.536398467432953
-cascade_whisper_large_v3_llama_3_8b_instruct,46.713147410358566,45.593869731800766,36.81992337164751
+MERaLiON-AudioLLM-Whisper-SEA-LION,49.103585657370516,52.452107279693486,44.17624521072797
+MERaLiON-AudioLLM-v2-2b,51.39442231075698,58.582375478927204,52.1455938697318
+MERaLiON-AudioLLM-v2-9b,62.54980079681275,68.85057471264368,59.808429118773944
+Qwen2.5-Omni-3B,34.36254980079681,30.421455938697317,34.32950191570881
+Qwen2.5-Omni-7B,36.55378486055777,27.77777777777778,30.07662835249042
+SALMONN_7B,26.195219123505975,42.26053639846744,32.298850574712645
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,41.98207171314741,58.39080459770115,44.272030651341
+cascade_whisper_large_v3_llama_3_8b_instruct,46.91235059760957,56.47509578544061,49.42528735632184
+phi_4_multimodal_instruct,32.07171314741036,49.11877394636016,40.84291187739464

results_organized/llama3_70b_judge/gender_recognition.csv

@@ -1,12 +1,17 @@
-Model,voxceleb_gender_test,iemocap_gender_test,imda_gr_sentence,imda_gr_dialogue
-Qwen-Audio-Chat,70.5990972507181,50.0996015936255,57.550000000000004,37.2
-MERaLiON-AudioLLM-Whisper-SEA-LION,99.75379565038982,93.48605577689243,66.13333333333333,93.76666666666667
-hy_whisper_local_cs,,,,
-Qwen2-Audio-7B-Instruct,99.1177677472302,92.80876494023903,68.38333333333333,61.56666666666667
-whisper_large_v3,,,,
-old_models,,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,34.94050061551087,15.737051792828685,26.35,19.6
-gemini-1.5-flash,,,,
-WavLLM_fairseq,69.61427985227739,51.932270916334666,49.06666666666666,46.766666666666666
-SALMONN_7B,88.79770209273697,81.31474103585658,59.766666666666666,42.733333333333334
-cascade_whisper_large_v3_llama_3_8b_instruct,42.921624948707425,44.22310756972111,36.016666666666666,25.433333333333337
+Model,voxceleb_gender_test,iemocap_gender_test
+Qwen-Audio-Chat,70.5990972507181,50.0996015936255
+hy_whisper_local_cs,,
+Qwen2-Audio-7B-Instruct,99.1177677472302,92.80876494023904
+whisper_large_v3,,
+old_models,,
+gemini-1.5-flash,,
+WavLLM_fairseq,69.61427985227739,51.932270916334666
+MERaLiON-AudioLLM-Whisper-SEA-LION,99.73327862125564,94.6215139442231
+MERaLiON-AudioLLM-v2-2b,99.69224456298728,87.92828685258964
+MERaLiON-AudioLLM-v2-9b,97.2507180960197,92.96812749003983
+Qwen2.5-Omni-3B,32.78621255642183,62.948207171314735
+Qwen2.5-Omni-7B,54.08288879770209,43.366533864541836
+SALMONN_7B,88.53098071399262,80.199203187251
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,26.631103816167418,12.211155378486056
+cascade_whisper_large_v3_llama_3_8b_instruct,69.69634796881412,44.38247011952191
+phi_4_multimodal_instruct,94.58350430857611,46.852589641434264

results_organized/llama3_70b_judge/music_understanding.csv

@@ -1,12 +1,17 @@
 Model,muchomusic_test
 Qwen-Audio-Chat,59.0564448188711
-MERaLiON-AudioLLM-Whisper-SEA-LION,57.7927548441449
 hy_whisper_local_cs,
 Qwen2-Audio-7B-Instruct,71.60909856781802
 whisper_large_v3,
 old_models,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,51.727042965459134
 gemini-1.5-flash,
 WavLLM_fairseq,44.3133951137321
-SALMONN_7B,50.88458298230834
-cascade_whisper_large_v3_llama_3_8b_instruct,56.44481887110362
+MERaLiON-AudioLLM-Whisper-SEA-LION,51.34793597304128
+MERaLiON-AudioLLM-v2-2b,55.602358887952825
+MERaLiON-AudioLLM-v2-9b,63.94271272114573
+Qwen2.5-Omni-3B,59.30918281381634
+Qwen2.5-Omni-7B,47.598989048020215
+SALMONN_7B,49.70513900589722
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,50.463352990732936
+cascade_whisper_large_v3_llama_3_8b_instruct,56.697556866048856
+phi_4_multimodal_instruct,55.2653748946925

results_organized/llama3_70b_judge/sds_singlish.csv

@@ -1,12 +1,17 @@
 Model,imda_part3_30s_ds_human_test,imda_part4_30s_ds_human_test,imda_part5_30s_ds_human_test,imda_part6_30s_ds_human_test
 Qwen-Audio-Chat,16.4,16.0,28.2,40.4
-MERaLiON-AudioLLM-Whisper-SEA-LION,48.4,46.4,57.0,62.599999999999994
 hy_whisper_local_cs,,,,
 Qwen2-Audio-7B-Instruct,33.8,24.8,40.4,46.2
 whisper_large_v3,,,,
 old_models,,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,45.4,44.0,58.0,65.4
 gemini-1.5-flash,,,,
-WavLLM_fairseq,31.6,31.6,45.199999999999996,49.400000000000006
-SALMONN_7B,9.0,7.0,17.2,24.2
-cascade_whisper_large_v3_llama_3_8b_instruct,37.400000000000006,36.0,49.0,57.199999999999996
+WavLLM_fairseq,31.6,31.6,45.2,49.400000000000006
+MERaLiON-AudioLLM-Whisper-SEA-LION,47.800000000000004,46.4,54.6,65.6
+MERaLiON-AudioLLM-v2-2b,42.2,40.199999999999996,51.8,60.0
+MERaLiON-AudioLLM-v2-9b,49.8,46.6,55.4,60.599999999999994
+Qwen2.5-Omni-3B,42.800000000000004,33.199999999999996,52.199999999999996,58.8
+Qwen2.5-Omni-7B,39.8,31.6,42.800000000000004,58.4
+SALMONN_7B,9.0,7.4,16.0,25.2
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,48.4,45.599999999999994,53.4,56.6
+cascade_whisper_large_v3_llama_3_8b_instruct,38.0,38.199999999999996,46.2,61.0
+phi_4_multimodal_instruct,43.6,42.800000000000004,55.599999999999994,61.0

results_organized/llama3_70b_judge/speech_instruction.csv

@@ -1,12 +1,17 @@
 Model,openhermes_audio_test,alpaca_audio_test
-Qwen-Audio-Chat,10.600000000000001,9.8
-MERaLiON-AudioLLM-Whisper-SEA-LION,65.6,74.80000000000001
+Qwen-Audio-Chat,10.6,9.8
 hy_whisper_local_cs,,
-Qwen2-Audio-7B-Instruct,44.800000000000004,52.599999999999994
+Qwen2-Audio-7B-Instruct,44.8,52.599999999999994
 whisper_large_v3,,
 old_models,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,72.2,73.8
 gemini-1.5-flash,,
 WavLLM_fairseq,19.2,21.6
-SALMONN_7B,15.8,17.2
-cascade_whisper_large_v3_llama_3_8b_instruct,63.0,70.8
+MERaLiON-AudioLLM-Whisper-SEA-LION,66.39999999999999,75.19999999999999
+MERaLiON-AudioLLM-v2-2b,12.6,25.6
+MERaLiON-AudioLLM-v2-9b,66.2,74.2
+Qwen2.5-Omni-3B,66.0,64.0
+Qwen2.5-Omni-7B,57.400000000000006,59.2
+SALMONN_7B,15.4,10.4
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,78.8,67.0
+cascade_whisper_large_v3_llama_3_8b_instruct,62.800000000000004,69.4
+phi_4_multimodal_instruct,39.0,33.4

results_organized/llama3_70b_judge/sqa_english.csv

@@ -1,12 +1,15 @@
-Model,slue_p2_sqa5_test,public_sg_speech_qa_test,spoken_squad_test,cn_college_listen_mcq_test,dream_tts_mcq_test
-Qwen-Audio-Chat,79.36274509803921,63.16860465116279,64.8327415436367,63.232056362835756,59.749085206481965
-MERaLiON-AudioLLM-Whisper-SEA-LION,86.76470588235293,59.7093023255814,73.66473556344609,88.50726552179657,84.31782540512285
-hy_whisper_local_cs,,,,,
-Qwen2-Audio-7B-Instruct,80.04901960784315,58.31395348837209,64.86264249672958,74.7247908410392,66.49242028227914
-whisper_large_v3,,,,,
-old_models,,,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,88.57843137254902,73.11046511627907,88.61894972902262,91.85380889476001,89.33612127548353
-gemini-1.5-flash,,,,89.25583443416997,
-WavLLM_fairseq,83.92156862745098,58.54651162790698,77.64903756307233,66.31439894319684,66.5446941975954
-SALMONN_7B,83.48039215686273,59.24418604651163,66.39506634273968,50.99075297225891,56.455828541557764
-cascade_whisper_large_v3_llama_3_8b_instruct,82.99019607843137,64.94186046511628,83.81984675761541,85.2928225451343,86.4610559330894
+Model,slue_p2_sqa5_test,public_sg_speech_qa_test,spoken_squad_test,cn_college_listen_mcq_test,dream_tts_mcq_test,mmau_mini
+Qwen-Audio-Chat,79.36274509803921,63.16860465116279,64.8327415436367,63.23205636283576,59.749085206481965,
+hy_whisper_local_cs,,,,,,
+Qwen2-Audio-7B-Instruct,80.04901960784315,58.31395348837209,64.86264249672958,74.7247908410392,66.49242028227914,
+whisper_large_v3,,,,,,
+old_models,,,,,,
+gemini-1.5-flash,,,,89.25583443416997,,
+WavLLM_fairseq,83.92156862745098,58.54651162790698,77.64903756307233,66.31439894319684,66.5446941975954,
+MERaLiON-AudioLLM-Whisper-SEA-LION,86.7156862745098,59.59302325581396,74.20669033825453,57.11140466754734,51.54208050182959,53.1
+MERaLiON-AudioLLM-v2-2b,83.18627450980392,69.47674418604652,81.4614090824145,66.00616468516073,61.16048092002091,50.99999999999999
+MERaLiON-AudioLLM-v2-9b,89.55882352941177,75.02906976744187,89.20949355260699,84.58828709819463,83.32462101411396,56.699999999999996
+SALMONN_7B,80.88235294117646,59.38953488372093,65.64754251541768,50.81461911052399,56.56037637219028,50.6
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,51.51960784313726,70.93023255813954,57.16314707531303,89.52003522677234,85.15420805018296,52.6
+cascade_whisper_large_v3_llama_3_8b_instruct,86.96078431372548,69.68023255813954,87.43412446271725,84.98458828709819,86.1996863565081,55.900000000000006
+phi_4_multimodal_instruct,83.72549019607844,74.18604651162791,83.19566436180153,75.6494936151475,77.5222164140094,58.8

results_organized/llama3_70b_judge/sqa_singlish.csv

@@ -1,12 +1,17 @@
 Model,imda_part3_30s_sqa_human_test,imda_part4_30s_sqa_human_test,imda_part5_30s_sqa_human_test,imda_part6_30s_sqa_human_test
-Qwen-Audio-Chat,32.2,37.8,47.800000000000004,51.4
-MERaLiON-AudioLLM-Whisper-SEA-LION,51.4,53.2,64.80000000000001,67.2
+Qwen-Audio-Chat,32.2,37.8,47.8,51.4
 hy_whisper_local_cs,,,,
 Qwen2-Audio-7B-Instruct,42.0,39.6,51.6,53.6
 whisper_large_v3,,,,
 old_models,,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,56.0,66.0,74.0,71.6
 gemini-1.5-flash,,,,
-WavLLM_fairseq,45.199999999999996,46.6,50.8,62.199999999999996
-SALMONN_7B,40.599999999999994,36.6,44.6,46.8
-cascade_whisper_large_v3_llama_3_8b_instruct,49.0,53.8,57.800000000000004,64.0
+WavLLM_fairseq,45.2,46.6,50.8,62.2
+MERaLiON-AudioLLM-Whisper-SEA-LION,55.199999999999996,50.0,63.0,67.4
+MERaLiON-AudioLLM-v2-2b,52.599999999999994,54.6,61.4,70.19999999999999
+MERaLiON-AudioLLM-v2-9b,59.400000000000006,63.0,72.0,71.8
+Qwen2.5-Omni-3B,52.400000000000006,54.400000000000006,66.0,69.2
+Qwen2.5-Omni-7B,54.2,52.0,62.800000000000004,64.6
+SALMONN_7B,42.0,35.4,45.8,49.6
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,51.4,46.4,54.6,62.599999999999994
+cascade_whisper_large_v3_llama_3_8b_instruct,51.6,55.599999999999994,62.0,68.2
+phi_4_multimodal_instruct,55.0,56.4,64.6,71.8

results_organized/llama3_70b_judge/under_development_llama3_70b_judge.csv

@@ -7,6 +7,6 @@ whisper_large_v3,,,
 old_models,,,
 cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,70.18719211822659,64.12654745529574,55.01831501831502
 gemini-1.5-flash,78.06896551724138,65.9697386519945,49.908424908424905
-WavLLM_fairseq,60.70935960591133,55.625859697386524,40.95238095238095
-SALMONN_7B,55.665024630541865,31.279229711141674,32.124542124542124
+WavLLM_fairseq,60.70935960591133,55.62585969738653,40.95238095238095
+SALMONN_7B,55.665024630541865,31.27922971114167,32.124542124542124
 cascade_whisper_large_v3_llama_3_8b_instruct,67.3103448275862,59.44979367262724,52.252747252747255

results_organized/meteor/audio_captioning.csv

@@ -1,12 +1,12 @@
 Model,audiocaps_test,wavcaps_test
-Qwen-Audio-Chat,0.27553015076950976,0.2355106805560457
-MERaLiON-AudioLLM-Whisper-SEA-LION,0.24920047034353812,0.3175511907248581
+Qwen-Audio-Chat,0.2755301507695097,0.2355106805560457
+MERaLiON-AudioLLM-Whisper-SEA-LION,0.2492004703435381,0.3175511907248581
 hy_whisper_local_cs,,
-Qwen2-Audio-7B-Instruct,0.19891712076314283,0.21342294856199182
+Qwen2-Audio-7B-Instruct,0.1989171207631428,0.2134229485619918
 whisper_large_v3,,
 old_models,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.05796819723943051,0.120421856260385
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.0579681972394305,0.120421856260385
 gemini-1.5-flash,,
-WavLLM_fairseq,0.041732965094428545,0.06399522524688675
-SALMONN_7B,0.20994052484339956,0.17175112770658157
-cascade_whisper_large_v3_llama_3_8b_instruct,0.07953048457785493,0.1388630786594543
+WavLLM_fairseq,0.0417329650944285,0.0639952252468867
+SALMONN_7B,0.2099405248433995,0.1717511277065815
+cascade_whisper_large_v3_llama_3_8b_instruct,0.0795304845778549,0.1388630786594543

results_organized/wer/asr_english.csv

@@ -1,12 +1,18 @@
 Model,librispeech_test_clean,librispeech_test_other,common_voice_15_en_test,peoples_speech_test,gigaspeech_test,earnings21_test,earnings22_test,tedlium3_test,tedlium3_long_form_test
-Qwen-Audio-Chat,0.020258799562379748,0.043467569561352074,0.11272421128398918,0.31419144746723354,0.13018910022587737,0.2655529121410546,0.3664994875132684,0.04052375714133636,0.2911540507002305
-MERaLiON-AudioLLM-Whisper-SEA-LION,0.024333195005092994,0.04212457676811621,0.07789795695400416,0.21632867288683053,0.14468436081215577,0.1384587164122689,0.16563713100701868,0.08094105957914907,0.10501684098564085
-hy_whisper_local_cs,0.02554042328441544,0.053417065466169825,0.1066766923091754,0.1991585778678581,0.0948233719154953,0.10871196540338629,0.1463228189913085,0.0467690997480572,0.05275660343910654
-Qwen2-Audio-7B-Instruct,0.035141660693401744,0.060415760304159495,0.11438872500819404,0.2165498391593041,0.11723812890302816,0.18872219319407232,0.23542555661330924,0.06114048472375004,0.08739585179932637
-whisper_large_v3,0.01878749009695552,0.03660128246354058,0.10001863741235596,0.14602420615337386,0.09459022434812692,0.11863959266711877,0.15887899737116104,0.037649480146197796,0.03208650948413402
+Qwen-Audio-Chat,0.0202587995623797,0.043467569561352,0.1127242112839891,0.3141914474672335,0.1301891002258773,0.2655529121410546,0.3664994875132684,0.0405237571413363,0.2911540507002305
+Qwen2-Audio-7B-Instruct,0.0351416606934017,0.0604157603041594,0.114388725008194,0.2165498391593041,0.1172381289030281,0.1887221931940723,0.2354255566133092,0.06114048472375,0.0873958517993263
+whisper_large_v3,0.0187874900969555,0.0366012824635405,0.1000186374123559,0.1460242061533738,0.0945902243481269,0.1186395926671187,0.158878997371161,0.0376494801461977,0.032086509484134
 old_models,,,,,,,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.032349945297468596,0.05307658841999735,0.10600831614192711,0.20140159998943682,0.09948381629977261,0.11416493424197618,0.1448629161356777,0.04900464852205386,0.04396383619925545
 gemini-1.5-flash,,,,,,,,,
-WavLLM_fairseq,0.02103218017882069,0.04798834811886432,0.14533325621300636,0.3792176325635977,0.15491778414546403,0.6447482518259942,0.6671766188447099,0.06621482559171073,0.4536784258110264
-SALMONN_7B,0.10270871845172973,0.09671439650443565,0.3062255383962828,0.23699946689025367,0.10765150204693537,0.2577708974886327,0.3597423676988383,0.0459884319222171,0.14231519234178336
-cascade_whisper_large_v3_llama_3_8b_instruct,0.018334779492209605,0.03714982881570734,0.09876543209876543,0.14540692118393275,0.09515429104337297,0.11773910240019567,0.15611126487402763,0.038146268762641496,0.04754476156709803
+WavLLM_fairseq,0.0210321801788206,0.0479883481188643,0.1453332562130063,0.3792176325635977,0.154917784145464,0.6447482518259942,0.6671766188447099,0.0662148255917107,0.4536784258110264
+MERaLiON-AudioLLM-Whisper-SEA-LION,0.023937073225940318,0.0422569845082944,0.07797507728099434,0.21620323529945748,0.14477210452030514,0.13838923413858656,0.16553574886426656,0.08154430289911642,0.10512320510547775
+MERaLiON-AudioLLM-v2-2b,0.027124910401026145,0.050958064577146425,0.09270505973611995,0.20627055897299626,0.09237908290276242,0.21886082422652334,0.23935918375209228,0.03456229374401192,0.13837971990781775
+MERaLiON-AudioLLM-v2-9b,0.02497453502848304,0.046607524542720415,0.08676036786395974,0.20476530792451958,0.09023061553464748,0.1084090226901313,0.15062142184399924,0.03513005216280473,0.043573834426520124
+MERaLiON-AudioLLM-v2-9b-asr,0.020956728411363035,0.04040327614579984,0.0761563229028091,0.1957668115250735,0.08768103407213536,0.09210848128425476,0.1277414998676963,0.0313686526383024,0.03495834071973054
+Qwen2.5-Omni-3B,0.01765571358509073,0.03898462178674788,0.08397118270448134,0.2217852079375585,0.09894231227233641,0.12490689375326566,0.18720009894897133,0.03211383556296796,0.052153873426697396
+Qwen2.5-Omni-7B,0.02252235258610933,0.04165169198176556,0.08635548614726127,0.31617534194121266,0.12679717916513114,0.23232370957521317,0.2807910240306093,0.0633760334977467,0.09094132246055664
+SALMONN_7B,0.09638963292715132,0.11776722719276675,0.315955552984878,0.24158949229136512,0.11024871580815716,0.27733154717568453,0.37956460424973665,0.039352755402576205,0.14139336996986349
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.03299128532085864,0.05381428868670437,0.10610471655066483,0.20285898669536326,0.09994259054523941,0.14091838890062366,0.17187922953626794,0.04939498243497392,0.08636766530756958
+cascade_whisper_large_v3_llama_3_8b_instruct,0.018032972422378994,0.035504189759207064,0.09879113887442882,0.14542012514049835,0.09501640807342393,0.10872308256717546,0.1459710229559586,0.038146268762641496,0.04935295160432548
+hy_whisper_local_cs,0.029086656354925113,0.05591389713810127,0.1066766923091754,0.17879147486544342,0.10212866235970408,0.14925070316060968,0.17014458107377883,0.04666264504453355,0.06973940790639957
+phi_4_multimodal_instruct,0.016844607084920964,0.03851173700039722,0.08109202383018103,0.2147161396912585,0.0988294989332872,0.1306461295594268,0.22572024408764688,0.028636315247862035,0.05062932104236838

results_organized/wer/asr_mandarin.csv

@@ -1,12 +1,18 @@
-Model,aishell_asr_zh_test
-Qwen-Audio-Chat,0.9469917443725129
-MERaLiON-AudioLLM-Whisper-SEA-LION,0.12812060739244918
-hy_whisper_local_cs,0.16361782582011838
-Qwen2-Audio-7B-Instruct,0.09260359129694522
-whisper_large_v3,0.12359684029221357
-old_models,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.20886539565639167
-gemini-1.5-flash,
-WavLLM_fairseq,0.7054601967888183
-SALMONN_7B,0.8259290055631446
-cascade_whisper_large_v3_llama_3_8b_instruct,0.12450753301261111
+Model,aishell_asr_zh_test,commonvoice_zh_asr
+Qwen-Audio-Chat,0.9469917443725128,
+Qwen2-Audio-7B-Instruct,0.0926035912969452,
+whisper_large_v3,0.1235968402922135,
+old_models,,
+gemini-1.5-flash,,
+WavLLM_fairseq,0.7054601967888183,
+MERaLiON-AudioLLM-Whisper-SEA-LION,0.12846706657955692,0.3269799259362027
+MERaLiON-AudioLLM-v2-2b,0.05010789728969927,0.13139387212789344
+MERaLiON-AudioLLM-v2-9b,0.05789827958266516,0.14684695260557293
+MERaLiON-AudioLLM-v2-9b-asr,0.043317297222387204,0.1183419954537208
+Qwen2.5-Omni-3B,0.08080418126744669,0.08551487145555639
+Qwen2.5-Omni-7B,0.08943596444338857,0.0775535468448182
+SALMONN_7B,0.9314703727900854,1.0013340021130595
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.20889509215814378,0.31938144990021666
+cascade_whisper_large_v3_llama_3_8b_instruct,0.12450753301261111,0.1962263748225777
+hy_whisper_local_cs,0.15675793391538476,0.287290695068461
+phi_4_multimodal_instruct,0.12232978955079092,0.154221316286565

results_organized/wer/asr_private.csv

@@ -0,0 +1,12 @@
+Model,cna_test,idpc_short_test,idpc_test,mediacorp_short_test,mediacorp_test,parliament_test,ukusnews_test,ytb_asr_batch1,ytb_asr_batch2,ytb_asr_batch3_chinese,ytb_asr_batch3_malay,ytb_asr_batch3_tamil
+MERaLiON-AudioLLM-Whisper-SEA-LION,0.14503898323187012,0.16498433693003828,0.20359281437125748,0.12828873397796267,0.12250898399215943,0.058780395496262655,0.1128757799205899,0.10724437274333563,0.13268461455292463,0.418102808691044,0.28989513404414025,0.6929759165018962
+MERaLiON-AudioLLM-v2-2b,0.13494606429563175,0.15106160807518274,0.17741659538066723,0.1208680008994828,0.12250898399215943,0.18544800832623712,0.17383248251087163,0.09933164323576861,0.15990917937074278,0.25613142554319024,0.2798911851169321,0.7504943113675407
+MERaLiON-AudioLLM-v2-9b,0.13334401367083198,0.15663069961712495,0.16030795551753635,0.11693276366089499,0.10454099967330938,0.06024694862333239,0.06972962752883342,0.09848659445340709,0.1110174072872743,0.19133015368309486,0.20907375718485366,0.6644679264853651
+MERaLiON-AudioLLM-v2-9b-asr,0.12709601623411299,0.14009745910198398,0.16612489307100087,0.11783224645828648,0.10372427311336165,0.05284322073989971,0.055965210814898844,0.09230237381885227,0.09936209319926478,0.1494223635400106,0.19463823439076827,0.5467894071504975
+Qwen2.5-Omni-3B,0.15224821104346897,0.3038635572572224,0.19743370402053037,0.13660894985383404,0.1391702058150931,0.09165957044185827,0.0828512006050293,0.12241683951755397,0.24802681370959023,0.2562374138844727,2.2815585099381335,1.2873650773070564
+Qwen2.5-Omni-7B,0.17280786072839902,0.4491820396797772,0.6198460222412319,0.26714639082527547,0.3391048676902973,0.2558898665909736,0.22628096048402344,0.20300376430821235,0.34827548924208024,0.19881293057763647,1.4799262866921152,1.0804025801432693
+SALMONN_7B,0.1492577165438428,0.2398190045248869,0.5414884516680923,0.19901056892286936,0.3636883371447239,0.20430031223389156,0.191869918699187,0.2207497887378044,0.3495513028435506,0.8858293587705353,1.0858672282918695,0.985267900554277
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.15171419416853574,0.19735468151757746,0.17040205303678357,0.1541488644029683,0.15754655341391702,0.09007474690131517,0.12278313480809226,0.12475992932319274,0.12552708400908205,0.3469210386857446,0.3143784827344127,0.9665002755178114
+cascade_whisper_large_v3_llama_3_8b_instruct,0.13815016554523124,0.15344926428434932,0.16184773310521813,0.11434675061839443,0.15125775890231952,0.06537988456807645,0.08943089430894309,0.10816624414227549,0.08387933830684398,0.2698675145733969,0.3119213724715897,0.8976532365239376
+hy_whisper_local_cs,0.14674783723165652,0.18308388444135051,0.17570573139435414,0.12885091072633237,0.1256125449199608,0.07257072570725707,0.16948383437322745,0.1284858262272413,0.14315061087685155,0.27520932697403283,0.2421569917950068,0.8339924151567211
+phi_4_multimodal_instruct,0.19080422941364947,0.5388096066829099,0.26073567151411464,0.1217674836968743,0.19813786344331918,0.2778645094143249,0.07521270561542824,0.16939386955519706,0.23232781922369986,0.44008479067302597,3.762932736606555,2.7500567242552916

results_organized/wer/asr_sea.csv

@@ -0,0 +1,12 @@
+Model,commonvoice_17_id_asr,commonvoice_17_ta_asr,commonvoice_17_vi_asr,fleurs_tamil_ta_30_asr,gigaspeech2_id_test,gigaspeech2_th_test,gigaspeech2_vi_test,lotus_thai_th_30_asr
+MERaLiON-AudioLLM-Whisper-SEA-LION,0.25954549636581103,0.5284951114826634,0.9221892864704637,0.4624736472241743,0.337184855698226,0.9866395307075302,0.9818897503814326,0.8520208370756243
+MERaLiON-AudioLLM-v2-2b,0.08547244456711749,0.13853008043879414,0.14196485284776625,0.1432185523541813,0.17842684134623737,0.19968394588770502,0.16825573283269715,0.014873360876594216
+MERaLiON-AudioLLM-v2-9b,0.11334989419449812,0.15591770571023683,0.15646834639000634,0.16085734364019677,0.1722759890883186,0.20004788698671136,0.11314793912959634,0.018681516076881625
+MERaLiON-AudioLLM-v2-9b-asr,0.07921611923820039,0.12871226564172622,0.1423883125132331,0.1383345045678145,0.16282383194620612,0.18238237758889023,0.09499798648962901,0.010670019759295851
+Qwen2.5-Omni-3B,0.13731714049130556,1.0276387288835422,0.2463476603853483,1.3477160927617708,0.3110002953799107,0.4670274152998923,0.19581530154444754,0.4822705227231902
+Qwen2.5-Omni-7B,0.18235348238108381,1.0684188526512177,0.22041075587550285,1.2090302178496135,0.26146334682814104,0.2936956781994493,0.22408385278119664,0.0984012933357284
+SALMONN_7B,1.1888858220627472,1.4272941368377052,1.496294727927165,1.507519325368939,2.1181172136986777,1.2470441757452413,1.5460526688938172,1.1351535836177475
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.09977918851780293,0.23805397249380653,0.1567859411391065,0.2724525650035137,0.2191718937327333,0.276058900993655,0.17136958408249153,0.06815160768816239
+cascade_whisper_large_v3_llama_3_8b_instruct,0.07815806421933941,0.24404355317218387,0.11676900275248782,0.28397751229796203,0.1926224523482703,0.20872022028013887,0.15538061017872032,0.031794503323154304
+hy_whisper_local_cs,0.10267733922163952,0.31793713743921215,0.1681134871903451,0.33113141250878425,0.21382030476256667,0.26486292350053875,0.1781020821398794,0.076019400035926
+phi_4_multimodal_instruct,1.327169012788665,1.1784589191228196,1.1070294304467498,1.7016514406184118,5.803850364012302,1.7344522925894887,2.5042567310800923,1.2856834920064666

results_organized/wer/asr_singlish.csv

@@ -1,12 +1,18 @@
 Model,imda_part1_asr_test,imda_part2_asr_test,imda_part3_30s_asr_test,imda_part4_30s_asr_test,imda_part5_30s_asr_test,imda_part6_30s_asr_test
-Qwen-Audio-Chat,0.10550313315290274,0.45479263046830615,0.6412550574306894,1.173131813552289,0.3016882870525747,0.31394240863063033
-MERaLiON-AudioLLM-Whisper-SEA-LION,0.042815692585277836,0.04719584449314179,0.2139462894072284,0.3002929748896629,0.15368227517473845,0.10833508293092589
-hy_whisper_local_cs,0.06319947333772219,0.2719340962584206,0.23856138159502538,0.33742408429629445,0.16663991478309087,0.12873269917149824
-Qwen2-Audio-7B-Instruct,0.07197717796796138,0.1905689473257041,0.35076166942732234,0.5613424034000176,0.27856006770658537,0.2245352799625317
-whisper_large_v3,0.06844171360300393,0.3171008846684522,0.27026366524560785,0.4618189591218298,0.2143555471246589,0.1698509342851144
+Qwen-Audio-Chat,0.1055031331529027,0.4547926304683061,0.6412550574306894,1.173131813552289,0.3016882870525747,0.3139424086306303
+Qwen2-Audio-7B-Instruct,0.0719771779679613,0.1905689473257041,0.3507616694273223,0.5613424034000176,0.2785600677065853,0.2245352799625317
+whisper_large_v3,0.0684417136030039,0.3171008846684522,0.2702636652456078,0.4618189591218298,0.2143555471246589,0.1698509342851144
 old_models,,,,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.07041669714480775,0.32988393799204613,0.3035544573275043,0.4779640131272869,0.22881615619208825,0.1789273082575623
 gemini-1.5-flash,,,,,,
-WavLLM_fairseq,0.10077292565771828,0.4463923382842302,0.7540934640345399,1.143645714142011,0.39796588405247263,0.42541061709652933
-SALMONN_7B,0.0925804013361617,0.42346400454508565,0.6569229098215983,0.7593582215292535,0.34868891450584405,0.24872817713464365
-cascade_whisper_large_v3_llama_3_8b_instruct,0.06922195401458074,0.31912994075156237,0.29992939962527493,0.4750971343786543,0.22004640235805695,0.17467982364056267
+WavLLM_fairseq,0.1007729256577182,0.4463923382842302,0.7540934640345399,1.143645714142011,0.3979658840524726,0.4254106170965293
+MERaLiON-AudioLLM-Whisper-SEA-LION,0.04303513520103382,0.0473581689797906,0.21299589974746788,0.29660878421707804,0.15406166552363165,0.1087388362215152
+MERaLiON-AudioLLM-v2-2b,0.049057615877892376,0.05819332846359873,0.26414044043772233,0.3595795244502006,0.20202536078562985,0.1493725673864242
+MERaLiON-AudioLLM-v2-9b,0.051959134908443665,0.14532099667234802,0.22654574089662477,0.2948987161915779,0.16760298259181977,0.12655243140231592
+MERaLiON-AudioLLM-v2-9b-asr,0.04362031550971643,0.054094635175716256,0.19622831075026476,0.24570911239925058,0.1403598371539887,0.0989680065892537
+Qwen2.5-Omni-3B,0.04657059956599127,0.11265319373427482,0.49541097564287073,1.0728162054093475,0.273861464154908,0.17795830036014793
+Qwen2.5-Omni-7B,0.04854558310779509,0.12052593133674215,0.6256143590300595,1.1316375158747123,0.34107192365498823,0.36374941455772863
+SALMONN_7B,0.09275107892619414,0.45783621459297136,0.681280039101746,0.7865181254636674,0.37533379054734356,0.25522053004731987
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.07053860970911661,0.3298433568703839,0.2810437993863198,0.4594298934979693,0.21829536997854984,0.17514817745764627
+cascade_whisper_large_v3_llama_3_8b_instruct,0.06922195401458074,0.31912994075156237,0.2770250088250468,0.4581096203900464,0.21391778902978215,0.1722411537654032
+hy_whisper_local_cs,0.06692999780557385,0.2735167600032465,0.25580416542210876,0.3612895924757007,0.186411988735025,0.14417222500363377
+phi_4_multimodal_instruct,0.057615877892375586,0.3451018586153721,0.4381839411301491,1.4697028756805695,0.23859275364433613,0.1439784234241509

results_organized/wer/under_development_wer.csv

@@ -1,14 +1,14 @@
-Model,cna_test,idpc_test,parliament_test,ukusnews_test,mediacorp_test,idpc_short_test,parliament_short_test,ukusnews_short_test,mediacorp_short_test,ytb_asr_batch1,ytb_asr_batch2,seame_dev_man,seame_dev_sge,ytb_asr_batch3_malay,ytb_asr_batch3_ms_ms_prompt,ytb_asr_batch3_chinese,ytb_asr_batch3_zh_zh_prompt
-Qwen-Audio-Chat,0.19753284203780838,0.7710863986313088,0.26279685873781816,0.3158631121194933,0.4498529892192094,0.6008025988916491,0.09347360821020603,0.10399586086125925,0.2548909377108163,0.2297764461857571,0.4315277327278625,0.8783373786407767,1.05567969634822,2.8890790224211313,2.8990790224211313,,
-MERaLiON-AudioLLM-Whisper-SEA-LION,0.15924383210509452,0.30008554319931563,0.058922319992430694,0.12554358101720553,0.170859196341065,0.24918784635964075,0.056935097083623425,0.10144869855926132,0.13301101866426804,0.11484981178458939,0.15162720294085846,0.388282092772384,0.35550521901496834,0.289500241,0.3031898556447721,0.29155272919978803,0.28269210386857446
-hy_whisper_local_cs,0.15710776460536152,0.19863130881094954,0.058638471000094616,0.07199848742673473,0.13124795818360013,0.17638066118861073,0.06559913359634872,0.07828544137546764,0.1154711041151338,0.11546439271721595,0.22990593577684074,0.3134101941747573,0.33199669411368576,,,,
-Qwen2-Audio-7B-Instruct,0.2067713339741536,0.19093242087254064,0.23270886555019396,0.13843826810361126,0.18694870957203527,0.21326199120963119,0.08416492612361723,0.1194380323171217,0.17180121430177647,0.16843358684796805,0.2080008649583739,0.5522518878101402,0.5486546879304539,0.9251458909218551,0.9981132903339037
-whisper_large_v3,0.13841717398269784,0.19880239520958085,0.0753619074652285,0.07135564378899603,0.12054884024828487,0.1662526275558953,0.05543951935226013,0.06168908700151238,0.11715763436024286,0.12226319428439733,0.17210509244242622,0.7225930420711975,0.5377268970583734,0.237374402,0.237374402,0.21278219395866454,0.21278219395866454
-whisper_large_v2,,,,,,,,,,,,,,,,0.2802967673555909,0.2802967673555909
-old_models,,,,,,,,,,,,,,,,,
-cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.15171419416853574,0.16766467065868262,0.06282524363705176,0.07388920400831915,0.12455080039202875,0.16931014714313014,0.07325752301384698,0.06877338215394412,0.14571621317742298,0.1400092187139894,0.2192622950819672,0.7824973031283711,0.5840399155162387,,
-gemini-1.5-flash,,,,,,,,,,0.1089344703080587,,0.9690871089536138,1.1100431601824359,,
-WavLLM_fairseq,0.26946491509131687,0.7686911890504705,0.5216434856656259,0.5911892607298166,0.3595230316889905,0.36728454041658704,0.09512390087929656,0.2066783411605508,0.2621992354396222,0.41876008296842593,0.48091685587631094,1.2913969795037756,1.2204842511249197,,
-SALMONN_7B,0.15395706504325538,0.4550898203592814,0.3010928186204939,0.18918510115333712,0.32089186540346293,0.26313777947639977,0.08676929424202573,0.09042426172092653,0.1751742747919946,0.21487285856956287,0.3238620391393664,1.2721817691477886,1.0189782362484312,,
-cascade_whisper_large_v3_llama_3_8b_instruct,0.13798996048275125,0.17741659538066723,0.07517267480367111,0.07642276422764227,0.13598497223129696,0.15803554366520162,0.05742502771975968,0.0700867627159118,0.11434675061839443,0.12579703464700007,0.23561466104443723,0.6848705501618123,0.507882090054792,,
-Phi4-Multimodal-Instruct,,,,,,,,,,,,,,,,0.3390567037625861,0.21534711181770005
+Model,seame_dev_man,seame_dev_sge,ytb_asr_batch3_ms_ms_prompt,ytb_asr_batch3_zh_zh_prompt
+Qwen-Audio-Chat,0.8783373786407767,1.05567969634822,2.899079022421131,
+MERaLiON-AudioLLM-Whisper-SEA-LION,0.388282092772384,0.3555052190149683,0.3031898556447721,0.2826921038685744
+hy_whisper_local_cs,0.3134101941747573,0.3319966941136857,,
+Qwen2-Audio-7B-Instruct,0.5522518878101402,0.5486546879304539,0.9981132903339036,
+whisper_large_v3,0.7225930420711975,0.5377268970583734,0.237374402,0.2127821939586645
+whisper_large_v2,,,,0.2802967673555909
+old_models,,,,
+cascade_whisper_large_v2_gemma2_9b_cpt_sea_lionv3_instruct,0.7824973031283711,0.5840399155162387,,
+gemini-1.5-flash,0.9690871089536138,1.110043160182436,,
+WavLLM_fairseq,1.2913969795037756,1.2204842511249197,,
+SALMONN_7B,1.2721817691477886,1.0189782362484312,,
+cascade_whisper_large_v3_llama_3_8b_instruct,0.6848705501618123,0.507882090054792,,
+Phi4-Multimodal-Instruct,,,,0.2153471118177