feat: Update leaderboards

ScandEval · Jan 11, 2025 · d1fac74 · d1fac74
1 parent 06f99f9
commit d1fac74
Show file tree

Hide file tree

Showing 38 changed files with 13,145 additions and 12,505 deletions.
diff --git a/danish-nlg.csv b/danish-nlg.csv
diff --git a/danish-nlg.md b/danish-nlg.md
diff --git a/danish-nlu.csv b/danish-nlu.csv
diff --git a/danish-nlu.md b/danish-nlu.md
diff --git a/dutch-nlg.csv b/dutch-nlg.csv
diff --git a/dutch-nlg.md b/dutch-nlg.md
diff --git a/dutch-nlu.csv b/dutch-nlu.csv
diff --git a/dutch-nlu.md b/dutch-nlu.md
diff --git a/english-nlg.csv b/english-nlg.csv
diff --git a/english-nlg.md b/english-nlg.md
diff --git a/english-nlu.csv b/english-nlu.csv
diff --git a/english-nlu.md b/english-nlu.md
diff --git a/faroese-nlu.csv b/faroese-nlu.csv
diff --git a/faroese-nlu.md b/faroese-nlu.md
diff --git a/german-nlg.csv b/german-nlg.csv
diff --git a/german-nlg.md b/german-nlg.md
diff --git a/german-nlu.csv b/german-nlu.csv
diff --git a/german-nlu.md b/german-nlu.md
diff --git a/germanic-nlg.csv b/germanic-nlg.csv
diff --git a/germanic-nlg.md b/germanic-nlg.md
diff --git a/germanic-nlu.csv b/germanic-nlu.csv
diff --git a/germanic-nlu.md b/germanic-nlu.md
diff --git a/icelandic-nlg.csv b/icelandic-nlg.csv
diff --git a/icelandic-nlg.md b/icelandic-nlg.md
diff --git a/icelandic-nlu.csv b/icelandic-nlu.csv
@@ -7,7 +7,7 @@ vesteinn/ScandiBERT-no-faroese,124,50,514,True,False,15436,1.48,83.94,48.51,58.6
 vesteinn/XLMR-ENIS,125,50,514,True,False,10711,1.52,82.2,49.16,48.51,27.06
 google/rembert,576,250,512,True,False,11736,1.67,78.05,36.87,48.29,29.38
 mideind/IceBERT-large,406,50,514,True,False,5677,1.73,85.14,49.4,59.31,12.84
-vesteinn/FoBERT,124,50,514,True,False,15623,1.76,85.04,48.45,50.78,17.76
+vesteinn/FoBERT,124,50,514,True,False,15623,1.77,85.04,48.45,50.78,17.76
 mideind/IceBERT,163,50,514,True,False,16697,1.78,85.32,47.49,60.44,13.31
 "claude-3-5-sonnet-20241022 (zero-shot, val)",-1,-1,200000,True,False,193,1.8,61.7,51.24,52.43,22.92
 mideind/IceBERT-xlmr-ic3,278,250,514,True,False,11004,1.83,84.35,48.85,59.12,11.18
@@ -36,12 +36,12 @@ nvidia/Llama-3.1-Nemotron-70B-Instruct-HF (few-shot),70554,128,131072,True,False
 CohereForAI/c4ai-command-r-08-2024 (few-shot),32296,256,131072,False,False,1909,2.46,62.68,31.96,11.81,30.49
 mistralai/Mixtral-8x7B-v0.1 (few-shot),46703,32,32768,True,False,2363,2.46,55.09,39.6,8.23,30.78
 meta-llama/Llama-3.1-8B (few-shot),8030,128,131072,True,False,2986,2.48,52.97,41.29,5.95,31.99
-meta-llama/Meta-Llama-3-8B-Instruct (few-shot),8030,128,8192,True,False,1007,2.53,60.2,38.09,9.14,28.66
+meta-llama/Meta-Llama-3-8B-Instruct (few-shot),8030,128,8192,True,False,1483,2.53,60.2,38.09,9.14,28.66
 intfloat/multilingual-e5-large,560,250,514,True,False,6732,2.57,78.43,48.52,10.78,13.79
-meta-llama/Meta-Llama-3-8B (few-shot),8030,128,8192,True,False,1335,2.58,50.45,34.68,8.69,31.94
+meta-llama/Meta-Llama-3-8B (few-shot),8030,128,8192,True,False,1477,2.58,50.45,34.68,8.69,31.94
 intfloat/multilingual-e5-large-instruct,560,250,514,True,False,5947,2.59,78.32,50.1,8.11,13.93
 intfloat/multilingual-e5-base,278,250,514,True,False,14965,2.63,75.46,42.08,15.21,10.82
-meta-llama/Llama-3.1-8B-Instruct (few-shot),8030,128,131072,True,False,1005,2.66,46.48,39.91,11.72,25.91
+meta-llama/Llama-3.1-8B-Instruct (few-shot),8030,128,131072,True,False,1473,2.66,46.48,39.91,11.72,25.91
 cardiffnlp/twitter-xlm-roberta-base,278,250,514,True,False,34475,2.72,72.69,35.62,28.72,8.46
 clips/mfaq,278,250,514,True,False,5591,2.85,77.3,43.25,3.51,11.31
 microsoft/xlm-align-base,278,250,514,True,False,14744,2.86,78.01,38.76,5.92,10.47
@@ -57,8 +57,8 @@ NorwAI/NorwAI-Mixtral-8x7B (few-shot),46998,68,32768,True,False,2368,2.95,36.73,
 Twitter/twhin-bert-base,279,250,512,True,False,11514,2.96,70.38,40.22,11.09,7.67
 Geotrend/bert-base-25lang-cased,151,85,512,True,False,13908,3.03,74.65,34.09,2.89,9.29
 CohereForAI/c4ai-command-r-v01 (few-shot),34981,256,8192,False,False,1919,3.05,42.29,38.87,0.28,18.74
-mistralai/Mistral-7B-v0.3 (few-shot),7248,33,32768,True,False,1364,3.06,46.73,26.28,1.5,25.17
 microsoft/infoxlm-base,278,250,514,True,False,34735,3.07,77.09,33.14,1.71,8.56
+mistralai/Mistral-7B-v0.3 (few-shot),7248,33,32768,True,False,1364,3.07,46.73,26.28,1.5,25.17
 KennethEnevoldsen/dfm-sentence-encoder-medium,124,50,514,True,False,14998,3.08,64.88,36.18,-0.6,12.39
 Nexusflow/Starling-LM-7B-beta (few-shot),7242,32,4096,False,False,4136,3.09,42.23,27.93,6.38,19.39
 sentence-transformers/stsb-xlm-r-multilingual,278,250,514,True,False,15040,3.09,66.23,37.79,0.04,10.04
@@ -89,9 +89,9 @@ DeepPavlov/rubert-base-cased,178,120,512,True,False,15785,3.34,61.95,29.51,2.4,6
 "claude-3-5-haiku-20241022 (zero-shot, val)",-1,-1,200000,True,False,277,3.34,34.99,31.19,-10.68,23.65
 sentence-transformers/distilbert-multilingual-nli-stsb-quora-ranking,135,120,512,True,False,33753,3.34,59.15,33.1,0.8,6.14
 Geotrend/distilbert-base-en-no-cased,69,33,512,True,False,26597,3.35,63.84,29.48,2.15,5.23
-KBLab/megatron-bert-large-swedish-cased-110k,370,64,512,True,False,7075,3.36,63.11,26.38,3.47,7.76
 NbAiLab/nb-llama-3.1-70B (few-shot),70554,128,131072,True,False,1220,3.36,62.28,38.08,2.85,0.86
 ibm-granite/granite-3.0-8b-instruct (few-shot),8171,49,4096,True,False,1118,3.36,42.67,9.95,1.11,22.25
+KBLab/megatron-bert-large-swedish-cased-110k,370,64,512,True,False,7075,3.37,63.11,26.38,3.47,7.76
 occiglot/occiglot-7b-eu5-instruct (few-shot),7242,32,4096,False,False,2088,3.37,40.71,14.7,0.71,20.66
 ltg/norbert3-base,124,50,512,True,False,11405,3.38,68.22,34.62,2.41,0.0
 DDSC/roberta-base-scandinavian,125,50,514,True,False,14491,3.4,51.53,34.36,0.89,5.19
@@ -113,7 +113,7 @@ pdelobelle/robbert-v2-dutch-base,117,40,514,True,False,15481,3.46,55.54,28.28,1.
 FacebookAI/roberta-base,125,50,514,True,False,13354,3.49,60.18,22.99,1.07,6.66
 google/gemma-2-2b-it (few-shot),2614,256,4096,True,False,5374,3.49,14.79,15.24,1.1,25.42
 meta-llama/Llama-2-7b-hf (few-shot),6738,32,4096,True,False,930,3.5,32.71,13.24,0.66,18.04
-microsoft/Phi-3-mini-4k-instruct (few-shot),3821,32,2047,True,False,3194,3.5,33.05,14.42,0.71,17.23
+microsoft/Phi-3-mini-4k-instruct (few-shot),3821,32,2047,True,False,8681,3.5,33.05,14.42,0.71,17.23
 sentence-transformers/distiluse-base-multilingual-cased-v2,135,120,512,True,False,33247,3.51,48.62,34.19,2.64,1.22
 01-ai/Yi-1.5-6B (few-shot),6061,64,4096,True,False,2867,3.53,38.15,5.32,0.98,20.39
 bineric/NorskGPT-Llama-7B-v0.1 (few-shot),6738,32,4096,False,False,5384,3.53,34.62,12.72,-0.24,18.1
@@ -148,7 +148,7 @@ Maltehb/aelaectra-danish-electra-small-cased,14,32,512,True,False,4593,3.81,35.7
 sarnikowski/convbert-small-da-cased,13,29,512,True,False,14273,3.83,25.49,24.23,1.63,5.28
 timpal0l/Mistral-7B-v0.1-flashback-v2-instruct (few-shot),7242,32,4096,False,False,5172,3.83,24.98,13.57,1.18,8.52
 nvidia/mistral-nemo-minitron-8b-instruct (few-shot),8414,131,8192,True,False,3161,3.84,43.65,10.7,10.77,0.29
-AI-Sweden-Models/gpt-sw3-1.3b (few-shot),1445,64,2048,True,False,4608,3.87,1.42,4.18,0.75,23.33
+AI-Sweden-Models/gpt-sw3-1.3b (few-shot),1445,64,2048,True,False,4608,3.86,1.42,4.18,0.75,23.33
 Maltehb/aelaectra-danish-electra-small-uncased,14,32,512,True,False,5995,3.87,30.5,25.53,3.59,0.06
 dbmdz/bert-tiny-historic-multilingual-cased,5,32,512,True,False,78027,3.87,43.93,9.46,0.04,6.13
 Qwen/Qwen1.5-4B (few-shot),3950,152,32768,True,False,3248,3.88,15.66,9.17,-0.55,14.11
@@ -158,15 +158,15 @@ ibm-granite/granite-3.0-3b-a800m-instruct (few-shot),3374,49,4096,True,False,102
 HuggingFaceTB/SmolLM2-1.7B (few-shot),1711,49,8192,True,False,16249,3.9,20.5,10.09,0.83,10.84
 HuggingFaceTB/SmolLM2-1.7B-Instruct (few-shot),1711,49,8192,True,False,15971,3.91,26.23,6.86,2.69,10.84
 ibm-granite/granite-3.0-2b-base (few-shot),2534,49,4096,True,False,10187,3.93,29.51,1.7,-0.32,12.36
+openGPT-X/Teuken-7B-instruct-commercial-v0.4 (few-shot),7453,251,4096,True,False,1438,3.93,26.58,-0.79,0.63,15.14
 ibm-granite/granite-3b-code-instruct-2k (few-shot),3483,49,2048,True,False,9059,3.94,33.57,0.6,0.0,11.27
 ibm-granite/granite-7b-base (few-shot),6738,32,4096,True,False,4405,3.94,28.06,4.92,0.1,11.07
-openGPT-X/Teuken-7B-instruct-commercial-v0.4 (few-shot),7453,251,4096,True,False,1438,3.94,26.58,-0.79,0.63,15.14
 stabilityai/stablelm-2-1_6b (few-shot),1645,100,4096,True,False,7259,3.97,32.19,8.99,0.37,6.58
 alexanderfalk/danbert-small-cased,83,52,514,True,False,30013,3.98,12.39,29.54,1.63,1.7
 Maltehb/danish-bert-botxo,111,32,512,True,False,16091,4.03,12.64,22.02,0.06,4.77
+fresh-xlm-roberta-base,278,250,514,True,False,2214,4.06,17.34,25.25,-0.06,1.02
 ibm-granite/granite-3.0-3b-a800m-base (few-shot),3374,49,4096,True,False,10504,4.06,18.07,0.65,-0.72,12.27
 fresh-electra-small,14,31,512,True,False,7840,4.07,9.96,30.18,-0.1,0.12
-fresh-xlm-roberta-base,278,250,514,True,False,2214,4.07,17.34,25.25,-0.06,1.02
 PleIAs/Pleias-1.2b-Preview (few-shot),1195,66,2048,True,False,10756,4.08,22.56,0.53,-0.26,11.77
 meta-llama/Llama-3.2-1B (few-shot),1236,128,131072,True,False,7577,4.08,17.77,7.64,-0.35,8.15
 google/gemma-2b-it (few-shot),2506,256,8192,False,False,6471,4.1,20.49,-0.75,-0.01,10.95
@@ -181,14 +181,15 @@ PleIAs/Pleias-3b-Preview (few-shot),3212,66,4096,True,False,6513,4.24,18.86,-0.6
 state-spaces/mamba-2.8b-hf (few-shot),2768,50,-1,True,False,2722,4.25,18.38,-1.7,0.49,6.3
 HuggingFaceTB/SmolLM2-360M (few-shot),362,49,8192,True,False,22023,4.27,13.43,3.82,1.14,3.71
 HuggingFaceTB/SmolLM2-360M-Instruct (few-shot),362,49,8192,True,False,21777,4.29,13.6,3.12,0.28,4.09
-PleIAs/Pleias-350m-Preview (few-shot),353,66,2048,True,False,10242,4.34,17.73,2.38,-0.18,1.59
+PleIAs/Pleias-350m-Preview (few-shot),353,66,2048,True,False,10242,4.33,17.73,2.38,-0.18,1.59
 Qwen/Qwen1.5-0.5B (few-shot),620,152,32768,True,False,11371,4.36,16.2,0.1,-0.57,3.31
 ibm-granite/granite-3.0-1b-a400m-base (few-shot),1385,49,4096,True,False,7808,4.36,5.62,4.82,-0.2,4.94
 HuggingFaceTB/SmolLM2-135M (few-shot),135,49,8192,True,False,26346,4.37,14.74,3.13,-0.25,1.35
+tiiuae/Falcon3-1B-Instruct (few-shot),1669,131,8192,True,False,9270,4.37,9.39,6.44,-0.72,3.34
 PleIAs/Pleias-Pico (few-shot),353,66,2048,True,False,2331,4.38,13.8,2.17,-0.63,1.29
 HuggingFaceTB/SmolLM2-135M-Instruct (few-shot),135,49,8192,True,False,25602,4.4,13.7,3.01,-0.83,0.94
 Qwen/Qwen1.5-0.5B-Chat (few-shot),620,152,32768,False,False,11740,4.41,9.5,1.63,1.76,3.14
-RuterNorway/Llama-2-7b-chat-norwegian (few-shot),6738,32,4096,False,False,10890,4.42,9.48,3.32,0.07,1.04
+RuterNorway/Llama-2-7b-chat-norwegian (few-shot),6738,32,4096,False,False,10890,4.41,9.48,3.32,0.07,1.04
 Qwen/Qwen1.5-1.8B (few-shot),1837,152,32768,True,False,5666,4.44,12.26,-9.69,0.94,6.31
 NorwAI/NorwAI-Mistral-7B-pretrain (few-shot),7537,68,4096,True,False,3024,4.45,3.69,5.91,1.24,0.29
 RJuro/kanelsnegl-v0.1 (few-shot),7242,32,4096,True,False,5847,4.49,0.0,10.54,0.0,0.0