DotDoRAG/eval_results.json at main · franciszekparma/DotDoRAG · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
{
  "dataset": "NFCorpus",
  "split": "TEST",
  "num_queries": 323,
  "num_corpus": 3633,
  "k": 10,
  "metrics": [
    "ndcg@10",
    "recall@10",
    "roc_auc",
    "auc_pr"
  ],
  "results": [
    {
      "method": "BM25",
      "type": "keyword",
      "base_model": null,
      "adapter": null,
      "ndcg@10": 0.3052,
      "recall@10": 0.1474,
      "roc_auc": 0.636,
      "auc_pr": 0.1491
    },
    {
      "method": "plain_bge",
      "type": "dense",
      "base_model": "BAAI/bge-small-en-v1.5",
      "adapter": null,
      "ndcg@10": 0.3456,
      "recall@10": 0.1631,
      "roc_auc": 0.7437,
      "auc_pr": 0.1782
    },
    {
      "method": "bge_lora",
      "type": "dense",
      "base_model": "BAAI/bge-small-en-v1.5",
      "adapter": "bge_lora/adapter_model.safetensors",
      "ndcg@10": 0.3923,
      "recall@10": 0.2013,
      "roc_auc": 0.8353,
      "auc_pr": 0.2401
    },
    {
      "method": "plain_etin",
      "type": "dense",
      "base_model": "jhu-clsp/ettin-encoder-150m",
      "adapter": null,
      "ndcg@10": 0.0115,
      "recall@10": 0.0019,
      "roc_auc": 0.5089,
      "auc_pr": 0.0128
    },
    {
      "method": "etin_lora",
      "type": "dense",
      "base_model": "jhu-clsp/ettin-encoder-150m",
      "adapter": "etin_lora/adapter_model.safetensors",
      "ndcg@10": 0.1549,
      "recall@10": 0.0615,
      "roc_auc": 0.7672,
      "auc_pr": 0.0947
    }
  ],
  "best_by_ndcg@10": "bge_lora",
  "eval_script": "src/eval_compare.py"
}