Skip to content

Commit

Permalink
updated evaluation table results generation
Browse files Browse the repository at this point in the history
  • Loading branch information
@alkidbaci
alkidbaci committed Jan 2, 2025
1 parent f3061b9 commit 77d2bb8
Show file tree
Hide file tree
Showing 2 changed files with 160 additions and 50 deletions.
160 changes: 160 additions & 0 deletions general_working_directory/[email protected]
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
import json


def find_mrr(data):
rr = 0
n = len(data)
for i, v in data.items():
rr += 1 / (v + 1)
mrr = rr / n
return mrr


def find_hit_k(data, k):
count = 0
n = len(data)
for i, v in data.items():
if v + 1 < k:
count += 1
return count / n


k1 = 10
k2 = 20
k3 = 50
k4 = 100

print("==== embedding-based model b1====/n")
with open('evaluation_results_embedding-based_on_second_KG.json', 'r') as file:
data = json.load(file)

print(f"Second KG MRR = {find_mrr(data)}")
print(f"Second KG Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Second KG Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Second KG Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Second KG Hits@{k4} = {find_hit_k(data, k4)}")

with open('evaluation_results_embedding-based_on_third_KG.json', 'r') as file:
data = json.load(file)

print(f"Third KG MRR = {find_mrr(data)}")
print(f"Third KG Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Third KG Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Third KG Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Third KG Hits@{k4} = {find_hit_k(data, k4)}")

print("==== BM25 model b1====/n")
with open('evaluation_results_bm25_on_second_KG.json', 'r') as file:
data = json.load(file)

print(f"Second KG MRR = {find_mrr(data)}")
print(f"Second KG Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Second KG Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Second KG Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Second KG Hits@{k4} = {find_hit_k(data, k4)}")

with open('evaluation_results_bm25_on_third_KG.json', 'r') as file:
data = json.load(file)

print(f"Third KG MRR = {find_mrr(data)}")
print(f"Third KG Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Third KG Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Third KG Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Third KG Hits@{k4} = {find_hit_k(data, k4)}")

print("==== embedding-based model b2====/n")
with open('evaluation_results2_embedding-based_on_second_KG.json', 'r') as file:
data = json.load(file)

print(f"Second KG MRR = {find_mrr(data)}")
print(f"Second KG Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Second KG Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Second KG Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Second KG Hits@{k4} = {find_hit_k(data, k4)}")

with open('evaluation_results2_embedding-based_on_third_KG.json', 'r') as file:
data = json.load(file)

print(f"Third KG MRR = {find_mrr(data)}")
print(f"Third KG Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Third KG Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Third KG Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Third KG Hits@{k4} = {find_hit_k(data, k4)}")

print("==== BM25 model b2====/n")
with open('evaluation_results2_bm25_on_second_KG.json', 'r') as file:
data = json.load(file)

print(f"Second KG MRR = {find_mrr(data)}")
print(f"Second KG Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Second KG Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Second KG Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Second KG Hits@{k4} = {find_hit_k(data, k4)}")

with open('evaluation_results2_bm25_on_third_KG.json', 'r') as file:
data = json.load(file)

print(f"Third KG MRR = {find_mrr(data)}")
print(f"Third KG Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Third KG Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Third KG Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Third KG Hits@{k4} = {find_hit_k(data, k4)}")

print("==== CEL-based 20-20 lp ====/n")
with open('rag2_eval_second_kg.json', 'r') as file:
data = json.load(file)

print(f"Second KG + CEL 20-20 MRR = {find_mrr(data)}")
print(f"Second KG + CEL 20-20 Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Second KG + CEL 20-20 Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Second KG + CEL 20-20 Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Second KG + CEL 20-20 Hits@{k4} = {find_hit_k(data, k4)}")

with open('rag2_eval_third_kg.json', 'r') as file:
data = json.load(file)

print(f"Third KG + CEL 20-20 MRR = {find_mrr(data)}")
print(f"Third KG + CEL 20-20 Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Third KG + CEL 20-20 Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Third KG + CEL 20-20 Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Third KG + CEL 20-20 Hits@{k4} = {find_hit_k(data, k4)}")

print("==== CEL-based 10-10 lp ====/n")

with open('rag2_eval_second_kg-10-10.json', 'r') as file:
data = json.load(file)

print(f"Second KG + CEL 10-10 MRR = {find_mrr(data)}")
print(f"Second KG + CEL 10-10 Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Second KG + CEL 10-10 Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Second KG + CEL 10-10 Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Second KG + CEL 10-10 Hits@{k4} = {find_hit_k(data, k4)}")

with open('rag2_eval_third_kg-10-10.json', 'r') as file:
data = json.load(file)

print(f"Third KG + CEL 10-10 MRR = {find_mrr(data)}")
print(f"Third KG + CEL 10-10 Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Third KG + CEL 10-10 Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Third KG + CEL 10-10 Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Third KG + CEL 10-10 Hits@{k4} = {find_hit_k(data, k4)}")

print("==== CEL-based 5-5 lp ====/n")

with open('rag2_eval_second_kg-5-5.json', 'r') as file:
data = json.load(file)

print(f"Second KG + CEL 5-5 MRR = {find_mrr(data)}")
print(f"Second KG + CEL 5-5 Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Second KG + CEL 5-5 Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Second KG + CEL 5-5 Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Second KG + CEL 5-5 Hits@{k4} = {find_hit_k(data, k4)}")

with open('rag2_eval_third_kg-5-5.json', 'r') as file:
data = json.load(file)

print(f"Third KG + CEL 5-5 MRR = {find_mrr(data)}")
print(f"Third KG + CEL 5-5 Hits@{k1} = {find_hit_k(data, k1)}")
print(f"Third KG + CEL 5-5 Hits@{k2} = {find_hit_k(data, k2)}")
print(f"Third KG + CEL 5-5 Hits@{k3} = {find_hit_k(data, k3)}")
print(f"Third KG + CEL 5-5 Hits@{k4} = {find_hit_k(data, k4)}")
50 changes: 0 additions & 50 deletions general_working_directory/retriever_eval_metrics.py
View file

This file was deleted.

0 comments on commit 77d2bb8

Please sign in to comment.