diff --git a/evaluation_results_embedding-based_on_second_KG.json b/evaluation_results_embedding-based_on_second_KG.json new file mode 100644 index 0000000..1a0d352 --- /dev/null +++ b/evaluation_results_embedding-based_on_second_KG.json @@ -0,0 +1 @@ +{"http://example.org/image_1082": 7010, "http://example.org/image_42527": 189, "http://example.org/image_39373": 7964, "http://example.org/image_34524": 1656, "http://example.org/image_38547": 7112, "http://example.org/image_21806": 14874, "http://example.org/image_41097": 2648, "http://example.org/image_5434": 957, "http://example.org/image_889": 170, "http://example.org/image_48024": 817, "http://example.org/image_23951": 20568, "http://example.org/image_34371": 1996, "http://example.org/image_26553": 28, "http://example.org/image_39479": 1271, "http://example.org/image_17925": 18811, "http://example.org/image_24171": 3633, "http://example.org/image_6829": 381, "http://example.org/image_4014": 43, "http://example.org/image_36802": 1333, "http://example.org/image_4174": 32296, "http://example.org/image_30775": 1389, "http://example.org/image_32118": 59, "http://example.org/image_39494": 2616, "http://example.org/image_1069": 9116, "http://example.org/image_30081": 3588, "http://example.org/image_14081": 9811, "http://example.org/image_8082": 5511, "http://example.org/image_21051": 4020, "http://example.org/image_49508": 1615, "http://example.org/image_14325": 72, "http://example.org/image_1352": 8541, "http://example.org/image_81": 58, "http://example.org/image_6051": 19945, "http://example.org/image_37671": 342, "http://example.org/image_15166": 25485, "http://example.org/image_14193": 188, "http://example.org/image_49546": 9, "http://example.org/image_12045": 6938, "http://example.org/image_6720": 19181, "http://example.org/image_33263": 4461, "http://example.org/image_2460": 950, "http://example.org/image_26441": 272, "http://example.org/image_20880": 3777, "http://example.org/image_2418": 1652, "http://example.org/image_47279": 5054, "http://example.org/image_45305": 1357, "http://example.org/image_27640": 449, "http://example.org/image_19870": 4992, "http://example.org/image_11359": 2854, "http://example.org/image_276": 20668, "http://example.org/image_30229": 5518, "http://example.org/image_967": 428, "http://example.org/image_26357": 6439, "http://example.org/image_2535": 4912, "http://example.org/image_11368": 792, "http://example.org/image_21672": 2644, "http://example.org/image_10979": 467, "http://example.org/image_37384": 1394, "http://example.org/image_204": 14, "http://example.org/image_12907": 3475, "http://example.org/image_48528": 20339, "http://example.org/image_34867": 24713, "http://example.org/image_32243": 6823, "http://example.org/image_29033": 1769, "http://example.org/image_31567": 347, "http://example.org/image_11241": 15659, "http://example.org/image_32496": 2019, "http://example.org/image_36098": 7014, "http://example.org/image_9747": 4483, "http://example.org/image_42949": 6337, "http://example.org/image_31097": 2672, "http://example.org/image_27229": 1841, "http://example.org/image_26932": 15593, "http://example.org/image_16304": 10407, "http://example.org/image_40627": 19, "http://example.org/image_44639": 9379, "http://example.org/image_39575": 18836, "http://example.org/image_3500": 32855, "http://example.org/image_50164": 34306, "http://example.org/image_6129": 1580, "http://example.org/image_44775": 4527, "http://example.org/image_39515": 277, "http://example.org/image_45006": 11, "http://example.org/image_36685": 18324, "http://example.org/image_11571": 7255, "http://example.org/image_28528": 276, "http://example.org/image_50016": 308, "http://example.org/image_29005": 599, "http://example.org/image_14200": 1507, "http://example.org/image_30547": 266, "http://example.org/image_43170": 501, "http://example.org/image_49984": 1913, "http://example.org/image_45098": 95, "http://example.org/image_50336": 101, "http://example.org/image_37321": 523, "http://example.org/image_44497": 56, "http://example.org/image_16047": 1620, "http://example.org/image_22278": 6312, "http://example.org/image_22395": 335, "http://example.org/image_15494": 3594} \ No newline at end of file diff --git a/evaluation_results_embedding-based_on_third_KG.json b/evaluation_results_embedding-based_on_third_KG.json new file mode 100644 index 0000000..fb8877a --- /dev/null +++ b/evaluation_results_embedding-based_on_third_KG.json @@ -0,0 +1 @@ +{"http://example.org/image_1082": 4802, "http://example.org/image_42527": 421, "http://example.org/image_39373": 5083, "http://example.org/image_34524": 1855, "http://example.org/image_38547": 9275, "http://example.org/image_21806": 1577, "http://example.org/image_41097": 1810, "http://example.org/image_5434": 179, "http://example.org/image_889": 315, "http://example.org/image_48024": 7, "http://example.org/image_23951": 5417, "http://example.org/image_34371": 59, "http://example.org/image_26553": 15, "http://example.org/image_39479": 21, "http://example.org/image_17925": 3, "http://example.org/image_24171": 6480, "http://example.org/image_6829": 369, "http://example.org/image_4014": 22, "http://example.org/image_36802": 224, "http://example.org/image_4174": 126, "http://example.org/image_30775": 2, "http://example.org/image_32118": 1, "http://example.org/image_39494": 341, "http://example.org/image_1069": 2560, "http://example.org/image_30081": 12, "http://example.org/image_14081": 887, "http://example.org/image_8082": 90, "http://example.org/image_21051": 7355, "http://example.org/image_49508": 146, "http://example.org/image_14325": 30063, "http://example.org/image_1352": 4584, "http://example.org/image_81": 0, "http://example.org/image_6051": 6974, "http://example.org/image_37671": 1074, "http://example.org/image_15166": 8996, "http://example.org/image_14193": 2729, "http://example.org/image_49546": 8, "http://example.org/image_12045": 99, "http://example.org/image_6720": 6535, "http://example.org/image_33263": 3055, "http://example.org/image_2460": 2432, "http://example.org/image_26441": 3, "http://example.org/image_20880": 8992, "http://example.org/image_2418": 9106, "http://example.org/image_47279": 40, "http://example.org/image_45305": 103, "http://example.org/image_27640": 2, "http://example.org/image_19870": 17105, "http://example.org/image_11359": 433, "http://example.org/image_276": 24410, "http://example.org/image_30229": 193, "http://example.org/image_967": 0, "http://example.org/image_26357": 1710, "http://example.org/image_2535": 2318, "http://example.org/image_11368": 2071, "http://example.org/image_21672": 182, "http://example.org/image_10979": 3, "http://example.org/image_37384": 364, "http://example.org/image_204": 519, "http://example.org/image_12907": 1282, "http://example.org/image_48528": 420, "http://example.org/image_34867": 6219, "http://example.org/image_32243": 159, "http://example.org/image_29033": 687, "http://example.org/image_31567": 5, "http://example.org/image_11241": 1345, "http://example.org/image_32496": 591, "http://example.org/image_36098": 13, "http://example.org/image_9747": 535, "http://example.org/image_42949": 53, "http://example.org/image_31097": 1628, "http://example.org/image_27229": 9, "http://example.org/image_26932": 7692, "http://example.org/image_16304": 665, "http://example.org/image_40627": 1, "http://example.org/image_44639": 391, "http://example.org/image_39575": 12026, "http://example.org/image_3500": 351, "http://example.org/image_50164": 3636, "http://example.org/image_6129": 983, "http://example.org/image_44775": 43, "http://example.org/image_39515": 14, "http://example.org/image_45006": 20, "http://example.org/image_36685": 2505, "http://example.org/image_11571": 385, "http://example.org/image_28528": 301, "http://example.org/image_50016": 89, "http://example.org/image_29005": 0, "http://example.org/image_14200": 71, "http://example.org/image_30547": 11899, "http://example.org/image_43170": 0, "http://example.org/image_49984": 878, "http://example.org/image_45098": 33, "http://example.org/image_50336": 77, "http://example.org/image_37321": 3, "http://example.org/image_44497": 21, "http://example.org/image_16047": 6, "http://example.org/image_22278": 2124, "http://example.org/image_22395": 383, "http://example.org/image_15494": 1647} \ No newline at end of file diff --git a/general_working_directory/bm25_mmr_calculator.py b/general_working_directory/bm25_mmr_calculator.py deleted file mode 100644 index 867ee31..0000000 --- a/general_working_directory/bm25_mmr_calculator.py +++ /dev/null @@ -1,21 +0,0 @@ -import json - - -def find_mrr(data): - rr = 0 - n = len(data) - for i, v in data.items(): - rr += 1 / (v + 1) - mrr = rr / n - return mrr - - -with open('../evaluation_results_bm25_on_third_KG.json', 'r') as file: - data = json.load(file) - -print(f"Third KG MMR = {find_mrr(data)}") - -with open('../evaluation_results_bm25_on_second_KG.json', 'r') as file: - data = json.load(file) - -print(f"Second KG MMR = {find_mrr(data)}") \ No newline at end of file diff --git a/general_working_directory/embedding-based_retriever_evaluation.py b/general_working_directory/embedding-based_retriever_evaluation.py new file mode 100644 index 0000000..e5d1164 --- /dev/null +++ b/general_working_directory/embedding-based_retriever_evaluation.py @@ -0,0 +1,143 @@ +import json +from openai import OpenAI +import pandas as pd +import numpy as np +from numpy.linalg import norm + +evaluation_samples = { + "http://example.org/image_1082": "What outfits feature black shorts ?", + "http://example.org/image_42527": "What are some jumpsuits with a high waist ?", + "http://example.org/image_39373": "What are some dress options that feature a round neckline ?", + "http://example.org/image_34524": "What are some short-sleeved tops ?", + "http://example.org/image_38547": "What are some fashionable dress options with a high waist ?", + "http://example.org/image_21806": "What casual dress styles with short sleeves and round necklines are available for warm weather ?", + "http://example.org/image_41097": "What formal dresses have a high-low silhouette ?", + "http://example.org/image_5434": "Featuring a black sleeveless top with a scoop neckline, what would you recommend ?", + "http://example.org/image_889": "What are some outerwear options that feature a fur collar ?", + "http://example.org/image_48024": "What are some light pink knitted sweaters ?", + "http://example.org/image_23951": "What are some fashionable outfits that include a red skirt that reaches just above the knees ?", + "http://example.org/image_34371": "Can you recommend any dresses with a heavy, possibly satin or silk fabric ?", + "http://example.org/image_26553": "What are some fashionable dress options with a floral print ?", + "http://example.org/image_39479": "What are some clothing options that include a black tank top with a unique graphic design on the front ?", + "http://example.org/image_17925": "What are some outfits that include a black mesh skirt ?", + "http://example.org/image_24171": "What are some dresses with a fitted bodice ?", + "http://example.org/image_6829": "What casual and comfortable outfit features blue denim jeans with a faded wash ?", + "http://example.org/image_4014": "What are some dress shirts without any special manufacturing techniques or non-textile materials ?", + "http://example.org/image_36802": "What are some clothing options that include a watch on the left wrist ?", + "http://example.org/image_4174": "What fashionable ensemble would you recommend for a high-waisted, gold embellished skirt ?", + "http://example.org/image_30775": "What are some fashionable outfit options for a slim-fit light blue suit ?", + "http://example.org/image_32118": "What is a leopard print blouse with a high neckline ?", + "http://example.org/image_39494": "What casual yet stylish outfits would you recommend featuring orange pants with a straight cut and a flat front ?", + "http://example.org/image_1069": "Dark blue jeans, what would you recommend ?", + "http://example.org/image_30081": "What light blue button-up shirts are available ?", + "http://example.org/image_14081": "What are some outfits featuring sandals with a strap across the foot ?", + "http://example.org/image_8082": "What are some leather outfits with a belted waist ?", + "http://example.org/image_21051": "What are some outfits that include a black belt ?", + "http://example.org/image_49508": "What are some long-sleeved sweatshirts ?", + "http://example.org/image_14325": "What are some jumpsuits with symmetrical silhouette ?", + "http://example.org/image_1352": "What are some dress options suitable for daytime events ?", + "http://example.org/image_81": "What are some outfits that include a black leather corset with a lace-up front ?", + "http://example.org/image_6051": "What are some clothing options that include black high heels ?", + "http://example.org/image_37671": "What are some clothing options that include a single-breasted blazer ?", + "http://example.org/image_15166": "Dark shoes or boots, what would you recommend ?", + "http://example.org/image_14193": "What are some wholebody dresses with a sequin finish ?", + "http://example.org/image_49546": "What are some upperbody garments with a plain pattern ?", + "http://example.org/image_12045": "What are some outfits suitable for a fashion show or similar event ?", + "http://example.org/image_6720": "What fashion ensemble would you recommend for a formal event ?", + "http://example.org/image_33263": "What are some wholebody dresses with a classic and elegant style ?", + "http://example.org/image_2460": "What are some clothing options that have a slash pocket ?", + "http://example.org/image_26441": "What casual hoodies are available ?", + "http://example.org/image_20880": "What are some fashionable and coordinated outfits that include a black handbag ?", + "http://example.org/image_2418": "High-heeled sandals with a strappy design ?", + "http://example.org/image_47279": "What casual and comfortable outfits include a bright red long-sleeved sweater ?", + "http://example.org/image_45305": "What are some edgy t-shirts ?", + "http://example.org/image_27640": "What are some t-shirts with a floral print ?", + "http://example.org/image_19870": "What are some attire options that pair with a dark shirt and brown leather shoes ?", + "http://example.org/image_11359": "What are some fashionable outfits that include a black skirt that reaches just above the knee ?", + "http://example.org/image_276": "What are some options made of a smooth fabric ?", + "http://example.org/image_30229": "What casual spring or summer outfits would you recommend for a person with a set-in sleeve ?", + "http://example.org/image_967": "What type of casual athletic outfit would be suitable for a workout setting ?", + "http://example.org/image_26357": "What are some tops made of lightweight fabric ?", + "http://example.org/image_2535": "What casual, urban-style outfits would you recommend for a relaxed setting ?", + "http://example.org/image_11368": "What are some casual and relaxed outfits suitable for a warm day outdoors ?", + "http://example.org/image_21672": "What type of formal wear dresses would you recommend for events like weddings, galas, or high-profile social gatherings ?", + "http://example.org/image_10979": "What are some jumpsuits with a bold, patterned design ?", + "http://example.org/image_37384": "What are some evening gowns with a fitted bodice ?", + "http://example.org/image_204": "What are some outfits featuring black high-waisted pants with a fly opening and peg silhouette ?", + "http://example.org/image_12907": "What are some outfit recommendations that include black shorts with a smooth finish ?", + "http://example.org/image_48528": "What traditional-style, long-sleeved jackets are available online ?", + "http://example.org/image_34867": "What dresses could be paired with black high heels ?", + "http://example.org/image_32243": "What casual, summery outfits with a polka dot pattern would you recommend for a warm day out ?", + "http://example.org/image_29033": "What formal or evening gowns have a fitted bodice with a sweetheart neckline ?", + "http://example.org/image_31567": "What are some navy blue cardigans with a V-neckline ?", + "http://example.org/image_11241": "What casual, summery outfits with a flared skirt and a fitted top would complement a red headband ?", + "http://example.org/image_32496": "What formal or semi-formal dresses with a sleeveless design would you recommend for an occasion ?", + "http://example.org/image_36098": "What are some formal dress options that feature a deep navy color ?", + "http://example.org/image_9747": "What type of sneakers would you recommend for a casual, summery outfit ?", + "http://example.org/image_42949": "What are some black jackets with a classic design ?", + "http://example.org/image_31097": "What are some dress options made of satin or similar smooth fabric ?", + "http://example.org/image_27229": "What are some fashionable denim jumpsuits ?", + "http://example.org/image_26932": "What are some fashionable and stylish dress options for a formal or semi-formal event ?", + "http://example.org/image_16304": "What color pattern would you recommend for a runway show ?", + "http://example.org/image_40627": "What casual pants with a straight cut would you recommend ?", + "http://example.org/image_44639": "What are some fashionable dresses with a bow neckline ?", + "http://example.org/image_39575": "What are some dress options that are suitable for smart-casual wardrobes ?", + "http://example.org/image_3500": "What are some clothing options that include a black and white patterned top with a round neckline ?", + "http://example.org/image_50164": "What color trousers would complement a classic tuxedo ?", + "http://example.org/image_6129": "What sunglasses would you recommend for a chic, bohemian look ?", + "http://example.org/image_44775": "Can you recommend any dresses with a coral or peach color and a sparkling, embellished look ?", + "http://example.org/image_39515": "What men's suit jackets with a welt pocket are available online ?", + "http://example.org/image_45006": "What casual graphic t-shirts are available for babies or toddlers ?", + "http://example.org/image_36685": "Can you recommend a beige blouse with a ruffled neckline as part of a fashionable outfit ?", + "http://example.org/image_11571": "What are some clothing options that include a light beige blazer with a notched collar and a front zipper ?", + "http://example.org/image_28528": "What are some professional clothing options for a single-button, notch lapel suit ?", + "http://example.org/image_50016": "What hoodies are suitable for everyday wear or casual outings ?", + "http://example.org/image_29005": "What are some North Face jackets ?", + "http://example.org/image_14200": "What casual outfit would you recommend that includes a white tank top ?", + "http://example.org/image_30547": "What are some midi length lower body pants ?", + "http://example.org/image_43170": "What are some fashionable and trendy upperbody outfits that include a black and white faux fur vest ?", + "http://example.org/image_49984": "What type of formal and professional attire would you recommend for a man ?", + "http://example.org/image_45098": "What are some short-sleeved polo shirts ?", + "http://example.org/image_50336": "What comfortable hoodies have a drawstring hood ?", + "http://example.org/image_37321": "What are some long-sleeved tops with a crew neckline ?", + "http://example.org/image_44497": "What are some tank tops made of cotton or cotton blend ?", + "http://example.org/image_16047": "What sunglasses would you recommend for a pink brick wall backdrop ?", + "http://example.org/image_22278": "Can you recommend brown suede boots ?", + "http://example.org/image_22395": "What are some fashionable and chic coats ?", + "http://example.org/image_15494": "What are some dresses with a cinched waist ?", +} + +df = pd.read_csv("embeddings.csv", index_col=0, nrows=None) +iris = df.index.values.tolist() +client = OpenAI(base_url="http://tentris-ml.cs.upb.de:8502/v1", api_key="token-tentris-upb") +docs = np.array(df.values) +docs_norms = docs / norm(docs, axis=1, keepdims=True) + +results = dict() +count = 0 + +for target_iri, query in evaluation_samples.items(): + try: + qr = np.array(client.embeddings.create(input=[query], model="tentris").data[0].embedding) + qr_norms = qr / norm(qr) + cosine_similarities = (docs_norms @ qr_norms).flatten() + + target_iri_index = iris.index(target_iri) + target_iri_score = cosine_similarities[target_iri_index] + cosine_similarities = cosine_similarities.tolist() + cosine_similarities.sort(reverse=True) + placement = cosine_similarities.index(target_iri_score) + + results[target_iri] = placement + count += 1 + print(f"Done {count}/100 {target_iri} with placement: {placement:,}/45,623") + except Exception as e: + print(e) + print(target_iri) + print(query) + with open("evaluation_results_embedding-based_uncompleted.json", "w") as outfile: + json.dump(results, outfile) + +print(results) +with open("evaluation_results_embedding-based_on_third_KG.json", "w") as outfile: + json.dump(results, outfile) \ No newline at end of file diff --git a/general_working_directory/mmr_calculator.py b/general_working_directory/mmr_calculator.py new file mode 100644 index 0000000..14713b7 --- /dev/null +++ b/general_working_directory/mmr_calculator.py @@ -0,0 +1,34 @@ +import json + + +def find_mrr(data_): + rr = 0 + n = len(data_) + for i, v in data_.items(): + rr += 1 / (v + 1) + mrr = rr / n + return mrr + + +print("==== Embedding-based model ====/n") + +with open('evaluation_results_embedding-based_on_second_KG.json', 'r') as file: + data = json.load(file) + +print(f"Second KG MRR = {find_mrr(data)}") + +with open('evaluation_results_embedding-based_on_third_KG.json', 'r') as file: + data = json.load(file) + +print(f"Third KG MRR = {find_mrr(data)}") + +print("==== BM25 model ====/n") +with open('evaluation_results_bm25_on_second_KG.json', 'r') as file: + data = json.load(file) + +print(f"Second KG MRR = {find_mrr(data)}") + +with open('evaluation_results_bm25_on_third_KG.json', 'r') as file: + data = json.load(file) + +print(f"Third KG MRR = {find_mrr(data)}")