added scripts for cel evaluation and tsne plot

cel_evaluation_tdl.json

@@ -0,0 +1 @@
+{"http://example.org/image_1082": [0.7349397590361445, 0.6901408450704225, 0.6421052631578947, 0.8591549295774648], "http://example.org/image_42527": [0.88, 0.8802816901408451, 0.8820754716981132, 0.8779342723004695], "http://example.org/image_39373": [0.8786610878661089, 0.863849765258216, 0.7924528301886793, 0.9859154929577465], "http://example.org/image_34524": [0.7848605577689244, 0.7464788732394366, 0.6816608996539792, 0.9248826291079812], "http://example.org/image_38547": [0.7056530214424952, 0.6455399061032864, 0.6033333333333334, 0.8497652582159625], "http://example.org/image_21806": [0.5377049180327869, 0.6690140845070423, 0.8913043478260869, 0.38497652582159625], "http://example.org/image_41097": [0.840085287846482, 0.823943661971831, 0.76953125, 0.9248826291079812], "http://example.org/image_5434": [0.586046511627907, 0.5821596244131455, 0.5806451612903226, 0.5915492957746479], "http://example.org/image_889": [0.694949494949495, 0.6455399061032864, 0.6099290780141844, 0.8075117370892019], "http://example.org/image_48024": [0.814498933901919, 0.795774647887324, 0.74609375, 0.8967136150234741], "http://example.org/image_23951": [0.8226221079691517, 0.8380281690140845, 0.9090909090909091, 0.7511737089201878], "http://example.org/image_34371": [0.8157248157248158, 0.823943661971831, 0.8556701030927835, 0.7793427230046949], "http://example.org/image_26553": [0.7630057803468208, 0.7112676056338029, 0.6470588235294118, 0.9295774647887324], "http://example.org/image_39479": [0.9066059225512528, 0.903755868544601, 0.8805309734513275, 0.9342723004694836], "http://example.org/image_17925": [0.4102564102564103, 0.568075117370892, 0.6464646464646465, 0.3004694835680751], "http://example.org/image_24171": [0.39867109634551495, 0.5751173708920188, 0.6818181818181818, 0.28169014084507044], "http://example.org/image_6829": [0.6265664160401002, 0.6502347417840375, 0.6720430107526881, 0.5868544600938967], "http://example.org/image_4014": [0.831896551724138, 0.8169014084507042, 0.7689243027888446, 0.9061032863849765], "http://example.org/image_36802": [0.8558758314855875, 0.8474178403755869, 0.8109243697478992, 0.9061032863849765], "http://example.org/image_4174": [0.43278688524590164, 0.5938967136150235, 0.717391304347826, 0.30985915492957744], "http://example.org/image_30775": [0.39552238805970147, 0.6197183098591549, 0.9636363636363636, 0.24882629107981222], "http://example.org/image_32118": [0.870488322717622, 0.8568075117370892, 0.7945736434108527, 0.9624413145539906], "http://example.org/image_39494": [0.4368600682593856, 0.6126760563380281, 0.8, 0.3004694835680751], "http://example.org/image_1069": [0.7810945273631841, 0.7934272300469484, 0.8306878306878307, 0.7370892018779343], "http://example.org/image_30081": [0.6933333333333334, 0.7300469483568075, 0.8024691358024691, 0.6103286384976526], "http://example.org/image_14081": [0.8621553884711779, 0.8708920187793427, 0.9247311827956989, 0.8075117370892019], "http://example.org/image_8082": [0.8238095238095238, 0.8262910798122066, 0.8357487922705314, 0.812206572769953], "http://example.org/image_21051": [0.7872860635696822, 0.795774647887324, 0.8214285714285714, 0.755868544600939], "http://example.org/image_49508": [0.8444444444444444, 0.8192488262910798, 0.7411347517730497, 0.9812206572769953], "http://example.org/image_14325": [0.9466666666666668, 0.9436619718309859, 0.8987341772151899, 1.0], "http://example.org/image_1352": [0.5517241379310345, 0.6643192488262911, 0.8301886792452831, 0.4131455399061033], "http://example.org/image_81": [0.8157349896480331, 0.7910798122065728, 0.7296296296296296, 0.9248826291079812], "http://example.org/image_6051": [0.8259860788863109, 0.823943661971831, 0.8165137614678899, 0.8356807511737089], "http://example.org/image_37671": [0.963855421686747, 0.9647887323943662, 0.9900990099009901, 0.9389671361502347], "http://example.org/image_15166": [0.8, 0.8145539906103286, 0.8681318681318682, 0.7417840375586855], "http://example.org/image_14193": [0.7238605898123324, 0.7582159624413145, 0.84375, 0.6338028169014085], "http://example.org/image_49546": [0.8091286307053943, 0.784037558685446, 0.724907063197026, 0.9154929577464789], "http://example.org/image_12045": [0.8585131894484412, 0.8615023474178404, 0.8774509803921569, 0.8403755868544601], "http://example.org/image_6720": [0.8709677419354838, 0.8685446009389671, 0.8552036199095022, 0.8873239436619719], "http://example.org/image_33263": [0.807531380753138, 0.784037558685446, 0.7283018867924528, 0.9061032863849765], "http://example.org/image_2460": [0.7132530120481928, 0.7206572769953051, 0.7326732673267327, 0.6948356807511737], "http://example.org/image_26441": [0.9659863945578232, 0.9647887323943662, 0.9342105263157895, 1.0], "http://example.org/image_20880": [0.9138755980861244, 0.9154929577464789, 0.9317073170731708, 0.8967136150234741], "http://example.org/image_2418": [0.9050000000000001, 0.9107981220657277, 0.9679144385026738, 0.8497652582159625], "http://example.org/image_47279": [0.7857142857142858, 0.8169014084507042, 0.9470198675496688, 0.6713615023474179], "http://example.org/image_45305": [0.8132295719844358, 0.7746478873239436, 0.6943521594684385, 0.9812206572769953], "http://example.org/image_27640": [0.9060240963855423, 0.9084507042253521, 0.9306930693069307, 0.8826291079812206], "http://example.org/image_19870": [0.6551724137931034, 0.5774647887323944, 0.5533980582524272, 0.8028169014084507], "http://example.org/image_11359": [0.9507389162561576, 0.9530516431924883, 1.0, 0.9061032863849765], "http://example.org/image_276": [0.8213552361396304, 0.795774647887324, 0.7299270072992701, 0.9389671361502347], "http://example.org/image_30229": [0.6217391304347826, 0.5915492957746479, 0.5789473684210527, 0.6713615023474179], "http://example.org/image_967": [0.9039301310043668, 0.8967136150234741, 0.8448979591836735, 0.971830985915493], "http://example.org/image_26357": [0.7191413237924865, 0.6314553990610329, 0.5809248554913294, 0.9436619718309859], "http://example.org/image_2535": [0.9441860465116279, 0.9436619718309859, 0.9354838709677419, 0.9530516431924883], "http://example.org/image_11368": [0.6016713091922005, 0.6643192488262911, 0.7397260273972602, 0.5070422535211268], "http://example.org/image_21672": [0.5222929936305732, 0.647887323943662, 0.8118811881188119, 0.38497652582159625], "http://example.org/image_10979": [0.8840262582056893, 0.8755868544600939, 0.8278688524590164, 0.9483568075117371], "http://example.org/image_37384": [0.8558558558558559, 0.8497652582159625, 0.8225108225108225, 0.892018779342723], "http://example.org/image_204": [0.7060998151571163, 0.6267605633802817, 0.5823170731707317, 0.8967136150234741], "http://example.org/image_12907": [0.9234135667396062, 0.9178403755868545, 0.8647540983606558, 0.9906103286384976], "http://example.org/image_48528": [0.8409586056644879, 0.8286384976525821, 0.7845528455284553, 0.9061032863849765], "http://example.org/image_34867": [0.8983451536643027, 0.8990610328638498, 0.9047619047619048, 0.892018779342723], "http://example.org/image_32243": [0.7240356083086055, 0.7816901408450704, 0.9838709677419355, 0.5727699530516432], "http://example.org/image_29033": [0.8371040723981901, 0.8309859154929577, 0.8078602620087336, 0.8685446009389671], "http://example.org/image_31567": [0.8632075471698113, 0.863849765258216, 0.8672985781990521, 0.8591549295774648], "http://example.org/image_11241": [0.854122621564482, 0.8380281690140845, 0.7769230769230769, 0.9483568075117371], "http://example.org/image_32496": [0.855036855036855, 0.8615023474178404, 0.8969072164948454, 0.8169014084507042], "http://example.org/image_36098": [0.7442748091603054, 0.6854460093896714, 0.6270096463022508, 0.9154929577464789], "http://example.org/image_9747": [0.6836363636363637, 0.5915492957746479, 0.5578635014836796, 0.8826291079812206], "http://example.org/image_42949": [0.8980582524271845, 0.9014084507042254, 0.9296482412060302, 0.8685446009389671], "http://example.org/image_31097": [0.9294403892944039, 0.931924882629108, 0.9646464646464646, 0.8967136150234741], "http://example.org/image_27229": [0.993006993006993, 0.9929577464788732, 0.9861111111111112, 1.0], "http://example.org/image_26932": [0.877030162412993, 0.8755868544600939, 0.8669724770642202, 0.8873239436619719], "http://example.org/image_16304": [0.6453333333333333, 0.687793427230047, 0.7469135802469136, 0.568075117370892], "http://example.org/image_40627": [0.8962472406181015, 0.8896713615023474, 0.8458333333333333, 0.9530516431924883], "http://example.org/image_44639": [0.8877805486284289, 0.8943661971830986, 0.9468085106382979, 0.8356807511737089], "http://example.org/image_39575": [0.8635235732009925, 0.8708920187793427, 0.9157894736842105, 0.8169014084507042], "http://example.org/image_3500": [0.7803617571059432, 0.8004694835680751, 0.867816091954023, 0.7089201877934272], "http://example.org/image_50164": [0.8850325379609544, 0.8755868544600939, 0.8225806451612904, 0.9577464788732394], "http://example.org/image_6129": [0.15, 0.5211267605633803, 0.6666666666666666, 0.08450704225352113], "http://example.org/image_44775": [0.718562874251497, 0.6690140845070423, 0.625, 0.8450704225352113], "http://example.org/image_39515": [0.7398843930635838, 0.6830985915492958, 0.6274509803921569, 0.9014084507042254], "http://example.org/image_45006": [0.7380281690140844, 0.7816901408450704, 0.9225352112676056, 0.6150234741784038], "http://example.org/image_36685": [0.6484018264840182, 0.6384976525821596, 0.6311111111111111, 0.6666666666666666], "http://example.org/image_11571": [0.6526315789473683, 0.6901408450704225, 0.7425149700598802, 0.5821596244131455], "http://example.org/image_28528": [0.9019607843137255, 0.9061032863849765, 0.9435897435897436, 0.863849765258216], "http://example.org/image_50016": [0.8571428571428571, 0.8591549295774648, 0.8695652173913043, 0.8450704225352113], "http://example.org/image_29005": [0.9605568445475637, 0.960093896713615, 0.9495412844036697, 0.971830985915493], "http://example.org/image_14200": [0.8195121951219512, 0.8262910798122066, 0.8527918781725888, 0.7887323943661971], "http://example.org/image_30547": [0.9036697247706422, 0.9014084507042254, 0.8834080717488789, 0.9248826291079812], "http://example.org/image_43170": [0.8316326530612246, 0.8450704225352113, 0.9106145251396648, 0.7652582159624414], "http://example.org/image_49984": [0.8812095032397408, 0.8708920187793427, 0.816, 0.9577464788732394], "http://example.org/image_45098": [0.9592760180995475, 0.9577464788732394, 0.925764192139738, 0.9953051643192489], "http://example.org/image_50336": [0.8817733990147782, 0.8873239436619719, 0.927461139896373, 0.8403755868544601], "http://example.org/image_37321": [0.7182044887780549, 0.7347417840375586, 0.7659574468085106, 0.676056338028169], "http://example.org/image_44497": [0.18666666666666668, 0.284037558685446, 0.21604938271604937, 0.1643192488262911], "http://example.org/image_16047": [0.9041095890410958, 0.9014084507042254, 0.88, 0.9295774647887324], "http://example.org/image_22278": [0.3137254901960784, 0.5892018779342723, 0.9523809523809523, 0.18779342723004694], "http://example.org/image_22395": [0.9857142857142858, 0.9859154929577465, 1.0, 0.971830985915493], "http://example.org/image_15494": [0.7668393782383419, 0.7887323943661971, 0.8554913294797688, 0.6948356807511737]}

general_working_directory/TSNE_plot.py

@@ -0,0 +1,45 @@
+import numpy as np
+from openai import OpenAI
+from declarations import evaluation_samples
+import matplotlib.pyplot as plt
+from sklearn import manifold
+
+client = OpenAI(base_url="http://tentris-ml.cs.upb.de:8502/v1", api_key="token-tentris-upb")
+
+
+def embed(txt):
+    em = client.embeddings.create(input=[txt], model="tentris").data[0].embedding
+    assert type(em) is list, f"{type(em)} is not a list"
+    return em
+
+
+X = np.array([embed(q) for iri, q in evaluation_samples.items()])
+
+assert X.shape == (100, 4096)
+
+n_components = 2
+t_sne = manifold.TSNE(
+    n_components=n_components,
+    perplexity=2,
+    init="random",
+    max_iter=500,
+    random_state=0,
+)
+(embeddings_2d) = t_sne.fit_transform(X)
+# print(type(embeddings_2d))
+# print(embeddings_2d)
+
+def plot_2d(points, title):
+    fig, ax = plt.subplots(figsize=(7, 7), facecolor="white", constrained_layout=True)
+    fig.suptitle(title, size=16)
+    x, y = points.T
+    ax.scatter(x, y, s=50, alpha=0.8)
+    ax.set_title(title)
+    ax.xaxis.set_major_formatter(plt.NullFormatter())
+    ax.yaxis.set_major_formatter(plt.NullFormatter())
+    # plt.axis([-50, 50, -50, 50])
+    plt.savefig("tsne_plot.png", dpi=300, format="png")
+    plt.show()
+
+
+plot_2d(embeddings_2d, "T-distributed Stochastic Neighbor Embedding (TSNE)")

general_working_directory/cel_evaluation.py

@@ -0,0 +1,78 @@
+import numpy as np
+import pandas as pd
+from declarations import evaluation_samples
+from random import sample
+from owlapy.owl_individual import OWLNamedIndividual
+from ontolearn.knowledge_base import KnowledgeBase
+from ontolearn.learners import TDL
+from ontolearn.learning_problem import PosNegLPStandard
+import json
+
+df = pd.read_csv("benchmark_knn.csv", index_col=0, nrows=None)
+iris = df.index.values.tolist()
+knn = df.values.tolist()
+
+k = 213  # k in knn
+kb = KnowledgeBase(path="fashionpedia-second-generation-v2.owl")
+model = TDL(knowledge_base=kb, use_nominals=True, max_runtime=10)
+
+
+def get_list_of_iris(target_iri):
+    target_iri_index = iris.index(target_iri)
+    neighbors_indexes = knn[target_iri_index]
+    neighbors_iris = list()
+    for i in neighbors_indexes:
+        neighbor_iri = iris[i]
+        neighbors_iris.append(neighbor_iri)
+    return neighbors_iris
+
+
+def get_random_examples(examples_to_avoid):
+    iris_to_consider = [iri for iri in iris if iri not in examples_to_avoid]
+    random_samples = sample(iris_to_consider, k)
+    return random_samples
+
+
+def get_performance_measurements(individuals, pos, neg):
+    assert type(individuals) == type(pos) == type(neg), f"Types must match:{type(individuals)},{type(pos)},{type(neg)}"
+    tp = len(pos.intersection(individuals))
+    tn = len(neg.difference(individuals))
+    fp = len(neg.intersection(individuals))
+    fn = len(pos.difference(individuals))
+    try:
+        recall = tp / (tp + fn)
+    except ZeroDivisionError:
+        return 0.0
+    try:
+        precision = tp / (tp + fp)
+    except ZeroDivisionError:
+        return 0.0
+    if precision == 0 or recall == 0:
+        return 0.0
+    acc = (tp + tn) / (tp + tn + fp + fn)
+    f_1 = 2 * ((precision * recall) / (precision + recall))
+    return f_1, acc, precision, recall
+
+
+results = dict()
+
+for iri in evaluation_samples.keys():
+    pos_examples = get_list_of_iris(iri)
+    neg_examples = get_random_examples(pos_examples)
+
+    typed_pos = set(map(OWLNamedIndividual, pos_examples))
+    typed_neg = set(map(OWLNamedIndividual, neg_examples))
+
+    lp = PosNegLPStandard(pos=typed_pos, neg=typed_neg)
+    prediction = model.fit(learning_problem=lp).best_hypotheses(n=1)
+    f1_score, accuracy, precision, recall = get_performance_measurements(
+        individuals=set({i for i in kb.individuals(prediction)}),
+        pos=typed_pos, neg=typed_neg)
+    results[iri] = [f1_score, accuracy, precision, recall]
+
+with open("cel_evaluation.json", "w") as outfile:
+    json.dump(results, outfile)
+
+values = np.array(list(results.values()))
+mean_values = np.mean(values, axis=0)
+print(list(mean_values))