deploy: 82a7878

SylphAI-Inc · Sep 10, 2024 · f99cf6e · f99cf6e
1 parent 9ba7944
commit f99cf6e
Show file tree

Hide file tree

Showing 174 changed files with 4,297 additions and 2,826 deletions.
diff --git a/.doctrees/apis/core/core.db.doctree b/.doctrees/apis/core/core.db.doctree
diff --git a/.doctrees/apis/datasets/datasets.types.doctree b/.doctrees/apis/datasets/datasets.types.doctree
diff --git a/.doctrees/apis/datasets/index.doctree b/.doctrees/apis/datasets/index.doctree
diff --git a/.doctrees/apis/eval/eval.answer_match_acc.doctree b/.doctrees/apis/eval/eval.answer_match_acc.doctree
diff --git a/.doctrees/apis/eval/eval.base.doctree b/.doctrees/apis/eval/eval.base.doctree
diff --git a/.doctrees/apis/eval/eval.functional.doctree b/.doctrees/apis/eval/eval.functional.doctree
diff --git a/.doctrees/apis/eval/eval.g_eval.doctree b/.doctrees/apis/eval/eval.g_eval.doctree
diff --git a/.doctrees/apis/eval/eval.llm_as_judge.doctree b/.doctrees/apis/eval/eval.llm_as_judge.doctree
diff --git a/.doctrees/apis/eval/eval.retriever_recall.doctree b/.doctrees/apis/eval/eval.retriever_recall.doctree
diff --git a/.doctrees/apis/eval/eval.retriever_relevance.doctree b/.doctrees/apis/eval/eval.retriever_relevance.doctree
diff --git a/.doctrees/apis/eval/index.doctree b/.doctrees/apis/eval/index.doctree
diff --git a/.doctrees/apis/index.doctree b/.doctrees/apis/index.doctree
diff --git a/.doctrees/apis/optim/optim.text_grad.backend_engine_prompt.doctree b/.doctrees/apis/optim/optim.text_grad.backend_engine_prompt.doctree
diff --git a/.doctrees/apis/utils/utils.lazy_import.doctree b/.doctrees/apis/utils/utils.lazy_import.doctree
diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle
diff --git a/.doctrees/tutorials/db.doctree b/.doctrees/tutorials/db.doctree
diff --git a/.doctrees/tutorials/evaluation.doctree b/.doctrees/tutorials/evaluation.doctree
diff --git a/.doctrees/tutorials/generator.doctree b/.doctrees/tutorials/generator.doctree
diff --git a/.doctrees/tutorials/index.doctree b/.doctrees/tutorials/index.doctree
diff --git a/.doctrees/tutorials/rag.doctree b/.doctrees/tutorials/rag.doctree
diff --git a/.doctrees/tutorials/retriever_xy.doctree b/.doctrees/tutorials/retriever_xy.doctree
diff --git a/.doctrees/use_cases/build_a_rag.doctree b/.doctrees/use_cases/build_a_rag.doctree
diff --git a/.doctrees/use_cases/eval_a_rag.doctree b/.doctrees/use_cases/eval_a_rag.doctree
diff --git a/.doctrees/use_cases/index.doctree b/.doctrees/use_cases/index.doctree
diff --git a/_images/G_eval_structure.png b/_images/G_eval_structure.png
diff --git a/_modules/components/retriever/faiss_retriever.html b/_modules/components/retriever/faiss_retriever.html
@@ -446,8 +446,6 @@ <h1>Source code for components.retriever.faiss_retriever</h1><div class="highlig
 <span class="kn">import</span> <span class="nn">logging</span>
 <span class="kn">import</span> <span class="nn">os</span>
 
-<span class="kn">import</span> <span class="nn">faiss</span>
-
 
 <span class="kn">from</span> <span class="nn">adalflow.core.retriever</span> <span class="kn">import</span> <span class="n">Retriever</span>
 <span class="kn">from</span> <span class="nn">adalflow.core.embedder</span> <span class="kn">import</span> <span class="n">Embedder</span>
@@ -462,6 +460,7 @@ <h1>Source code for components.retriever.faiss_retriever</h1><div class="highlig
 <span class="kn">from</span> <span class="nn">adalflow.utils.lazy_import</span> <span class="kn">import</span> <span class="n">safe_import</span><span class="p">,</span> <span class="n">OptionalPackages</span>
 
 <span class="n">safe_import</span><span class="p">(</span><span class="n">OptionalPackages</span><span class="o">.</span><span class="n">FAISS</span><span class="o">.</span><span class="n">value</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">OptionalPackages</span><span class="o">.</span><span class="n">FAISS</span><span class="o">.</span><span class="n">value</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
+<span class="kn">import</span> <span class="nn">faiss</span>
 
 <span class="n">log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
 

diff --git a/_modules/core/db.html b/_modules/core/db.html
diff --git a/_modules/core/generator.html b/_modules/core/generator.html
@@ -437,7 +437,6 @@ <h1>Source code for core.generator</h1><div class="highlight"><pre>
 <span class="kn">import</span> <span class="nn">json</span>
 
 <span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Any</span><span class="p">,</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">Callable</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">,</span> <span class="n">List</span>
-<span class="kn">from</span> <span class="nn">copy</span> <span class="kn">import</span> <span class="n">deepcopy</span>
 <span class="kn">import</span> <span class="nn">logging</span>
 
 
@@ -543,11 +542,6 @@ <h1>Source code for core.generator</h1><div class="highlight"><pre>
             <span class="p">)</span>
 
         <span class="n">template</span> <span class="o">=</span> <span class="n">template</span> <span class="ow">or</span> <span class="n">DEFAULT_LIGHTRAG_SYSTEM_PROMPT</span>
-        <span class="k">try</span><span class="p">:</span>
-            <span class="n">prompt_kwargs</span> <span class="o">=</span> <span class="n">deepcopy</span><span class="p">(</span><span class="n">prompt_kwargs</span><span class="p">)</span>
-        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
-            <span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Error copying the prompt_kwargs: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
-            <span class="n">prompt_kwargs</span> <span class="o">=</span> <span class="n">prompt_kwargs</span>
 
         <span class="c1"># Cache</span>
         <span class="n">model_str</span> <span class="o">=</span> <span class="p">(</span>
@@ -1314,7 +1308,17 @@ <h1>Source code for core.generator</h1><div class="highlight"><pre>
             <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">call</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">_extra_repr</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+        <span class="c1"># Create the string for model_kwargs</span>
         <span class="n">s</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;model_kwargs=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">model_kwargs</span><span class="si">}</span><span class="s2">, &quot;</span>
+
+        <span class="c1"># Create the string for trainable prompt_kwargs</span>
+        <span class="n">prompt_kwargs_repr</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="n">k</span>
+            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">prompt_kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">Parameter</span><span class="p">)</span> <span class="ow">and</span> <span class="n">v</span><span class="o">.</span><span class="n">requires_opt</span>
+        <span class="p">]</span>
+
+        <span class="n">s</span> <span class="o">+=</span> <span class="sa">f</span><span class="s2">&quot;trainable_prompt_kwargs=</span><span class="si">{</span><span class="n">prompt_kwargs_repr</span><span class="si">}</span><span class="s2">&quot;</span>
         <span class="k">return</span> <span class="n">s</span>
 
 <div class="viewcode-block" id="Generator.to_dict">

diff --git a/_modules/eval/answer_match_acc.html b/_modules/eval/answer_match_acc.html
@@ -429,7 +429,7 @@
                 <article class="bd-article">
 
   <h1>Source code for eval.answer_match_acc</h1><div class="highlight"><pre>
-<span></span><span class="sd">&quot;&quot;&quot;This is the metric for answer matching. It compares the predicted answer with the ground truth answer.&quot;&quot;&quot;</span>
+<span></span><span class="sd">&quot;&quot;&quot;This is the metric for QA generation. It compares the predicted answer with the ground truth answer.&quot;&quot;&quot;</span>
 
 <span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">List</span><span class="p">,</span> <span class="n">Literal</span>
 <span class="kn">from</span> <span class="nn">adalflow.eval.base</span> <span class="kn">import</span> <span class="n">BaseEvaluator</span><span class="p">,</span> <span class="n">EvaluationResult</span>
@@ -504,38 +504,6 @@ <h1>Source code for eval.answer_match_acc</h1><div class="highlight"><pre>
             <span class="k">raise</span> <span class="ne">NotImplementedError</span></div>
 
 
-    <span class="c1"># def compute_single_item(self, pred_answer: object, gt_answer: object) -&gt; float:</span>
-    <span class="c1">#     r&quot;&quot;&quot;</span>
-    <span class="c1">#     Compute the match accuracy of the predicted answer for a single query.</span>
-
-    <span class="c1">#     Allow any type of input for pred_answer and gt_answer.</span>
-    <span class="c1">#     When evaluating, the input will be converted to string.</span>
-
-    <span class="c1">#     Args:</span>
-    <span class="c1">#         pred_answer (object): Predicted answer.</span>
-    <span class="c1">#         gt_answer (object): Ground truth answer.</span>
-
-    <span class="c1">#     Returns:</span>
-    <span class="c1">#         float: Match accuracy.</span>
-    <span class="c1">#     &quot;&quot;&quot;</span>
-    <span class="c1">#     if isinstance(pred_answer, Parameter):</span>
-    <span class="c1">#         pred_answer = pred_answer.data</span>
-    <span class="c1">#     if isinstance(gt_answer, Parameter):</span>
-    <span class="c1">#         gt_answer = gt_answer.data</span>
-    <span class="c1">#     try:</span>
-    <span class="c1">#         pred_answer = str(pred_answer).split(&quot; &quot;)</span>
-    <span class="c1">#         gt_answer = str(gt_answer).split(&quot; &quot;)</span>
-    <span class="c1">#     except Exception as e:</span>
-    <span class="c1">#         raise ValueError(</span>
-    <span class="c1">#             f&quot;Error converting pred_answer and gt_answer to string: {e}&quot;</span>
-    <span class="c1">#         )</span>
-    <span class="c1">#     if self.type == &quot;exact_match&quot;:</span>
-    <span class="c1">#         return 1.0 if pred_answer == gt_answer else 0.0</span>
-    <span class="c1">#     elif self.type == &quot;fuzzy_match&quot;:</span>
-    <span class="c1">#         return 1.0 if gt_answer in pred_answer else 0.0</span>
-    <span class="c1">#     else:</span>
-    <span class="c1">#         raise NotImplementedError</span>
-
 <div class="viewcode-block" id="AnswerMatchAcc.compute">
 <a class="viewcode-back" href="../../apis/eval/index.html#eval.answer_match_acc.AnswerMatchAcc.compute">[docs]</a>
     <span class="k">def</span> <span class="nf">compute</span><span class="p">(</span>

diff --git a/_modules/eval/base.html b/_modules/eval/base.html
@@ -431,7 +431,7 @@
   <h1>Source code for eval.base</h1><div class="highlight"><pre>
 <span></span><span class="sd">&quot;&quot;&quot;Abstract base class for evaluation metrics.&quot;&quot;&quot;</span>
 
-<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">List</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Any</span>
 
 <span class="kn">from</span> <span class="nn">dataclasses</span> <span class="kn">import</span> <span class="n">dataclass</span>
 
@@ -461,9 +461,10 @@ <h1>Source code for eval.base</h1><div class="highlight"><pre>
         <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">&quot;Subclasses must implement this method.&quot;</span><span class="p">)</span></div>
 
 
+    <span class="c1"># TODO: support multi-threading or async to speed up evaluation</span>
 <div class="viewcode-block" id="BaseEvaluator.compute">
 <a class="viewcode-back" href="../../apis/eval/eval.base.html#eval.base.BaseEvaluator.compute">[docs]</a>
-    <span class="k">def</span> <span class="nf">compute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">EvaluationResult</span><span class="p">:</span>
+    <span class="k">def</span> <span class="nf">compute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Any</span><span class="p">:</span>
 <span class="w">        </span><span class="sd">&quot;&quot;&quot;Evaluate a list of predictions and ground truth values. and return overall score and per-item scores.&quot;&quot;&quot;</span>
         <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">&quot;Subclasses must implement this method.&quot;</span><span class="p">)</span></div>