Support embedders setting

src/features.rs

@@ -109,24 +109,17 @@ mod tests {
     use super::*;
     use meilisearch_test_macro::meilisearch_test;
 
-    #[meilisearch_test]
-    async fn test_experimental_features_get(client: Client) {
-        let mut features = ExperimentalFeatures::new(&client);
-        features.set_vector_store(false);
-        let _ = features.update().await.unwrap();
-
-        let res = features.get().await.unwrap();
-
-        assert!(!res.vector_store);
-    }
-
+    /// there is purposely no test which disables this feature to prevent impact on other testcases
+    /// the setting is shared amongst all indexes
     #[meilisearch_test]
     async fn test_experimental_features_enable_vector_store(client: Client) {
         let mut features = ExperimentalFeatures::new(&client);
         features.set_vector_store(true);
 
         let res = features.update().await.unwrap();
+        assert!(res.vector_store);
 
+        let res = features.get().await.unwrap();
         assert!(res.vector_store);
     }
 }

src/search.rs

@@ -142,6 +142,22 @@ pub enum Selectors<T> {
     All,
 }
 
+/// EXPERIMENTAL
+#[derive(Debug, Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct HybridSearch<'a> {
+    /// Indicates one of the embedders configured for the queried index
+    ///
+    /// **Default: `"default"`**
+    embedder: &'a str,
+    /// number between `0` and `1`:
+    /// - `0.0` indicates full keyword search
+    /// - `1.0` indicates full semantic search
+    ///
+    /// **Default: `0.5`**
+    semantic_ratio: f32,
+}
+
 type AttributeToCrop<'a> = (&'a str, Option<usize>);
 
 /// A struct representing a query.
@@ -346,6 +362,24 @@ pub struct SearchQuery<'a, Http: HttpClient> {
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) index_uid: Option<&'a str>,
+
+    /// EXPERIMENTAL
+    /// Defines whether to utilise previously defined embedders for semantic searching
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub hybrid: Option<HybridSearch<'a>>,
+
+    /// EXPERIMENTAL
+    /// Defines what vectors an userprovided embedder has gotten for semantic searching
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub vector: Option<&'a [f32]>,
+
+    /// EXPERIMENTAL
+    /// Defines whether vectors for semantic searching are returned in the search results
+    /// Can Significantly increase the response size.
+    ///
+    /// **Default: `false`**
+    #[serde(skip_serializing_if = "Option::is_none")]
+    retrieve_vectors: Option<bool>,
 }
 
 #[allow(missing_docs)]
@@ -375,6 +409,9 @@ impl<'a, Http: HttpClient> SearchQuery<'a, Http> {
             show_ranking_score_details: None,
             matching_strategy: None,
             index_uid: None,
+            hybrid: None,
+            vector: None,
+            retrieve_vectors: None,
             distinct: None,
             ranking_score_threshold: None,
         }
@@ -469,6 +506,13 @@ impl<'a, Http: HttpClient> SearchQuery<'a, Http> {
         self.filter = Some(Filter::new(Either::Right(filter)));
         self
     }
+    pub fn with_retrieve_vectors<'b>(
+        &'b mut self,
+        retrieve_vectors: bool,
+    ) -> &'b mut SearchQuery<'a, Http> {
+        self.retrieve_vectors = Some(retrieve_vectors);
+        self
+    }
     pub fn with_facets<'b>(
         &'b mut self,
         facets: Selectors<&'a [&'a str]>,
@@ -569,6 +613,26 @@ impl<'a, Http: HttpClient> SearchQuery<'a, Http> {
         self.index_uid = Some(&self.index.uid);
         self
     }
+    /// EXPERIMENTAL
+    /// Defines whether to utilise previously defined embedders for semantic searching
+    pub fn with_hybrid<'b>(
+        &'b mut self,
+        embedder: &'a str,
+        semantic_ratio: f32,
+    ) -> &'b mut SearchQuery<'a, Http> {
+        self.hybrid = Some(HybridSearch {
+            embedder,
+            semantic_ratio,
+        });
+        self
+    }
+    /// EXPERIMENTAL
+    /// Defines what vectors an userprovided embedder has gotten for semantic searching
+    pub fn with_vector<'b>(&'b mut self, vector: &'a [f32]) -> &'b mut SearchQuery<'a, Http> {
+        self.vector = Some(vector);
+        self
+    }
+    #[must_use]
     pub fn with_distinct<'b>(&'b mut self, distinct: &'a str) -> &'b mut SearchQuery<'a, Http> {
         self.distinct = Some(distinct);
         self
@@ -666,6 +730,36 @@ mod tests {
         kind: String,
         number: i32,
         nested: Nested,
+        #[serde(default)]
+        _vectors: Option<Vectors>,
+    }
+
+    #[derive(Debug, Serialize, Deserialize, PartialEq)]
+    struct Vector {
+        embeddings: SingleOrMultipleVectors,
+        regenerate: bool,
+    }
+
+    #[derive(Serialize, Deserialize, Debug, PartialEq)]
+    #[serde(untagged)]
+    enum SingleOrMultipleVectors {
+        Single(Vec<f32>),
+        Multiple(Vec<Vec<f32>>),
+    }
+
+    #[derive(Debug, Serialize, Deserialize, PartialEq)]
+    struct Vectors(HashMap<String, Vector>);
+
+    impl From<&[f32; 1]> for Vectors {
+        fn from(value: &[f32; 1]) -> Self {
+            Vectors(HashMap::from([(
+                S("default"),
+                Vector {
+                    embeddings: SingleOrMultipleVectors::Multiple(Vec::from([value.to_vec()])),
+                    regenerate: false,
+                },
+            )]))
+        }
     }
 
     impl PartialEq<Map<String, Value>> for Document {
@@ -678,17 +772,24 @@ mod tests {
     }
 
     async fn setup_test_index(client: &Client, index: &Index) -> Result<(), Error> {
+        // Vector store is enabled for all to have consistent test runs
+        // This setting is shared by every index
+        let features = crate::features::ExperimentalFeatures::new(&client)
+            .set_vector_store(true)
+            .update()
+            .await?;
+        assert_eq!(features.vector_store, true);
         let t0 = index.add_documents(&[
-            Document { id: 0, kind: "text".into(), number: 0, value: S("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."), nested: Nested { child: S("first") } },
-            Document { id: 1, kind: "text".into(), number: 10, value: S("dolor sit amet, consectetur adipiscing elit"), nested: Nested { child: S("second") } },
-            Document { id: 2, kind: "title".into(), number: 20, value: S("The Social Network"), nested: Nested { child: S("third") } },
-            Document { id: 3, kind: "title".into(), number: 30, value: S("Harry Potter and the Sorcerer's Stone"), nested: Nested { child: S("fourth") } },
-            Document { id: 4, kind: "title".into(), number: 40, value: S("Harry Potter and the Chamber of Secrets"), nested: Nested { child: S("fift") } },
-            Document { id: 5, kind: "title".into(), number: 50, value: S("Harry Potter and the Prisoner of Azkaban"), nested: Nested { child: S("sixth") } },
-            Document { id: 6, kind: "title".into(), number: 60, value: S("Harry Potter and the Goblet of Fire"), nested: Nested { child: S("seventh") } },
-            Document { id: 7, kind: "title".into(), number: 70, value: S("Harry Potter and the Order of the Phoenix"), nested: Nested { child: S("eighth") } },
-            Document { id: 8, kind: "title".into(), number: 80, value: S("Harry Potter and the Half-Blood Prince"), nested: Nested { child: S("ninth") } },
-            Document { id: 9, kind: "title".into(), number: 90, value: S("Harry Potter and the Deathly Hallows"), nested: Nested { child: S("tenth") } },
+            Document { id: 0, kind: "text".into(), number: 0, value: S("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."), nested: Nested { child: S("first") }, _vectors: Some(Vectors::from(&[1000.0]))},
+            Document { id: 1, kind: "text".into(), number: 10, value: S("dolor sit amet, consectetur adipiscing elit"), nested: Nested { child: S("second") }, _vectors: Some(Vectors::from(&[2000.0]))  },
+            Document { id: 2, kind: "title".into(), number: 20, value: S("The Social Network"), nested: Nested { child: S("third") }, _vectors: Some(Vectors::from(&[3000.0]))  },
+            Document { id: 3, kind: "title".into(), number: 30, value: S("Harry Potter and the Sorcerer's Stone"), nested: Nested { child: S("fourth") }, _vectors: Some(Vectors::from(&[4000.0]))  },
+            Document { id: 4, kind: "title".into(), number: 40, value: S("Harry Potter and the Chamber of Secrets"), nested: Nested { child: S("fift") }, _vectors: Some(Vectors::from(&[5000.0]))  },
+            Document { id: 5, kind: "title".into(), number: 50, value: S("Harry Potter and the Prisoner of Azkaban"), nested: Nested { child: S("sixth") }, _vectors: Some(Vectors::from(&[6000.0]))  },
+            Document { id: 6, kind: "title".into(), number: 60, value: S("Harry Potter and the Goblet of Fire"), nested: Nested { child: S("seventh") }, _vectors: Some(Vectors::from(&[7000.0]))  },
+            Document { id: 7, kind: "title".into(), number: 70, value: S("Harry Potter and the Order of the Phoenix"), nested: Nested { child: S("eighth") }, _vectors: Some(Vectors::from(&[8000.0]))  },
+            Document { id: 8, kind: "title".into(), number: 80, value: S("Harry Potter and the Half-Blood Prince"), nested: Nested { child: S("ninth") }, _vectors: Some(Vectors::from(&[9000.0]))  },
+            Document { id: 9, kind: "title".into(), number: 90, value: S("Harry Potter and the Deathly Hallows"), nested: Nested { child: S("tenth") }, _vectors: Some(Vectors::from(&[10000.0]))  },
         ], None).await?;
         let t1 = index
             .set_filterable_attributes(["kind", "value", "number"])
@@ -776,7 +877,8 @@ mod tests {
                 value: S("dolor sit amet, consectetur adipiscing elit"),
                 kind: S("text"),
                 number: 10,
-                nested: Nested { child: S("second") }
+                nested: Nested { child: S("second") },
+                _vectors: Some(Vectors::from(&[2000.0])),
             },
             &results.hits[0].result
         );
@@ -948,7 +1050,8 @@ mod tests {
                 value: S("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do…"),
                 kind: S("text"),
                 number: 0,
-                nested: Nested { child: S("first") }
+                nested: Nested { child: S("first") },
+                _vectors: None,
             },
             results.hits[0].formatted_result.as_ref().unwrap()
         );
@@ -963,7 +1066,8 @@ mod tests {
                 value: S("Lorem ipsum dolor sit amet…"),
                 kind: S("text"),
                 number: 0,
-                nested: Nested { child: S("first") }
+                nested: Nested { child: S("first") },
+                _vectors: None,
             },
             results.hits[0].formatted_result.as_ref().unwrap()
         );
@@ -984,7 +1088,8 @@ mod tests {
             value: S("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."),
             kind: S("text"),
             number: 0,
-            nested: Nested { child: S("first") }
+            nested: Nested { child: S("first") },
+            _vectors: None,
         },
         results.hits[0].formatted_result.as_ref().unwrap());
 
@@ -999,7 +1104,8 @@ mod tests {
                 value: S("Lorem ipsum dolor sit amet…"),
                 kind: S("text"),
                 number: 0,
-                nested: Nested { child: S("first") }
+                nested: Nested { child: S("first") },
+                _vectors: None,
             },
             results.hits[0].formatted_result.as_ref().unwrap()
         );
@@ -1024,7 +1130,8 @@ mod tests {
                 value: S("(ꈍᴗꈍ)sed do eiusmod tempor incididunt ut(ꈍᴗꈍ)"),
                 kind: S("text"),
                 number: 0,
-                nested: Nested { child: S("first") }
+                nested: Nested { child: S("first") },
+                _vectors: None,
             },
             results.hits[0].formatted_result.as_ref().unwrap()
         );
@@ -1051,7 +1158,8 @@ mod tests {
                 value: S("The (⊃｡•́‿•̀｡)⊃ Social ⊂(´• ω •`⊂) Network"),
                 kind: S("title"),
                 number: 20,
-                nested: Nested { child: S("third") }
+                nested: Nested { child: S("third") },
+                _vectors: None,
             },
             results.hits[0].formatted_result.as_ref().unwrap()
         );
@@ -1073,7 +1181,8 @@ mod tests {
                 value: S("<em>dolor</em> sit amet, consectetur adipiscing elit"),
                 kind: S("<em>text</em>"),
                 number: 10,
-                nested: Nested { child: S("first") }
+                nested: Nested { child: S("second") },
+                _vectors: None,
             },
             results.hits[0].formatted_result.as_ref().unwrap(),
         );
@@ -1088,7 +1197,8 @@ mod tests {
                 value: S("<em>dolor</em> sit amet, consectetur adipiscing elit"),
                 kind: S("text"),
                 number: 10,
-                nested: Nested { child: S("first") }
+                nested: Nested { child: S("second") },
+                _vectors: None,
             },
             results.hits[0].formatted_result.as_ref().unwrap()
         );
@@ -1274,4 +1384,77 @@ mod tests {
 
         Ok(())
     }
+
+    /// enable vector searching and configure an userProvided embedder
+    async fn setup_hybrid_searching(client: &Client, index: &Index) -> Result<(), Error> {
+        use crate::settings::{Embedder, UserProvidedEmbedderSettings};
+        let embedder_setting =
+            Embedder::UserProvided(UserProvidedEmbedderSettings { dimensions: 1 });
+        let t3 = index
+            .set_settings(&crate::settings::Settings {
+                embedders: Some(HashMap::from([("default".to_string(), embedder_setting)])),
+                ..crate::settings::Settings::default()
+            })
+            .await?;
+        t3.wait_for_completion(&client, None, None).await?;
+        Ok(())
+    }
+
+    #[meilisearch_test]
+    async fn test_with_vectors(client: Client, index: Index) -> Result<(), Error> {
+        setup_hybrid_searching(&client, &index).await?;
+        setup_test_index(&client, &index).await?;
+
+        let results: SearchResults<Document> = index
+            .search()
+            .with_query("lorem ipsum")
+            .with_retrieve_vectors(true)
+            .execute()
+            .await?;
+        assert_eq!(results.hits.len(), 1);
+        let expected = Vectors::from(&[1000.0]);
+        assert_eq!(results.hits[0].result._vectors, Some(expected));
+
+        let results: SearchResults<Document> = index
+            .search()
+            .with_query("lorem ipsum")
+            .with_retrieve_vectors(false)
+            .execute()
+            .await?;
+        assert_eq!(results.hits.len(), 1);
+        assert_eq!(results.hits[0].result._vectors, None);
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_hybrid() -> Result<(), Error> {
+        // this is mocked as I could not get the hybrid searching to work
+        // See https://github.com/meilisearch/meilisearch-rust/pull/554 for further context
+        let mut s = mockito::Server::new_async().await;
+        let mock_server_url = s.url();
+        let client = Client::new(mock_server_url, None::<String>)?;
+        let index = client.index("mocked_index");
+
+        let req = r#"{"q":"hello hybrid searching","hybrid":{"embedder":"default","semanticRatio":0.0},"vector":[1000.0]}"#.to_string();
+        let response = r#"{"hits":[],"offset":null,"limit":null,"estimatedTotalHits":null,"page":null,"hitsPerPage":null,"totalHits":null,"totalPages":null,"facetDistribution":null,"facetStats":null,"processingTimeMs":0,"query":"","indexUid":null}"#.to_string();
+        let mock_res = s
+            .mock("POST", "/indexes/mocked_index/search")
+            .with_status(200)
+            .match_body(mockito::Matcher::Exact(req))
+            .with_body(&response)
+            .expect(1)
+            .create_async()
+            .await;
+        let results: Result<SearchResults<Document>, Error> = index
+            .search()
+            .with_query("hello hybrid searching")
+            .with_hybrid("default", 0.0)
+            .with_vector(&[1000.0])
+            .execute()
+            .await;
+        mock_res.assert_async().await;
+        results?; // purposely not done above to have better debugging output
+
+        Ok(())
+    }
 }