From e2e522966f6e44974402d6b9c592fb8b7e85b764 Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Wed, 20 Mar 2024 14:25:26 -0400 Subject: [PATCH 1/2] first pass at in memory storage suffix searches with domain labels --- Cargo.lock | 62 +++ Cargo.toml | 3 + icann-rdap-srv/Cargo.toml | 1 + .../src/storage/mem/label_search.rs | 361 ++++++++++++++++++ icann-rdap-srv/src/storage/mem/mod.rs | 1 + 5 files changed, 428 insertions(+) create mode 100644 icann-rdap-srv/src/storage/mem/label_search.rs diff --git a/Cargo.lock b/Cargo.lock index 4f7c51e..a232774 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,19 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ab-radix-trie" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f46b50157150f6d6f20e5b5fc783a91f4cdb5e2197b322ed2f595b436b8fd9" +dependencies = [ + "env_logger", + "log", + "rand", + "serde", + "serde_json", +] + [[package]] name = "addr2line" version = "0.21.0" @@ -803,6 +816,19 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + [[package]] name = "envmnt" version = "0.10.4" @@ -1254,6 +1280,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.27" @@ -1435,6 +1467,7 @@ dependencies = [ name = "icann-rdap-srv" version = "0.0.16" dependencies = [ + "ab-radix-trie", "assert_cmd", "async-trait", "axum", @@ -1515,6 +1548,17 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "itertools" version = "0.11.0" @@ -3012,6 +3056,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "termimad" version = "0.26.1" @@ -3541,6 +3594,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 4313d2b..4dc88bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,9 @@ keywords = ["whois", "rdap"] [workspace.dependencies] +# for suffix string searchs +ab-radix-trie = "0.2.0" + # easy error handling anyhow = "1.0" diff --git a/icann-rdap-srv/Cargo.toml b/icann-rdap-srv/Cargo.toml index 02fe2be..cca4478 100644 --- a/icann-rdap-srv/Cargo.toml +++ b/icann-rdap-srv/Cargo.toml @@ -13,6 +13,7 @@ An RDAP Server. icann-rdap-client = { version = "0.0.16", path = "../icann-rdap-client" } icann-rdap-common = { version = "0.0.16", path = "../icann-rdap-common" } +ab-radix-trie.workspace = true async-trait.workspace = true axum.workspace = true axum-extra.workspace = true diff --git a/icann-rdap-srv/src/storage/mem/label_search.rs b/icann-rdap-srv/src/storage/mem/label_search.rs new file mode 100644 index 0000000..5f9430a --- /dev/null +++ b/icann-rdap-srv/src/storage/mem/label_search.rs @@ -0,0 +1,361 @@ +use std::collections::HashMap; + +use ab_radix_trie::Trie; +use buildstructor::Builder; + +use crate::error::RdapServerError; + +/// A structure for searching DNS labels as specified in RFC 9082. +/// For RDAP, type T is likely RdapResponse or Arc. +#[derive(Builder)] +pub struct SearchLabels { + label_suffixes: HashMap>, +} + +impl SearchLabels { + /// Insert a value based on a domain name. + fn insert(&mut self, text: &str, value: T) { + for (i, char) in text.chars().enumerate() { + if char == '.' && i != 0 { + let prefix = &text[..i]; + let suffix = &text[(i + 1)..]; + self.label_suffixes + .entry(suffix.to_owned()) + .or_insert(Trie::new()) + .insert(prefix, Some(value.clone())); + } + } + // the root + self.label_suffixes + .entry(String::default()) + .or_insert(Trie::new()) + .insert(text, Some(value.clone())); + } + + /// Search values based on a label search + fn search(&mut self, search: &str) -> Result, RdapServerError> { + // search string is invalid if it doesn't have only one asterisk ('*') + if search.chars().filter(|c| *c == '*').count() != 1 { + return Err(RdapServerError::InvalidArg( + "Search string must contain one and only one asterisk ('*')".to_string(), + )); + } + // asterisk must not be followed by a character other than dot ('.') + let star = search + .find('*') + .expect("internal error. previous check should have caught this"); + if star != search.chars().count() - 1 + && search + .chars() + .nth(star + 1) + .expect("should have been short circuited") + != '.' + { + return Err(RdapServerError::InvalidArg( + "Search string asterisk ('*') must terminate domain label".to_string(), + )); + } + + let parts = search + .split_once('*') + .expect("internal error. previous check should insure there is an asterisk"); + + // this is a limitation of the trie in that it requires a prefix + // see https://github.com/avnerbarr/radix-trie/issues/1 + if parts.0.is_empty() { + return Err(RdapServerError::InvalidArg( + "Search string must have a prefix".to_string(), + )); + } + + if let Some(trie) = self.label_suffixes.get(parts.1.trim_start_matches('.')) { + if let Some(entries) = trie.get_suffixes_values(parts.0) { + if !entries.is_empty() { + let values = entries + .iter() + .filter_map(|e| e.val.clone()) + .collect::>(); + return Ok(values); + } + } + } + + Ok(vec![]) + } +} + +#[cfg(test)] +#[allow(non_snake_case)] +mod tests { + + use ab_radix_trie::{Entry, Trie}; + + use super::SearchLabels; + + #[test] + fn GIVEN_domain_names_WHEN_inserting_THEN_search_labels_is_correct() { + // GIVEN + let mut search = SearchLabels::builder().build(); + + // WHEN + search.insert("foo.example.com", "foo.example.com".to_owned()); + search.insert("bar.example.com", "bar.example.com".to_owned()); + search.insert("foo.example.net", "foo.example.net".to_owned()); + search.insert("bar.example.net", "bar.example.net".to_owned()); + + // THEN + dbg!(&search.label_suffixes); + assert_eq!(search.label_suffixes.len(), 5); + // root + let root = search.label_suffixes.get("").expect("no root"); + assert_trie( + root, + "foo.example.", + &["foo.example.com", "foo.example.net"], + &["bar.example.com", "bar.example.net"], + ); + assert_trie( + root, + "bar.example.", + &["bar.example.com", "bar.example.net"], + &["foo.example.com", "foo.example.net"], + ); + // com + let com = search.label_suffixes.get("com").expect("no trie"); + assert_trie( + com, + "foo.example", + &["foo.example.com"], + &["bar.example.com", "bar.example.net", "foo.example.net"], + ); + assert_trie( + com, + "bar.example", + &["bar.example.com"], + &["foo.example.com", "foo.example.net", "bar.example.net"], + ); + // net + let net = search.label_suffixes.get("net").expect("no trie"); + assert_trie( + net, + "foo.example", + &["foo.example.net"], + &["bar.example.net", "bar.example.com", "foo.example.com"], + ); + assert_trie( + net, + "bar.example", + &["bar.example.net"], + &["foo.example.com", "foo.example.net", "bar.example.com"], + ); + // example.com + let example_com = search.label_suffixes.get("example.com").expect("no trie"); + assert_trie( + example_com, + "foo", + &["foo.example.com"], + &["bar.example.com", "bar.example.net", "foo.example.net"], + ); + assert_trie( + example_com, + "bar", + &["bar.example.com"], + &["foo.example.com", "foo.example.net", "bar.example.net"], + ); + // example.net + let example_net = search.label_suffixes.get("example.net").expect("no trie"); + assert_trie( + example_net, + "foo", + &["foo.example.net"], + &["bar.example.net", "bar.example.com", "foo.example.com"], + ); + assert_trie( + example_net, + "bar", + &["bar.example.net"], + &["foo.example.com", "foo.example.net", "bar.example.com"], + ); + } + + fn assert_trie(trie: &Trie, suffix: &str, must_have: &[&str], must_not_have: &[&str]) { + let entries = trie + .get_suffixes_values(suffix) + .expect("no values in entries"); + for s in must_have { + assert!( + trie_contains(&entries, s), + "suffix = {suffix} did not find {s}" + ); + } + for s in must_not_have { + assert!(!trie_contains(&entries, s), "suffix = {suffix} found {s}"); + } + } + + fn trie_contains(entries: &[Entry<'_, String>], value: &str) -> bool { + entries + .iter() + .any(|e| e.val.as_ref().expect("no entry value") == value) + } + + #[test] + fn GIVEN_search_string_with_two_asterisks_WHEN_search_THEN_error() { + // GIVEN + let mut labels: SearchLabels = SearchLabels::builder().build(); + let search = "foo.*.*"; + + // WHEN + let actual = labels.search(search); + + // THEN + assert!(actual.is_err()); + } + + #[test] + fn GIVEN_search_string_with_asterisk_suffix_WHEN_search_THEN_error() { + // GIVEN + let mut labels: SearchLabels = SearchLabels::builder().build(); + let search = "foo.*example.net"; + + // WHEN + let actual = labels.search(search); + + // THEN + assert!(actual.is_err()); + } + + #[test] + fn GIVEN_search_string_with_no_asterisk_WHEN_search_THEN_error() { + // GIVEN + let mut labels: SearchLabels = SearchLabels::builder().build(); + let search = "foo.example.net"; + + // WHEN + let actual = labels.search(search); + + // THEN + assert!(actual.is_err()); + } + + #[test] + fn GIVEN_empty_search_string_WHEN_search_THEN_error() { + // GIVEN + let mut labels: SearchLabels = SearchLabels::builder().build(); + let search = ""; + + // WHEN + let actual = labels.search(search); + + // THEN + assert!(actual.is_err()); + } + + #[test] + fn GIVEN_root_search_WHEN_search_THEN_correct_values_found() { + // GIVEN + let mut labels = SearchLabels::builder().build(); + labels.insert("foo.example.com", "foo.example.com".to_owned()); + labels.insert("bar.example.com", "bar.example.com".to_owned()); + labels.insert("foo.example.net", "foo.example.net".to_owned()); + labels.insert("bar.example.net", "bar.example.net".to_owned()); + + // WHEN + let actual = labels.search("foo.example.*").expect("search is invalid"); + + // THEN + dbg!(&actual); + assert_eq!(actual.len(), 2); + assert!(actual.contains(&"foo.example.com".to_string())); + assert!(actual.contains(&"foo.example.net".to_string())); + } + + #[test] + fn GIVEN_root_search_WHEN_search_with_prefix_THEN_correct_values_found() { + // GIVEN + let mut labels = SearchLabels::builder().build(); + labels.insert("foo.example.com", "foo.example.com".to_owned()); + labels.insert("bar.example.com", "bar.example.com".to_owned()); + labels.insert("foo.example.net", "foo.example.net".to_owned()); + labels.insert("bar.example.net", "bar.example.net".to_owned()); + + // WHEN + let actual = labels.search("foo.example.n*").expect("search is invalid"); + + // THEN + dbg!(&actual); + assert_eq!(actual.len(), 1); + assert!(actual.contains(&"foo.example.net".to_string())); + } + + #[test] + fn GIVEN_labels_WHEN_sld_search_with_prefix_THEN_correct_values_found() { + // GIVEN + let mut labels = SearchLabels::builder().build(); + labels.insert("foo.example.com", "foo.example.com".to_owned()); + labels.insert("bar.example.com", "bar.example.com".to_owned()); + labels.insert("foo.example.net", "foo.example.net".to_owned()); + labels.insert("bar.example.net", "bar.example.net".to_owned()); + + // WHEN + let actual = labels.search("foo.ex*.com").expect("search is invalid"); + + // THEN + dbg!(&actual); + assert_eq!(actual.len(), 1); + assert!(actual.contains(&"foo.example.com".to_string())); + } + + #[test] + fn GIVEN_labels_WHEN_3ld_search_with_prefix_THEN_correct_values_found() { + // GIVEN + let mut labels = SearchLabels::builder().build(); + labels.insert("foo.example.com", "foo.example.com".to_owned()); + labels.insert("bar.example.com", "bar.example.com".to_owned()); + labels.insert("foo.example.net", "foo.example.net".to_owned()); + labels.insert("bar.example.net", "bar.example.net".to_owned()); + + // WHEN + let actual = labels.search("fo*.example.com").expect("search is invalid"); + + // THEN + dbg!(&actual); + assert_eq!(actual.len(), 1); + assert!(actual.contains(&"foo.example.com".to_string())); + } + + #[test] + fn GIVEN_labels_WHEN_sld_search_THEN_correct_values_found() { + // GIVEN + let mut labels = SearchLabels::builder().build(); + labels.insert("foo.example.com", "foo.example.com".to_owned()); + labels.insert("bar.example.com", "bar.example.com".to_owned()); + labels.insert("foo.example.net", "foo.example.net".to_owned()); + labels.insert("bar.example.net", "bar.example.net".to_owned()); + + // WHEN + let actual = labels.search("foo.*.com").expect("search is invalid"); + + // THEN + dbg!(&actual); + assert_eq!(actual.len(), 1); + assert!(actual.contains(&"foo.example.com".to_string())); + } + + #[test] + fn GIVEN_labels_WHEN_3ld_search_THEN_error() { + // GIVEN + let mut labels = SearchLabels::builder().build(); + labels.insert("foo.example.com", "foo.example.com".to_owned()); + labels.insert("bar.example.com", "bar.example.com".to_owned()); + labels.insert("foo.example.net", "foo.example.net".to_owned()); + labels.insert("bar.example.net", "bar.example.net".to_owned()); + + // WHEN + let actual = labels.search("*.example.com"); + + // THEN + dbg!(&actual); + assert!(actual.is_err()); + } +} diff --git a/icann-rdap-srv/src/storage/mem/mod.rs b/icann-rdap-srv/src/storage/mem/mod.rs index 04990e2..5a36145 100644 --- a/icann-rdap-srv/src/storage/mem/mod.rs +++ b/icann-rdap-srv/src/storage/mem/mod.rs @@ -1,5 +1,6 @@ #![allow(dead_code)] // TODO remove pub mod config; +mod label_search; pub mod ops; pub mod tx; From d94713601a0dd00ed2a41fcce706721a5cc6df00 Mon Sep 17 00:00:00 2001 From: Andy Newton Date: Sat, 31 Aug 2024 09:48:34 -0400 Subject: [PATCH 2/2] domain search by name now implemented in srv --- Cargo.lock | 4 +- Cargo.toml | 2 +- icann-rdap-cli/tests/integration/queries.rs | 18 +++++ icann-rdap-cli/tests/integration/test_jig.rs | 16 +++- icann-rdap-srv/src/bin/rdap-srv-data.rs | 7 +- icann-rdap-srv/src/bootstrap.rs | 6 +- icann-rdap-srv/src/config.rs | 19 ++++- icann-rdap-srv/src/rdap/domains.rs | 36 +++++++++ icann-rdap-srv/src/rdap/mod.rs | 1 + icann-rdap-srv/src/rdap/router.rs | 3 +- icann-rdap-srv/src/storage/mem/config.rs | 6 +- .../src/storage/mem/label_search.rs | 23 +++--- icann-rdap-srv/src/storage/mem/ops.rs | 43 ++++++++-- icann-rdap-srv/src/storage/mem/tx.rs | 28 ++++++- icann-rdap-srv/src/storage/mod.rs | 18 +++++ icann-rdap-srv/src/storage/pg/config.rs | 3 + icann-rdap-srv/src/storage/pg/ops.rs | 3 + .../tests/integration/srv/domain.rs | 64 ++++++++++++++- .../tests/integration/storage/data.rs | 6 +- .../tests/integration/storage/mem/mod.rs | 78 ++++++++++++++++++- icann-rdap-srv/tests/integration/test_jig.rs | 23 ++++++ 21 files changed, 372 insertions(+), 35 deletions(-) create mode 100644 icann-rdap-srv/src/rdap/domains.rs diff --git a/Cargo.lock b/Cargo.lock index 5be53d6..218b25f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "ab-radix-trie" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20f46b50157150f6d6f20e5b5fc783a91f4cdb5e2197b322ed2f595b436b8fd9" +checksum = "1996a0a7d153953579280bca89e620b9bc0a609727984e53f1f4073776746866" dependencies = [ "env_logger", "log", diff --git a/Cargo.toml b/Cargo.toml index e352dca..5af6127 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ keywords = ["whois", "rdap"] [workspace.dependencies] # for suffix string searchs -ab-radix-trie = "0.2.0" +ab-radix-trie = "0.2.1" # easy error handling anyhow = "1.0" diff --git a/icann-rdap-cli/tests/integration/queries.rs b/icann-rdap-cli/tests/integration/queries.rs index a4dec24..a2abd73 100644 --- a/icann-rdap-cli/tests/integration/queries.rs +++ b/icann-rdap-cli/tests/integration/queries.rs @@ -175,3 +175,21 @@ async fn GIVEN_idn_WHEN_query_a_label_THEN_success() { let assert = test_jig.cmd.assert(); assert.success(); } + +#[tokio::test(flavor = "multi_thread")] +async fn GIVEN_domain_WHEN_search_domain_names_THEN_success() { + // GIVEN + let mut test_jig = TestJig::new_with_enable_domain_name_search().await; + let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); + tx.add_domain(&Domain::basic().ldh_name("foo.example").build()) + .await + .expect("add domain in tx"); + tx.commit().await.expect("tx commit"); + + // WHEN + test_jig.cmd.arg("-t").arg("domain-name").arg("foo.*"); + + // THEN + let assert = test_jig.cmd.assert(); + assert.success(); +} diff --git a/icann-rdap-cli/tests/integration/test_jig.rs b/icann-rdap-cli/tests/integration/test_jig.rs index 5e5db3e..a762784 100644 --- a/icann-rdap-cli/tests/integration/test_jig.rs +++ b/icann-rdap-cli/tests/integration/test_jig.rs @@ -2,7 +2,9 @@ use assert_cmd::Command; use icann_rdap_srv::config::ListenConfig; use icann_rdap_srv::server::AppState; use icann_rdap_srv::server::Listener; +use icann_rdap_srv::storage::mem::config::MemConfig; use icann_rdap_srv::storage::mem::ops::Mem; +use icann_rdap_srv::storage::CommonConfig; use std::time::Duration; use test_dir::DirBuilder; use test_dir::FileType; @@ -18,7 +20,19 @@ pub struct TestJig { impl TestJig { pub async fn new() -> TestJig { - let mem = Mem::default(); + let common_config = CommonConfig::default(); + TestJig::new_common_config(common_config).await + } + + pub async fn new_with_enable_domain_name_search() -> TestJig { + let common_config = CommonConfig::builder() + .domain_search_by_name_enable(true) + .build(); + TestJig::new_common_config(common_config).await + } + + pub async fn new_common_config(common_config: CommonConfig) -> TestJig { + let mem = Mem::new(MemConfig::builder().common_config(common_config).build()); let app_state = AppState { storage: mem.clone(), bootstrap: false, diff --git a/icann-rdap-srv/src/bin/rdap-srv-data.rs b/icann-rdap-srv/src/bin/rdap-srv-data.rs index ec6bd13..08757bf 100644 --- a/icann-rdap-srv/src/bin/rdap-srv-data.rs +++ b/icann-rdap-srv/src/bin/rdap-srv-data.rs @@ -48,6 +48,7 @@ use icann_rdap_srv::storage::data::NetworkOrError; use icann_rdap_srv::storage::data::Template; use icann_rdap_srv::storage::mem::config::MemConfig; use icann_rdap_srv::storage::mem::ops::Mem; +use icann_rdap_srv::storage::CommonConfig; use icann_rdap_srv::storage::StoreOps; use icann_rdap_srv::util::bin::check::check_rdap; use icann_rdap_srv::util::bin::check::to_check_classes; @@ -524,7 +525,11 @@ async fn main() -> Result<(), RdapServerError> { let data_dir = cli.data_dir.clone(); let config = ServiceConfig::non_server().data_dir(&data_dir).build()?; - let storage = Mem::new(MemConfig::builder().build()); + let storage = Mem::new( + MemConfig::builder() + .common_config(CommonConfig::default()) + .build(), + ); storage.init().await?; load_data(&config, &storage, false).await?; diff --git a/icann-rdap-srv/src/bootstrap.rs b/icann-rdap-srv/src/bootstrap.rs index 6b0d194..07224d9 100644 --- a/icann-rdap-srv/src/bootstrap.rs +++ b/icann-rdap-srv/src/bootstrap.rs @@ -379,7 +379,7 @@ mod tests { storage::{ data::load_data, mem::{config::MemConfig, ops::Mem}, - StoreOps, + CommonConfig, StoreOps, }, }; @@ -618,7 +618,9 @@ mod tests { } async fn new_and_init_mem(data_dir: String) -> Mem { - let mem_config = MemConfig::builder().build(); + let mem_config = MemConfig::builder() + .common_config(CommonConfig::default()) + .build(); let mem = Mem::new(mem_config.clone()); mem.init().await.expect("initialzing memeory"); load_data( diff --git a/icann-rdap-srv/src/config.rs b/icann-rdap-srv/src/config.rs index adfd83b..0d33ec8 100644 --- a/icann-rdap-srv/src/config.rs +++ b/icann-rdap-srv/src/config.rs @@ -1,11 +1,11 @@ use buildstructor::Builder; -use envmnt::get_or; +use envmnt::{get_or, get_parse_or}; use strum_macros::Display; use tracing::debug; use crate::{ error::RdapServerError, - storage::{mem::config::MemConfig, pg::config::PgConfig}, + storage::{mem::config::MemConfig, pg::config::PgConfig, CommonConfig}, }; pub const LOG: &str = "RDAP_SRV_LOG"; @@ -17,6 +17,7 @@ pub const DATA_DIR: &str = "RDAP_SRV_DATA_DIR"; pub const AUTO_RELOAD: &str = "RDAP_SRV_AUTO_RELOAD"; pub const BOOTSTRAP: &str = "RDAP_SRV_BOOTSTRAP"; pub const UPDATE_ON_BOOTSTRAP: &str = "RDAP_SRV_UPDATE_ON_BOOTSTRAP"; +pub const DOMAIN_SEARCH_BY_NAME_ENABLE: &str = "RDAP_SRV_DOMAIN_SEARCH_BY_NAME"; pub fn debug_config_vars() { let var_list = [ @@ -29,6 +30,7 @@ pub fn debug_config_vars() { AUTO_RELOAD, BOOTSTRAP, UPDATE_ON_BOOTSTRAP, + DOMAIN_SEARCH_BY_NAME_ENABLE, ]; envmnt::vars() .iter() @@ -65,12 +67,21 @@ pub enum StorageType { impl StorageType { pub fn new_from_env() -> Result { + let domain_search_by_name = get_parse_or(DOMAIN_SEARCH_BY_NAME_ENABLE, false)?; + let common_config = CommonConfig::builder() + .domain_search_by_name_enable(domain_search_by_name) + .build(); let storage = get_or(STORAGE, "memory"); let storage_type = if storage == "memory" { - StorageType::Memory(MemConfig::builder().build()) + StorageType::Memory(MemConfig::builder().common_config(common_config).build()) } else if storage == "postgres" { let db_url = get_or(DB_URL, "postgresql://127.0.0.1/rdap"); - StorageType::Postgres(PgConfig::builder().db_url(db_url).build()) + StorageType::Postgres( + PgConfig::builder() + .db_url(db_url) + .common_config(common_config) + .build(), + ) } else { return Err(RdapServerError::Config(format!( "storage type of '{storage}' is invalid" diff --git a/icann-rdap-srv/src/rdap/domains.rs b/icann-rdap-srv/src/rdap/domains.rs new file mode 100644 index 0000000..2018dec --- /dev/null +++ b/icann-rdap-srv/src/rdap/domains.rs @@ -0,0 +1,36 @@ +use axum::{ + extract::{Query, State}, + response::Response, +}; + +use serde::Deserialize; + +use crate::{error::RdapServerError, rdap::response::ResponseUtil, server::DynServiceState}; + +use super::response::NOT_IMPLEMENTED; + +#[derive(Debug, Deserialize)] +pub(crate) struct DomainsParams { + name: Option, + + #[serde(rename = "nsLdhName")] + _ns_ldh_name: Option, + + #[serde(rename = "nsIp")] + _ns_ip: Option, +} + +#[axum_macros::debug_handler] +#[tracing::instrument(level = "debug")] +pub(crate) async fn domains( + Query(params): Query, + state: State, +) -> Result { + if let Some(name) = params.name { + let storage = state.get_storage().await?; + let results = storage.search_domains_by_name(&name).await?; + Ok(results.response()) + } else { + Ok(NOT_IMPLEMENTED.response()) + } +} diff --git a/icann-rdap-srv/src/rdap/mod.rs b/icann-rdap-srv/src/rdap/mod.rs index 22d7600..27b98b5 100644 --- a/icann-rdap-srv/src/rdap/mod.rs +++ b/icann-rdap-srv/src/rdap/mod.rs @@ -2,6 +2,7 @@ use icann_rdap_common::response::{error::Error, RdapResponse}; pub mod autnum; pub mod domain; +pub mod domains; pub mod entity; pub mod ip; pub mod nameserver; diff --git a/icann-rdap-srv/src/rdap/router.rs b/icann-rdap-srv/src/rdap/router.rs index f55fe1b..235d1fa 100644 --- a/icann-rdap-srv/src/rdap/router.rs +++ b/icann-rdap-srv/src/rdap/router.rs @@ -3,6 +3,7 @@ use axum::{response::IntoResponse, routing::get, Router}; use super::{ autnum::autnum_by_num, domain::domain_by_name, + domains::domains, entity::entity_by_handle, ip::network_by_netid, nameserver::nameserver_by_name, @@ -17,7 +18,7 @@ pub(crate) fn rdap_router() -> Router { .route("/autnum/:asnumber", get(autnum_by_num)) .route("/nameserver/:name", get(nameserver_by_name)) .route("/entity/:handle", get(entity_by_handle)) - .route("/domains", get(not_implemented)) + .route("/domains", get(domains)) .route("/nameservers", get(not_implemented)) .route("/entities", get(not_implemented)) .route("/help", get(srvhelp)) diff --git a/icann-rdap-srv/src/storage/mem/config.rs b/icann-rdap-srv/src/storage/mem/config.rs index eb6e7b4..0637db6 100644 --- a/icann-rdap-srv/src/storage/mem/config.rs +++ b/icann-rdap-srv/src/storage/mem/config.rs @@ -1,4 +1,8 @@ use buildstructor::Builder; +use crate::storage::CommonConfig; + #[derive(Debug, Builder, Clone)] -pub struct MemConfig {} +pub struct MemConfig { + pub common_config: CommonConfig, +} diff --git a/icann-rdap-srv/src/storage/mem/label_search.rs b/icann-rdap-srv/src/storage/mem/label_search.rs index 5f9430a..3004aea 100644 --- a/icann-rdap-srv/src/storage/mem/label_search.rs +++ b/icann-rdap-srv/src/storage/mem/label_search.rs @@ -14,11 +14,17 @@ pub struct SearchLabels { impl SearchLabels { /// Insert a value based on a domain name. - fn insert(&mut self, text: &str, value: T) { - for (i, char) in text.chars().enumerate() { + pub(crate) fn insert(&mut self, text: &str, value: T) { + // char_indices gets the UTF8 indices as well as the character + for (i, char) in text.char_indices() { if char == '.' && i != 0 { let prefix = &text[..i]; - let suffix = &text[(i + 1)..]; + // find the next UTF8 character index + let mut next_i = i + 1; + while !text.is_char_boundary(next_i) { + next_i += 1; + } + let suffix = &text[next_i..]; self.label_suffixes .entry(suffix.to_owned()) .or_insert(Trie::new()) @@ -33,7 +39,7 @@ impl SearchLabels { } /// Search values based on a label search - fn search(&mut self, search: &str) -> Result, RdapServerError> { + pub(crate) fn search(&self, search: &str) -> Result, RdapServerError> { // search string is invalid if it doesn't have only one asterisk ('*') if search.chars().filter(|c| *c == '*').count() != 1 { return Err(RdapServerError::InvalidArg( @@ -61,7 +67,6 @@ impl SearchLabels { .expect("internal error. previous check should insure there is an asterisk"); // this is a limitation of the trie in that it requires a prefix - // see https://github.com/avnerbarr/radix-trie/issues/1 if parts.0.is_empty() { return Err(RdapServerError::InvalidArg( "Search string must have a prefix".to_string(), @@ -202,7 +207,7 @@ mod tests { #[test] fn GIVEN_search_string_with_two_asterisks_WHEN_search_THEN_error() { // GIVEN - let mut labels: SearchLabels = SearchLabels::builder().build(); + let labels: SearchLabels = SearchLabels::builder().build(); let search = "foo.*.*"; // WHEN @@ -215,7 +220,7 @@ mod tests { #[test] fn GIVEN_search_string_with_asterisk_suffix_WHEN_search_THEN_error() { // GIVEN - let mut labels: SearchLabels = SearchLabels::builder().build(); + let labels: SearchLabels = SearchLabels::builder().build(); let search = "foo.*example.net"; // WHEN @@ -228,7 +233,7 @@ mod tests { #[test] fn GIVEN_search_string_with_no_asterisk_WHEN_search_THEN_error() { // GIVEN - let mut labels: SearchLabels = SearchLabels::builder().build(); + let labels: SearchLabels = SearchLabels::builder().build(); let search = "foo.example.net"; // WHEN @@ -241,7 +246,7 @@ mod tests { #[test] fn GIVEN_empty_search_string_WHEN_search_THEN_error() { // GIVEN - let mut labels: SearchLabels = SearchLabels::builder().build(); + let labels: SearchLabels = SearchLabels::builder().build(); let search = ""; // WHEN diff --git a/icann-rdap-srv/src/storage/mem/ops.rs b/icann-rdap-srv/src/storage/mem/ops.rs index 412e93c..68963ab 100644 --- a/icann-rdap-srv/src/storage/mem/ops.rs +++ b/icann-rdap-srv/src/storage/mem/ops.rs @@ -2,18 +2,20 @@ use std::{collections::HashMap, net::IpAddr, str::FromStr, sync::Arc}; use async_trait::async_trait; use btree_range_map::RangeMap; -use icann_rdap_common::response::RdapResponse; +use icann_rdap_common::response::{ + domain::Domain, search::DomainSearchResults, types::Common, RdapResponse, +}; use ipnet::{IpNet, Ipv4Net, Ipv6Net}; use prefix_trie::PrefixMap; use tokio::sync::RwLock; use crate::{ error::RdapServerError, - rdap::response::NOT_FOUND, - storage::{StoreOps, TxHandle}, + rdap::response::{NOT_FOUND, NOT_IMPLEMENTED}, + storage::{CommonConfig, StoreOps, TxHandle}, }; -use super::{config::MemConfig, tx::MemTx}; +use super::{config::MemConfig, label_search::SearchLabels, tx::MemTx}; #[derive(Clone)] pub struct Mem { @@ -21,6 +23,7 @@ pub struct Mem { pub(crate) ip4: Arc>>>, pub(crate) ip6: Arc>>>, pub(crate) domains: Arc>>>, + pub(crate) domains_by_name: Arc>>>, pub(crate) idns: Arc>>>, pub(crate) nameservers: Arc>>>, pub(crate) entities: Arc>>>, @@ -35,6 +38,7 @@ impl Mem { ip4: Arc::new(RwLock::new(PrefixMap::new())), ip6: Arc::new(RwLock::new(PrefixMap::new())), domains: Arc::new(RwLock::new(HashMap::new())), + domains_by_name: Arc::new(RwLock::new(SearchLabels::builder().build())), idns: Arc::new(RwLock::new(HashMap::new())), nameservers: Arc::new(RwLock::new(HashMap::new())), entities: Arc::new(RwLock::new(HashMap::new())), @@ -46,7 +50,11 @@ impl Mem { impl Default for Mem { fn default() -> Self { - Mem::new(MemConfig::builder().build()) + Mem::new( + MemConfig::builder() + .common_config(CommonConfig::default()) + .build(), + ) } } @@ -164,4 +172,29 @@ impl StoreOps for Mem { None => Ok(NOT_FOUND.clone()), } } + + async fn search_domains_by_name(&self, name: &str) -> Result { + if !self.config.common_config.domain_search_by_name_enable { + return Ok(NOT_IMPLEMENTED.clone()); + } + //else + let domains_by_name = self.domains_by_name.read().await; + let results = domains_by_name + .search(name) + .unwrap_or_default() + .into_iter() + .map(Arc::::unwrap_or_clone) + .filter_map(|d| match d { + RdapResponse::Domain(d) => Some(d), + _ => None, + }) + .collect::>(); + let response = RdapResponse::DomainSearchResults( + DomainSearchResults::builder() + .common(Common::new_level0(vec![], vec![])) + .results(results) + .build(), + ); + Ok(response) + } } diff --git a/icann-rdap-srv/src/storage/mem/tx.rs b/icann-rdap-srv/src/storage/mem/tx.rs index 405ee87..221b89f 100644 --- a/icann-rdap-srv/src/storage/mem/tx.rs +++ b/icann-rdap-srv/src/storage/mem/tx.rs @@ -17,7 +17,7 @@ use crate::{ }, }; -use super::ops::Mem; +use super::{label_search::SearchLabels, ops::Mem}; pub struct MemTx { mem: Mem, @@ -25,6 +25,7 @@ pub struct MemTx { ip4: PrefixMap>, ip6: PrefixMap>, domains: HashMap>, + domains_by_name: SearchLabels>, idns: HashMap>, nameservers: HashMap>, entities: HashMap>, @@ -33,12 +34,23 @@ pub struct MemTx { impl MemTx { pub async fn new(mem: &Mem) -> Self { + let domains = Arc::clone(&mem.domains).read_owned().await.clone(); + let mut domains_by_name = SearchLabels::builder().build(); + + // only do load up domain search labels if search by domain names is supported + if mem.config.common_config.domain_search_by_name_enable { + for (name, value) in domains.iter() { + domains_by_name.insert(name, value.clone()); + } + } + Self { mem: mem.clone(), autnums: Arc::clone(&mem.autnums).read_owned().await.clone(), ip4: Arc::clone(&mem.ip4).read_owned().await.clone(), ip6: Arc::clone(&mem.ip6).read_owned().await.clone(), - domains: Arc::clone(&mem.domains).read_owned().await.clone(), + domains, + domains_by_name, idns: Arc::clone(&mem.idns).read_owned().await.clone(), nameservers: Arc::clone(&mem.nameservers).read_owned().await.clone(), entities: Arc::clone(&mem.entities).read_owned().await.clone(), @@ -53,6 +65,7 @@ impl MemTx { ip4: PrefixMap::new(), ip6: PrefixMap::new(), domains: HashMap::new(), + domains_by_name: SearchLabels::builder().build(), idns: HashMap::new(), nameservers: HashMap::new(), entities: HashMap::new(), @@ -101,9 +114,14 @@ impl TxHandle for MemTx { // add the domain by unicodeName if let Some(unicode_name) = domain.unicode_name.as_ref() { - self.idns.insert(unicode_name.to_owned(), domain_response); + self.idns + .insert(unicode_name.to_owned(), domain_response.clone()); }; + if self.mem.config.common_config.domain_search_by_name_enable { + self.domains_by_name.insert(ldh_name, domain_response); + } + Ok(()) } @@ -282,6 +300,10 @@ impl TxHandle for MemTx { let mut domains_g = self.mem.domains.write().await; std::mem::swap(&mut self.domains, &mut domains_g); + //domains by name + let mut domains_by_name_g = self.mem.domains_by_name.write().await; + std::mem::swap(&mut self.domains_by_name, &mut domains_by_name_g); + //idns let mut idns_g = self.mem.idns.write().await; std::mem::swap(&mut self.idns, &mut idns_g); diff --git a/icann-rdap-srv/src/storage/mod.rs b/icann-rdap-srv/src/storage/mod.rs index ac164e9..f535a28 100644 --- a/icann-rdap-srv/src/storage/mod.rs +++ b/icann-rdap-srv/src/storage/mod.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use buildstructor::Builder; use icann_rdap_common::response::{ autnum::Autnum, domain::Domain, entity::Entity, help::Help, nameserver::Nameserver, network::Network, RdapResponse, @@ -52,6 +53,9 @@ pub trait StoreOps: Send + Sync { /// Get server help. async fn get_srv_help(&self, host: Option<&str>) -> Result; + + /// Search for domains by name. + async fn search_domains_by_name(&self, name: &str) -> Result; } /// Represents a handle to a transaction. @@ -121,3 +125,17 @@ pub trait TxHandle: Send { /// Rollback the transaction. async fn rollback(self: Box) -> Result<(), RdapServerError>; } + +/// Common configuration for storage back ends. +#[derive(Debug, Clone, Copy, Builder)] +pub struct CommonConfig { + pub domain_search_by_name_enable: bool, +} + +impl Default for CommonConfig { + fn default() -> Self { + CommonConfig { + domain_search_by_name_enable: true, + } + } +} diff --git a/icann-rdap-srv/src/storage/pg/config.rs b/icann-rdap-srv/src/storage/pg/config.rs index bb76a90..94051df 100644 --- a/icann-rdap-srv/src/storage/pg/config.rs +++ b/icann-rdap-srv/src/storage/pg/config.rs @@ -1,6 +1,9 @@ use buildstructor::Builder; +use crate::storage::CommonConfig; + #[derive(Debug, Builder, Clone)] pub struct PgConfig { pub db_url: String, + pub common_config: CommonConfig, } diff --git a/icann-rdap-srv/src/storage/pg/ops.rs b/icann-rdap-srv/src/storage/pg/ops.rs index cad3148..d3b5207 100644 --- a/icann-rdap-srv/src/storage/pg/ops.rs +++ b/icann-rdap-srv/src/storage/pg/ops.rs @@ -70,4 +70,7 @@ impl StoreOps for Pg { async fn get_srv_help(&self, _host: Option<&str>) -> Result { todo!() } + async fn search_domains_by_name(&self, _name: &str) -> Result { + todo!() + } } diff --git a/icann-rdap-srv/tests/integration/srv/domain.rs b/icann-rdap-srv/tests/integration/srv/domain.rs index 7a17b8d..b3d2413 100644 --- a/icann-rdap-srv/tests/integration/srv/domain.rs +++ b/icann-rdap-srv/tests/integration/srv/domain.rs @@ -1,11 +1,14 @@ #![allow(non_snake_case)] -use icann_rdap_client::query::{qtype::QueryType, request::rdap_request}; +use icann_rdap_client::{ + query::{qtype::QueryType, request::rdap_request}, + RdapClientError, +}; use icann_rdap_common::{ client::{create_client, ClientConfig}, response::domain::Domain, }; -use icann_rdap_srv::storage::StoreOps; +use icann_rdap_srv::storage::{CommonConfig, StoreOps}; use crate::test_jig::SrvTestJig; @@ -63,3 +66,60 @@ async fn GIVEN_server_with_idn_WHEN_query_domain_THEN_status_code_200() { // THEN assert_eq!(response.http_data.status_code, 200); } + +#[tokio::test] +async fn GIVEN_server_with_domain_and_search_disabled_WHEN_query_domain_THEN_status_code_501() { + // GIVEN + let common_config = CommonConfig::builder() + .domain_search_by_name_enable(false) + .build(); + let test_srv = SrvTestJig::new_common_config(common_config).await; + let mut tx = test_srv.mem.new_tx().await.expect("new transaction"); + tx.add_domain(&Domain::basic().ldh_name("foo.example").build()) + .await + .expect("add domain in tx"); + tx.commit().await.expect("tx commit"); + + // WHEN + let client_config = ClientConfig::builder() + .https_only(false) + .follow_redirects(false) + .build(); + let client = create_client(&client_config).expect("creating client"); + let query = QueryType::DomainNameSearch("foo.*".to_string()); + let response = rdap_request(&test_srv.rdap_base, &query, &client).await; + + // THEN + let RdapClientError::Client(error) = response.expect_err("not an error response") else { + panic!("the error was not an HTTP error") + }; + assert_eq!(error.status().expect("no status code"), 501); +} + +#[tokio::test] +async fn GIVEN_server_with_domain_and_search_enabled_WHEN_query_domain_THEN_status_code_200() { + // GIVEN + let common_config = CommonConfig::builder() + .domain_search_by_name_enable(true) + .build(); + let test_srv = SrvTestJig::new_common_config(common_config).await; + let mut tx = test_srv.mem.new_tx().await.expect("new transaction"); + tx.add_domain(&Domain::basic().ldh_name("foo.example").build()) + .await + .expect("add domain in tx"); + tx.commit().await.expect("tx commit"); + + // WHEN + let client_config = ClientConfig::builder() + .https_only(false) + .follow_redirects(false) + .build(); + let client = create_client(&client_config).expect("creating client"); + let query = QueryType::DomainNameSearch("foo.*".to_string()); + let response = rdap_request(&test_srv.rdap_base, &query, &client) + .await + .expect("quering server"); + + // THEN + assert_eq!(response.http_data.status_code, 200); +} diff --git a/icann-rdap-srv/tests/integration/storage/data.rs b/icann-rdap-srv/tests/integration/storage/data.rs index b2cde80..afe176c 100644 --- a/icann-rdap-srv/tests/integration/storage/data.rs +++ b/icann-rdap-srv/tests/integration/storage/data.rs @@ -19,13 +19,15 @@ use icann_rdap_srv::{ NetworkId, NetworkIdType, NetworkOrError::NetworkObject, Template, }, mem::{config::MemConfig, ops::Mem}, - StoreOps, + CommonConfig, StoreOps, }, }; use test_dir::{DirBuilder, TestDir}; async fn new_and_init_mem(data_dir: String) -> Mem { - let mem_config = MemConfig::builder().build(); + let mem_config = MemConfig::builder() + .common_config(CommonConfig::default()) + .build(); let mem = Mem::new(mem_config.clone()); mem.init().await.expect("initialzing memeory"); load_data( diff --git a/icann-rdap-srv/tests/integration/storage/mem/mod.rs b/icann-rdap-srv/tests/integration/storage/mem/mod.rs index b7811c6..0de0925 100644 --- a/icann-rdap-srv/tests/integration/storage/mem/mod.rs +++ b/icann-rdap-srv/tests/integration/storage/mem/mod.rs @@ -10,7 +10,10 @@ use icann_rdap_common::response::{ types::{Common, Notice, NoticeOrRemark, ObjectCommon}, RdapResponse, }; -use icann_rdap_srv::storage::{mem::ops::Mem, StoreOps}; +use icann_rdap_srv::storage::{ + mem::{config::MemConfig, ops::Mem}, + CommonConfig, StoreOps, +}; use rstest::rstest; #[tokio::test] @@ -95,6 +98,79 @@ async fn GIVEN_domain_in_mem_WHEN_lookup_domain_by_unicode_THEN_domain_returned( ) } +#[tokio::test] +async fn GIVEN_domain_in_mem_WHEN_search_domain_by_name_THEN_domain_returned() { + // GIVEN + let mem = Mem::default(); + let mut tx = mem.new_tx().await.expect("new transaction"); + tx.add_domain( + &Domain::idn() + .unicode_name("foo.example.com") + .ldh_name("foo.example.com") + .build(), + ) + .await + .expect("add domain in tx"); + tx.commit().await.expect("tx commit"); + + // WHEN + let actual = mem + .search_domains_by_name("foo.example.*") + .await + .expect("getting domain by unicode"); + + // THEN + let RdapResponse::DomainSearchResults(domains) = actual else { + panic!() + }; + assert_eq!(domains.clone().results.len(), 1); + assert_eq!( + domains + .results + .first() + .expect("at least one") + .unicode_name + .as_ref() + .expect("unicodeName is none"), + "foo.example.com" + ) +} + +#[tokio::test] +async fn GIVEN_domain_in_mem_but_search_not_enabled_WHEN_search_domain_by_name_THEN_not_implemented( +) { + // GIVEN + let mem_config = MemConfig::builder() + .common_config( + CommonConfig::builder() + .domain_search_by_name_enable(false) + .build(), + ) + .build(); + let mem = Mem::new(mem_config); + let mut tx = mem.new_tx().await.expect("new transaction"); + tx.add_domain( + &Domain::idn() + .unicode_name("foo.example.com") + .ldh_name("foo.example.com") + .build(), + ) + .await + .expect("add domain in tx"); + tx.commit().await.expect("tx commit"); + + // WHEN + let actual = mem + .search_domains_by_name("foo.example.*") + .await + .expect("getting domain by unicode"); + + // THEN + let RdapResponse::ErrorResponse(_e) = actual else { + panic!() + }; +} + #[tokio::test] async fn GIVEN_no_domain_in_mem_WHEN_lookup_domain_by_ldh_THEN_404_returned() { // GIVEN diff --git a/icann-rdap-srv/tests/integration/test_jig.rs b/icann-rdap-srv/tests/integration/test_jig.rs index 2584d4c..eb0e8d1 100644 --- a/icann-rdap-srv/tests/integration/test_jig.rs +++ b/icann-rdap-srv/tests/integration/test_jig.rs @@ -2,7 +2,9 @@ use assert_cmd::Command; use icann_rdap_srv::config::ListenConfig; use icann_rdap_srv::server::AppState; use icann_rdap_srv::server::Listener; +use icann_rdap_srv::storage::mem::config::MemConfig; use icann_rdap_srv::storage::mem::ops::Mem; +use icann_rdap_srv::storage::CommonConfig; use std::time::Duration; use test_dir::DirBuilder; use test_dir::TestDir; @@ -95,6 +97,27 @@ impl SrvTestJig { SrvTestJig { mem, rdap_base } } + pub async fn new_common_config(common_config: CommonConfig) -> SrvTestJig { + let mem_config = MemConfig::builder().common_config(common_config).build(); + let mem = Mem::new(mem_config); + let app_state = AppState { + storage: mem.clone(), + bootstrap: false, + }; + let _ = tracing_subscriber::fmt().with_test_writer().try_init(); + let listener = Listener::listen(&ListenConfig::default()) + .await + .expect("listening on interface"); + let rdap_base = listener.rdap_base(); + tokio::spawn(async move { + listener + .start_with_state(app_state) + .await + .expect("starting server"); + }); + SrvTestJig { mem, rdap_base } + } + pub async fn new_bootstrap() -> SrvTestJig { let mem = Mem::default(); let app_state = AppState {