From 701aba4a40573a2d89c7e258d12cf01b8a9d53aa Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Mon, 18 Dec 2023 08:11:25 -0500 Subject: [PATCH 01/11] small update to release process --- release.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release.md b/release.md index fb7f0d7..8494409 100644 --- a/release.md +++ b/release.md @@ -6,6 +6,7 @@ with proper GitHub and crates.io credentials. Without both, this will fail. 1. Install the Cargo release plugin (if it is not already installed): `cargo install cargo-release` +1. Go to the 'dev' branch and get the latest changes: `git switch dev` and then `git pull`. 1. On the 'dev' branch, use the cargo release plugin to bump either the patch, minor, or major version: `cargo release version patch -x`, `cargo release version minor -x` or `cargo release version major -x`. 1. Run the tests: `cargo test` 1. Commit these changes to git. From 1dfb5279e912ad1f552ba4b660be9fbe02cebc5a Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Tue, 5 Mar 2024 11:41:18 -0500 Subject: [PATCH 02/11] change the default for paging to be none --- icann-rdap-cli/README.md | 20 +++++++++++++++----- icann-rdap-cli/src/main.rs | 2 +- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/icann-rdap-cli/README.md b/icann-rdap-cli/README.md index 8f50137..c7b9cb5 100644 --- a/icann-rdap-cli/README.md +++ b/icann-rdap-cli/README.md @@ -58,12 +58,22 @@ For more advanced usage, run `rdap --help` which should yield the extensive help Paging Output ------------- -By default, the client will attempt to determine if paging the output (showing information one page at a time) -is appropriate. This is done by attempting to determine if the terminal is interactive or not. If the terminal -is not interactive, paging will be turned off otherwise it will be on. +The client has a built-in (embedded) pager. Use of this pager is controlled via the `RDAP_PAGING` +environment variable and the `-P` command argument. + +It takes three values: + +* "embedded" - use the built-in pager +* "auto" - use the built-in pager if the program is being run from a terminal +* "none" - use no paging -You can explicitly control this behavior using the `-P` command argument such as `-P none` to specify no paging. -This is also controlled via the `RDAP_PAGING` environmental variable (see configuration below). +For example, `-P embedded` will default to using the built-in pager. + +By default, the client will not use a pager. + +When set to "auto", the client determines if a pager is appropriate. +This is done by attempting to determine if the terminal is interactive or not. If the terminal +is not interactive, paging will be turned off otherwise it will be on. Output Format ------------- diff --git a/icann-rdap-cli/src/main.rs b/icann-rdap-cli/src/main.rs index c28dcaf..63b289d 100644 --- a/icann-rdap-cli/src/main.rs +++ b/icann-rdap-cli/src/main.rs @@ -148,7 +148,7 @@ struct Cli { required = false, env = "RDAP_PAGING", value_enum, - default_value_t = PagerType::Auto, + default_value_t = PagerType::None, )] page_output: PagerType, From d613bdeed0baff82bc6870744270947505c1620f Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Tue, 5 Mar 2024 13:56:58 -0500 Subject: [PATCH 03/11] accounting for the iana dns file to have root in it. --- icann-rdap-common/src/iana.rs | 40 ++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/icann-rdap-common/src/iana.rs b/icann-rdap-common/src/iana.rs index 5b96149..699b8ec 100644 --- a/icann-rdap-common/src/iana.rs +++ b/icann-rdap-common/src/iana.rs @@ -84,7 +84,8 @@ impl BootstrapRegistry for IanaRegistry { .first() .ok_or(BootstrapRegistryError::EmptyService)?; for tld in tlds { - if ldh.ends_with(tld) { + // if the ldh domain ends with the tld or the tld is the empty string which means the root + if ldh.ends_with(tld) || tld.eq("") { let urls = service.last().ok_or(BootstrapRegistryError::EmptyUrlSet)?; let longest = longest_match.get_or_insert_with(|| (tld.len(), urls.to_owned())); if longest.0 < tld.len() { @@ -440,6 +441,43 @@ mod tests { ); } + #[test] + fn GIVEN_domain_bootstrap_with_root_WHEN_find_THEN_url_matches() { + // GIVEN + let bootstrap = r#" + { + "version": "1.0", + "publication": "2024-01-07T10:11:12Z", + "description": "Some text", + "services": [ + [ + ["net", "com"], + [ + "https://registry.example.com/myrdap/" + ] + ], + [ + [""], + [ + "https://example.org/" + ] + ] + ] + } + "#; + let iana = + serde_json::from_str::(bootstrap).expect("cannot parse domain bootstrap"); + + // WHEN + let actual = iana.get_dns_bootstrap_urls("foo.org"); + + // THEN + assert_eq!( + actual.expect("no vec").first().expect("vec is empty"), + "https://example.org/" + ); + } + #[test] fn GIVEN_autnum_bootstrap_with_match_WHEN_find_with_string_THEN_return_match() { // GIVEN From 5fc8207bc5bfc11da0afa610a2d445a35ea82fdb Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Wed, 6 Mar 2024 13:44:25 -0500 Subject: [PATCH 04/11] code to check IDNs --- Cargo.lock | 1 + Cargo.toml | 3 ++ icann-rdap-common/Cargo.toml | 1 + icann-rdap-common/src/check/domain.rs | 68 ++++++++++++++++++++++++ icann-rdap-common/src/check/items.rs | 20 +++++++ icann-rdap-common/src/check/mod.rs | 8 +++ icann-rdap-common/src/check/string.rs | 27 ++++++++++ icann-rdap-common/src/response/domain.rs | 55 ++++++++++++++++++- 8 files changed, 182 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index dfcbe31..2cb91c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1417,6 +1417,7 @@ dependencies = [ "chrono", "cidr-utils 0.6.1", "const_format", + "idna", "ipnet", "lazy_static", "prefix-trie", diff --git a/Cargo.toml b/Cargo.toml index f86d13d..3693b3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,9 @@ http = "1.0" # hyper (http implementation used by axum) hyper = { version = "1.0", features = ["full"] } +# internationalized domain names for applications +idna = "0.5" + # for use prefixmap ipnet = { version = "2.9", features = ["json"] } diff --git a/icann-rdap-common/Cargo.toml b/icann-rdap-common/Cargo.toml index 4447f52..e634ca0 100644 --- a/icann-rdap-common/Cargo.toml +++ b/icann-rdap-common/Cargo.toml @@ -13,6 +13,7 @@ chrono.workspace = true cidr-utils.workspace = true const_format.workspace = true buildstructor.workspace = true +idna.workspace = true ipnet.workspace = true lazy_static.workspace = true prefix-trie.workspace = true diff --git a/icann-rdap-common/src/check/domain.rs b/icann-rdap-common/src/check/domain.rs index 577aa64..80501c1 100644 --- a/icann-rdap-common/src/check/domain.rs +++ b/icann-rdap-common/src/check/domain.rs @@ -52,6 +52,27 @@ impl GetChecks for Domain { { items.push(CheckItem::documentation_name()) } + + // if there is also a unicodeName + if let Some(unicode_name) = &self.unicode_name { + let expected = idna::domain_to_ascii(unicode_name); + if let Ok(expected) = expected { + if !expected.eq_ignore_ascii_case(ldh) { + items.push(CheckItem::unicode_does_not_match_ldh()) + } + } + } + } + + // check unicode_name + if let Some(unicode_name) = &self.unicode_name { + if !unicode_name.is_unicode_domain_name() { + items.push(CheckItem::invalid_unicode_domain_name()); + } + let expected = idna::domain_to_ascii(unicode_name); + if expected.is_err() { + items.push(CheckItem::invalid_unicode_name()); + } } Checks { @@ -93,4 +114,51 @@ mod tests { .iter() .any(|c| c.check == Check::InvalidLdhName)); } + + #[rstest] + #[case("")] + #[case(" ")] + fn GIVEN_domain_with_bad_unicode_WHEN_checked_THEN_invalid_ldh(#[case] unicode: &str) { + // GIVEN + let domain = Domain::idn().unicode_name(unicode).build(); + let rdap = RdapResponse::Domain(domain); + + // WHEN + let checks = rdap.get_checks(CheckParams { + do_subchecks: true, + root: &rdap, + parent_type: rdap.get_type(), + }); + + // THEN + dbg!(&checks); + assert!(checks + .items + .iter() + .any(|c| c.check == Check::InvalidUnicodeDomainName)); + } + + #[test] + fn GIVEN_domain_with_mismatch_ldh_and_unicode_WHEN_checked_THEN_unicode_does_not_match() { + // GIVEN + let domain = Domain::idn() + .unicode_name("foo.com") + .ldh_name("xn--foo.com") + .build(); + let rdap = RdapResponse::Domain(domain); + + // WHEN + let checks = rdap.get_checks(CheckParams { + do_subchecks: true, + root: &rdap, + parent_type: rdap.get_type(), + }); + + // THEN + dbg!(&checks); + assert!(checks + .items + .iter() + .any(|c| c.check == Check::UnicodeDoesNotMatchLdh)); + } } diff --git a/icann-rdap-common/src/check/items.rs b/icann-rdap-common/src/check/items.rs index 540578b..373c6c9 100644 --- a/icann-rdap-common/src/check/items.rs +++ b/icann-rdap-common/src/check/items.rs @@ -115,6 +115,26 @@ impl CheckItem { check: Check::DocumentataionName, } } + pub fn unicode_does_not_match_ldh() -> CheckItem { + CheckItem { + check_class: CheckClass::SpecificationWarning, + check: Check::UnicodeDoesNotMatchLdh, + } + } + + // Unicode Name + pub fn invalid_unicode_domain_name() -> CheckItem { + CheckItem { + check_class: CheckClass::SpecificationError, + check: Check::InvalidUnicodeDomainName, + } + } + pub fn invalid_unicode_name() -> CheckItem { + CheckItem { + check_class: CheckClass::SpecificationError, + check: Check::InvalidUnicodeName, + } + } // Network or Autnum Name pub fn name_is_empty() -> CheckItem { diff --git a/icann-rdap-common/src/check/mod.rs b/icann-rdap-common/src/check/mod.rs index 38fb222..19d4e60 100644 --- a/icann-rdap-common/src/check/mod.rs +++ b/icann-rdap-common/src/check/mod.rs @@ -193,6 +193,14 @@ pub enum Check { InvalidLdhName, #[strum(message = "Documentation domain name. See RFC 6761")] DocumentataionName, + #[strum(message = "Unicode name does not match LDH")] + UnicodeDoesNotMatchLdh, + + // Unicode Name + #[strum(message = "unicodeName does not appear to be a domain name")] + InvalidUnicodeDomainName, + #[strum(message = "unicodeName does not appear to be valid Unicode")] + InvalidUnicodeName, // Network or Autnum Name #[strum(message = "name appears to be empty or only whitespace")] diff --git a/icann-rdap-common/src/check/string.rs b/icann-rdap-common/src/check/string.rs index ef616d6..6f43874 100644 --- a/icann-rdap-common/src/check/string.rs +++ b/icann-rdap-common/src/check/string.rs @@ -8,6 +8,9 @@ pub trait StringCheck { /// Tests if a string is an LDH doamin name. This is not to be confused with [StringCheck::is_ldh_string], /// which checks individual domain labels. fn is_ldh_domain_name(&self) -> bool; + + /// Tests if a string is a Unicode domain name. + fn is_unicode_domain_name(&self) -> bool; } impl StringCheck for T { @@ -25,6 +28,11 @@ impl StringCheck for T { let s = self.to_string(); s == "." || (!s.is_empty() && s.split_terminator('.').all(|s| s.is_ldh_string())) } + + fn is_unicode_domain_name(&self) -> bool { + let s = self.to_string(); + s == "." || !s.is_whitespace_or_empty() + } } pub trait StringListCheck { @@ -187,4 +195,23 @@ mod tests { // THEN assert_eq!(actual, expected); } + + #[rstest] + #[case("foo", true)] + #[case("", false)] + #[case(".", true)] + #[case("foo.bar", true)] + #[case("foo.bar.", true)] + fn GIVEN_string_WHEN_is_unicode_domain_name_THEN_correct_result( + #[case] test_string: &str, + #[case] expected: bool, + ) { + // GIVEN in parameters + + // WHEN + let actual = test_string.is_unicode_domain_name(); + + // THEN + assert_eq!(actual, expected); + } } diff --git a/icann-rdap-common/src/response/domain.rs b/icann-rdap-common/src/response/domain.rs index 3ea671a..ea000f4 100644 --- a/icann-rdap-common/src/response/domain.rs +++ b/icann-rdap-common/src/response/domain.rs @@ -155,6 +155,7 @@ impl Domain { #[builder(entry = "basic")] pub fn new_ldh>( ldh_name: T, + unicode_name: Option, nameservers: Option>, handle: Option, remarks: Vec, @@ -182,7 +183,59 @@ impl Domain { .and_entities(entities) .build(), ldh_name: Some(ldh_name.into()), - unicode_name: None, + unicode_name, + variants: None, + secure_dns: None, + nameservers, + public_ids: None, + network: None, + } + } + + /// Builds an IDN object. + /// + /// ```rust + /// use icann_rdap_common::response::domain::Domain; + /// use icann_rdap_common::response::types::StatusValue; + /// + /// let domain = Domain::idn() + /// .unicode_name("foo.example.com") + /// .handle("foo_example_com-1") + /// .status("active") + /// .build(); + /// ``` + #[builder(entry = "idn")] + pub fn new_idn>( + ldh_name: Option, + unicode_name: T, + nameservers: Option>, + handle: Option, + remarks: Vec, + links: Vec, + events: Vec, + statuses: Vec, + port_43: Option, + entities: Vec, + notices: Vec, + ) -> Self { + let entities = (!entities.is_empty()).then_some(entities); + let remarks = (!remarks.is_empty()).then_some(remarks); + let links = (!links.is_empty()).then_some(links); + let events = (!events.is_empty()).then_some(events); + let notices = (!notices.is_empty()).then_some(notices); + Self { + common: Common::builder().and_notices(notices).build(), + object_common: ObjectCommon::domain() + .and_handle(handle) + .and_remarks(remarks) + .and_links(links) + .and_events(events) + .and_status(to_option_status(statuses)) + .and_port_43(port_43) + .and_entities(entities) + .build(), + ldh_name, + unicode_name: Some(unicode_name.into()), variants: None, secure_dns: None, nameservers, From 6273a2ab7aeb0029cae862aa75f1002c5ca0cbf8 Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Wed, 6 Mar 2024 15:57:11 -0500 Subject: [PATCH 05/11] server support for IDNs --- Cargo.lock | 1 + icann-rdap-srv/Cargo.toml | 1 + icann-rdap-srv/README.md | 1 + icann-rdap-srv/src/bin/rdap-srv-data.rs | 39 ++++++++++++++++-- icann-rdap-srv/src/rdap/domain.rs | 9 ++++- icann-rdap-srv/src/storage/mem/ops.rs | 12 ++++++ icann-rdap-srv/src/storage/mem/tx.rs | 17 ++++++++ icann-rdap-srv/src/storage/mod.rs | 3 ++ icann-rdap-srv/src/storage/pg/ops.rs | 4 ++ .../tests/integration/bin/rdap_srv_data.rs | 40 +++++++++++++++++++ .../tests/integration/srv/domain.rs | 30 ++++++++++++++ .../tests/integration/storage/mem/mod.rs | 31 ++++++++++++++ 12 files changed, 182 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2cb91c3..e872060 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1452,6 +1452,7 @@ dependencies = [ "hyper 1.0.1", "icann-rdap-client", "icann-rdap-common", + "idna", "ipnet", "lazy_static", "pct-str", diff --git a/icann-rdap-srv/Cargo.toml b/icann-rdap-srv/Cargo.toml index 4bb88cf..1c23497 100644 --- a/icann-rdap-srv/Cargo.toml +++ b/icann-rdap-srv/Cargo.toml @@ -25,6 +25,7 @@ cidr-utils.workspace = true clap.workspace = true dotenv.workspace = true envmnt.workspace = true +idna.workspace = true ipnet.workspace = true headers.workspace = true http.workspace = true diff --git a/icann-rdap-srv/README.md b/icann-rdap-srv/README.md index 4c6d036..100db19 100644 --- a/icann-rdap-srv/README.md +++ b/icann-rdap-srv/README.md @@ -15,6 +15,7 @@ More information on ICANN's role in RDAP can be found [here](https://www.icann.o RDAP core support in this server is as follows: - [X] LDH Domain lookup (`/domain/ldh`) +- [X] IDN U-Label lookup (`/domain/unicode`) - [X] Entity lookup (`/entity/handle`) - [X] Nameserver lookup (`/nameserver/fqdn`) - [X] Autnum lookup (`/autnum/123`) diff --git a/icann-rdap-srv/src/bin/rdap-srv-data.rs b/icann-rdap-srv/src/bin/rdap-srv-data.rs index 3514805..ec6bd13 100644 --- a/icann-rdap-srv/src/bin/rdap-srv-data.rs +++ b/icann-rdap-srv/src/bin/rdap-srv-data.rs @@ -366,7 +366,11 @@ struct DomainArgs { /// Letters-Digits-Hyphen name. #[arg(long)] - ldh: String, + ldh: Option, + + /// IDN U-Label name. + #[arg(long)] + idn: Option, /// Zone is signed. #[arg(long)] @@ -542,7 +546,14 @@ async fn do_the_work( let output = match cli.command { Commands::Entity(args) => make_entity(args, storage).await?, Commands::Nameserver(args) => make_nameserver(args, storage).await?, - Commands::Domain(args) => make_domain(args, storage).await?, + Commands::Domain(args) => { + if args.ldh.is_none() && args.idn.is_none() { + return Err(RdapServerError::InvalidArg( + "domain must specify either LDH or U-Label (idn) options".to_string(), + )); + } + make_domain(args, storage).await? + } Commands::Autnum(args) => make_autnum(args, storage).await?, Commands::Network(args) => make_network(args, storage).await?, Commands::SrvHelp(args) => { @@ -988,7 +999,26 @@ async fn make_domain( args: Box, store: &dyn StoreOps, ) -> Result { - let self_href = QueryType::Domain(args.ldh.to_owned()) + // get ldh from idn u-label if ldh is not given + let ldh; + if let Some(ldh_arg) = args.ldh.as_ref() { + ldh = ldh_arg.to_owned(); + } else if let Some(idn_arg) = args.idn.as_ref() { + ldh = idna::domain_to_ascii(idn_arg) + .map_err(|_| RdapServerError::InvalidArg("Invalid IDN U-Lable".to_string()))?; + } else { + panic!("neither ldh or idn specified. this should have been caught in arg parsing.") + } + + // get unicodeName (idn) from ldh if idn is not given + let unicode_name; + if let Some(idn_arg) = args.idn { + unicode_name = idn_arg; + } else { + unicode_name = idna::domain_to_unicode(&ldh).0; + }; + + let self_href = QueryType::Domain(ldh.to_owned()) .query_url(&args.object_args.base_url) .expect("domain self href"); let secure_dns = if !args.ds.is_empty() @@ -1007,7 +1037,8 @@ async fn make_domain( None }; let domain = Domain::builder() - .ldh_name(args.ldh) + .ldh_name(ldh) + .unicode_name(unicode_name) .and_secure_dns(secure_dns) .and_nameservers(nameservers(store, args.ns).await?) .common( diff --git a/icann-rdap-srv/src/rdap/domain.rs b/icann-rdap-srv/src/rdap/domain.rs index 16f1ee5..25ebf2f 100644 --- a/icann-rdap-srv/src/rdap/domain.rs +++ b/icann-rdap-srv/src/rdap/domain.rs @@ -8,7 +8,7 @@ use crate::{error::RdapServerError, rdap::response::ResponseUtil, server::DynSer use super::ToBootStrap; -/// Gets a domain object by the name path, which can be either A-label or U-lable +/// Gets a domain object by the name path, which can be either A-label or U-label /// according to RFC 9082. #[axum_macros::debug_handler] #[tracing::instrument(level = "debug")] @@ -27,7 +27,12 @@ pub(crate) async fn domain_by_name( // TODO add option to verify it looks like a domain name and return BAD REQUEST if it does not. // not all servers may want to enforce that it has multiple labels, such as an IANA server. let storage = state.get_storage().await?; - let domain = storage.get_domain_by_ldh(&domain_name).await?; + let mut domain = storage.get_domain_by_ldh(&domain_name).await?; + + // if not found in domain names, check if it is an IDN + if !matches!(domain, RdapResponse::Domain(_)) && !domain.is_redirect() { + domain = storage.get_domain_by_unicode(&domain_name).await?; + } if state.get_bootstrap() && !matches!(domain, RdapResponse::Domain(_)) && !domain.is_redirect() { diff --git a/icann-rdap-srv/src/storage/mem/ops.rs b/icann-rdap-srv/src/storage/mem/ops.rs index 07f2475..412e93c 100644 --- a/icann-rdap-srv/src/storage/mem/ops.rs +++ b/icann-rdap-srv/src/storage/mem/ops.rs @@ -21,6 +21,7 @@ pub struct Mem { pub(crate) ip4: Arc>>>, pub(crate) ip6: Arc>>>, pub(crate) domains: Arc>>>, + pub(crate) idns: Arc>>>, pub(crate) nameservers: Arc>>>, pub(crate) entities: Arc>>>, pub(crate) srvhelps: Arc>>>, @@ -34,6 +35,7 @@ impl Mem { ip4: Arc::new(RwLock::new(PrefixMap::new())), ip6: Arc::new(RwLock::new(PrefixMap::new())), domains: Arc::new(RwLock::new(HashMap::new())), + idns: Arc::new(RwLock::new(HashMap::new())), nameservers: Arc::new(RwLock::new(HashMap::new())), entities: Arc::new(RwLock::new(HashMap::new())), srvhelps: Arc::new(RwLock::new(HashMap::new())), @@ -71,6 +73,15 @@ impl StoreOps for Mem { } } + async fn get_domain_by_unicode(&self, unicode: &str) -> Result { + let idns = self.idns.read().await; + let result = idns.get(unicode); + match result { + Some(domain) => Ok(RdapResponse::clone(domain)), + None => Ok(NOT_FOUND.clone()), + } + } + async fn get_entity_by_handle(&self, handle: &str) -> Result { let entities = self.entities.read().await; let result = entities.get(handle); @@ -79,6 +90,7 @@ impl StoreOps for Mem { None => Ok(NOT_FOUND.clone()), } } + async fn get_nameserver_by_ldh(&self, ldh: &str) -> Result { let nameservers = self.nameservers.read().await; let result = nameservers.get(ldh); diff --git a/icann-rdap-srv/src/storage/mem/tx.rs b/icann-rdap-srv/src/storage/mem/tx.rs index 76e86de..2dc241c 100644 --- a/icann-rdap-srv/src/storage/mem/tx.rs +++ b/icann-rdap-srv/src/storage/mem/tx.rs @@ -25,6 +25,7 @@ pub struct MemTx { ip4: PrefixMap>, ip6: PrefixMap>, domains: HashMap>, + idns: HashMap>, nameservers: HashMap>, entities: HashMap>, srvhelps: HashMap>, @@ -38,6 +39,7 @@ impl MemTx { ip4: Arc::clone(&mem.ip4).read_owned().await.clone(), ip6: Arc::clone(&mem.ip6).read_owned().await.clone(), domains: Arc::clone(&mem.domains).read_owned().await.clone(), + idns: Arc::clone(&mem.idns).read_owned().await.clone(), nameservers: Arc::clone(&mem.nameservers).read_owned().await.clone(), entities: Arc::clone(&mem.entities).read_owned().await.clone(), srvhelps: Arc::clone(&mem.srvhelps).read_owned().await.clone(), @@ -51,6 +53,7 @@ impl MemTx { ip4: PrefixMap::new(), ip6: PrefixMap::new(), domains: HashMap::new(), + idns: HashMap::new(), nameservers: HashMap::new(), entities: HashMap::new(), srvhelps: HashMap::new(), @@ -86,6 +89,7 @@ impl TxHandle for MemTx { } async fn add_domain(&mut self, domain: &Domain) -> Result<(), RdapServerError> { + // add the domain as LDH, which is required. let ldh_name = domain .ldh_name .as_ref() @@ -94,6 +98,15 @@ impl TxHandle for MemTx { ldh_name.to_owned(), Arc::new(RdapResponse::Domain(domain.clone())), ); + + // add the domain by unicodeName + if let Some(unicode_name) = domain.unicode_name.as_ref() { + self.idns.insert( + unicode_name.to_owned(), + Arc::new(RdapResponse::Domain(domain.clone())), + ); + }; + Ok(()) } @@ -272,6 +285,10 @@ impl TxHandle for MemTx { let mut domains_g = self.mem.domains.write().await; std::mem::swap(&mut self.domains, &mut domains_g); + //idns + let mut idns_g = self.mem.idns.write().await; + std::mem::swap(&mut self.idns, &mut idns_g); + // nameservers let mut nameservers_g = self.mem.nameservers.write().await; std::mem::swap(&mut self.nameservers, &mut nameservers_g); diff --git a/icann-rdap-srv/src/storage/mod.rs b/icann-rdap-srv/src/storage/mod.rs index 17e8c53..ac164e9 100644 --- a/icann-rdap-srv/src/storage/mod.rs +++ b/icann-rdap-srv/src/storage/mod.rs @@ -29,6 +29,9 @@ pub trait StoreOps: Send + Sync { /// Get a domain from storage using the 'ldhName' as the key. async fn get_domain_by_ldh(&self, ldh: &str) -> Result; + /// Get a domain from storage using the 'unicodeName' as the key. + async fn get_domain_by_unicode(&self, unicode: &str) -> Result; + /// Get an entity from storage using the 'handle' of the entity as the key. async fn get_entity_by_handle(&self, handle: &str) -> Result; diff --git a/icann-rdap-srv/src/storage/pg/ops.rs b/icann-rdap-srv/src/storage/pg/ops.rs index 81e674b..cad3148 100644 --- a/icann-rdap-srv/src/storage/pg/ops.rs +++ b/icann-rdap-srv/src/storage/pg/ops.rs @@ -45,6 +45,10 @@ impl StoreOps for Pg { todo!() } + async fn get_domain_by_unicode(&self, _unicode: &str) -> Result { + todo!() + } + async fn get_entity_by_handle(&self, _handle: &str) -> Result { todo!() } diff --git a/icann-rdap-srv/tests/integration/bin/rdap_srv_data.rs b/icann-rdap-srv/tests/integration/bin/rdap_srv_data.rs index 19d00f6..8fdcad6 100644 --- a/icann-rdap-srv/tests/integration/bin/rdap_srv_data.rs +++ b/icann-rdap-srv/tests/integration/bin/rdap_srv_data.rs @@ -138,6 +138,46 @@ fn GIVEN_domain_options_WHEN_create_data_THEN_success() { assert.success(); } +#[test] +fn GIVEN_domain_with_idn_WHEN_create_data_THEN_success() { + // GIVEN + let mut test_jig = make_foo1234(); + + // WHEN + test_jig + .cmd + .arg("domain") + .arg("--ldh") + .arg("example.com") + .arg("--idn") + .arg("example.com") + .arg("--registrant") + .arg("foo1234"); + + // THEN + let assert = test_jig.cmd.assert(); + assert.success(); +} + +#[test] +fn GIVEN_idn_WHEN_create_data_THEN_success() { + // GIVEN + let mut test_jig = make_foo1234(); + + // WHEN + test_jig + .cmd + .arg("domain") + .arg("--idn") + .arg("example.com") + .arg("--registrant") + .arg("foo1234"); + + // THEN + let assert = test_jig.cmd.assert(); + assert.success(); +} + #[test] fn GIVEN_autnum_options_WHEN_create_data_THEN_success() { // GIVEN diff --git a/icann-rdap-srv/tests/integration/srv/domain.rs b/icann-rdap-srv/tests/integration/srv/domain.rs index 8b1c01e..7a17b8d 100644 --- a/icann-rdap-srv/tests/integration/srv/domain.rs +++ b/icann-rdap-srv/tests/integration/srv/domain.rs @@ -33,3 +33,33 @@ async fn GIVEN_server_with_domain_WHEN_query_domain_THEN_status_code_200() { // THEN assert_eq!(response.http_data.status_code, 200); } + +#[tokio::test] +async fn GIVEN_server_with_idn_WHEN_query_domain_THEN_status_code_200() { + // GIVEN + let test_srv = SrvTestJig::new().await; + let mut tx = test_srv.mem.new_tx().await.expect("new transaction"); + tx.add_domain( + &Domain::idn() + .unicode_name("café.example") + .ldh_name("cafe.example") + .build(), + ) + .await + .expect("add domain in tx"); + tx.commit().await.expect("tx commit"); + + // WHEN + let client_config = ClientConfig::builder() + .https_only(false) + .follow_redirects(false) + .build(); + let client = create_client(&client_config).expect("creating client"); + let query = QueryType::Domain("café.example".to_string()); + let response = rdap_request(&test_srv.rdap_base, &query, &client) + .await + .expect("quering server"); + + // THEN + assert_eq!(response.http_data.status_code, 200); +} diff --git a/icann-rdap-srv/tests/integration/storage/mem/mod.rs b/icann-rdap-srv/tests/integration/storage/mem/mod.rs index d3aa33d..b7811c6 100644 --- a/icann-rdap-srv/tests/integration/storage/mem/mod.rs +++ b/icann-rdap-srv/tests/integration/storage/mem/mod.rs @@ -64,6 +64,37 @@ async fn GIVEN_domain_in_mem_WHEN_lookup_domain_by_ldh_THEN_domain_returned() { ) } +#[tokio::test] +async fn GIVEN_domain_in_mem_WHEN_lookup_domain_by_unicode_THEN_domain_returned() { + // GIVEN + let mem = Mem::default(); + let mut tx = mem.new_tx().await.expect("new transaction"); + tx.add_domain( + &Domain::idn() + .unicode_name("foo.example") + .ldh_name("foo.example") + .build(), + ) + .await + .expect("add domain in tx"); + tx.commit().await.expect("tx commit"); + + // WHEN + let actual = mem + .get_domain_by_unicode("foo.example") + .await + .expect("getting domain by unicode"); + + // THEN + let RdapResponse::Domain(domain) = actual else { + panic!() + }; + assert_eq!( + domain.unicode_name.as_ref().expect("unicodeName is none"), + "foo.example" + ) +} + #[tokio::test] async fn GIVEN_no_domain_in_mem_WHEN_lookup_domain_by_ldh_THEN_404_returned() { // GIVEN From 8ea43d98fe97f482fce9dc36d1b495a3154c7879 Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Thu, 7 Mar 2024 11:34:34 -0500 Subject: [PATCH 06/11] remove a-label only restriction on cli, add a-label query type --- Cargo.lock | 1 + icann-rdap-cli/src/main.rs | 4 ++++ icann-rdap-cli/tests/integration/queries.rs | 19 +++++++++++++++++++ icann-rdap-client/Cargo.toml | 1 + icann-rdap-client/src/query/qtype.rs | 19 ++++++++++++++++++- icann-rdap-common/src/check/string.rs | 9 ++++++++- 6 files changed, 51 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e872060..9bb89ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1396,6 +1396,7 @@ dependencies = [ "cidr-utils 0.6.1", "const_format", "icann-rdap-common", + "idna", "lazy_static", "pct-str", "regex", diff --git a/icann-rdap-cli/src/main.rs b/icann-rdap-cli/src/main.rs index 63b289d..caecd5c 100644 --- a/icann-rdap-cli/src/main.rs +++ b/icann-rdap-cli/src/main.rs @@ -240,6 +240,9 @@ enum QtypeArg { /// Domain Lookup Domain, + /// A-Label Domain Lookup + ALabel, + /// Entity Lookup Entity, @@ -520,6 +523,7 @@ fn query_type_from_cli(cli: &Cli) -> QueryType { QtypeArg::V6Cidr => QueryType::IpV6Cidr(query_value), QtypeArg::Autnum => QueryType::AsNumber(query_value), QtypeArg::Domain => QueryType::Domain(query_value), + QtypeArg::ALabel => QueryType::ALable(query_value), QtypeArg::Entity => QueryType::Entity(query_value), QtypeArg::Ns => QueryType::Nameserver(query_value), QtypeArg::EntityName => QueryType::EntityNameSearch(query_value), diff --git a/icann-rdap-cli/tests/integration/queries.rs b/icann-rdap-cli/tests/integration/queries.rs index 5608855..a4dec24 100644 --- a/icann-rdap-cli/tests/integration/queries.rs +++ b/icann-rdap-cli/tests/integration/queries.rs @@ -12,6 +12,7 @@ use crate::test_jig::TestJig; #[case("foo.example", "foo.example")] #[case("foo.example", "foo.example.")] #[case("foo.example", "FOO.EXAMPLE")] +#[case("foó.example", "foó.example")] // unicode #[tokio::test(flavor = "multi_thread")] async fn GIVEN_domain_WHEN_query_THEN_success(#[case] db_domain: &str, #[case] q_domain: &str) { // GIVEN @@ -156,3 +157,21 @@ async fn GIVEN_url_WHEN_query_THEN_success() { let assert = test_jig.cmd.assert(); assert.success(); } + +#[tokio::test(flavor = "multi_thread")] +async fn GIVEN_idn_WHEN_query_a_label_THEN_success() { + // GIVEN + let mut test_jig = TestJig::new().await; + let mut tx = test_jig.mem.new_tx().await.expect("new transaction"); + tx.add_domain(&Domain::basic().ldh_name("xn--caf-dma.example").build()) + .await + .expect("add domain in tx"); + tx.commit().await.expect("tx commit"); + + // WHEN + test_jig.cmd.arg("-t").arg("a-label").arg("café.example"); + + // THEN + let assert = test_jig.cmd.assert(); + assert.success(); +} diff --git a/icann-rdap-client/Cargo.toml b/icann-rdap-client/Cargo.toml index 66e7609..5a12fd0 100644 --- a/icann-rdap-client/Cargo.toml +++ b/icann-rdap-client/Cargo.toml @@ -16,6 +16,7 @@ buildstructor.workspace = true cidr-utils.workspace = true chrono.workspace = true const_format.workspace = true +idna.workspace = true lazy_static.workspace = true pct-str.workspace = true regex.workspace = true diff --git a/icann-rdap-client/src/query/qtype.rs b/icann-rdap-client/src/query/qtype.rs index eca1701..3b6a210 100644 --- a/icann-rdap-client/src/query/qtype.rs +++ b/icann-rdap-client/src/query/qtype.rs @@ -1,6 +1,7 @@ use std::{net::IpAddr, str::FromStr}; use cidr_utils::cidr::IpInet; +use icann_rdap_common::check::string::StringCheck; use lazy_static::lazy_static; use pct_str::{PctString, URIReserved}; use regex::Regex; @@ -29,6 +30,9 @@ pub enum QueryType { #[strum(serialize = "Domain Lookup")] Domain(String), + #[strum(serialize = "A-Label Domain Lookup")] + ALable(String), + #[strum(serialize = "Entity Lookup")] Entity(String), @@ -89,6 +93,7 @@ impl QueryType { "{base_url}/domain/{}", PctString::encode(value.chars(), URIReserved) )), + QueryType::ALable(value) => a_label_query(value, base_url), QueryType::Entity(value) => Ok(format!( "{base_url}/entity/{}", PctString::encode(value.chars(), URIReserved) @@ -116,6 +121,14 @@ impl QueryType { } } +fn a_label_query(value: &str, base_url: &str) -> Result { + let domain = idna::domain_to_ascii(value).map_err(|_| RdapClientError::InvalidQueryValue)?; + Ok(format!( + "{base_url}/domain/{}", + PctString::encode(domain.chars(), URIReserved), + )) +} + fn ip_cidr_query(value: &str, base_url: &str) -> Result { let values = value .split_once('/') @@ -167,7 +180,7 @@ impl FromStr for QueryType { } // if it looks like a domain name - if is_ldh_domain(s) { + if is_domain_name(s) { if is_nameserver(s) { return Ok(QueryType::Nameserver(s.to_owned())); } else { @@ -193,6 +206,10 @@ fn is_ldh_domain(text: &str) -> bool { LDH_DOMAIN_RE.is_match(text) } +fn is_domain_name(text: &str) -> bool { + text.contains('.') && text.is_unicode_domain_name() +} + fn is_nameserver(text: &str) -> bool { lazy_static! { static ref NS_RE: Regex = diff --git a/icann-rdap-common/src/check/string.rs b/icann-rdap-common/src/check/string.rs index 6f43874..3d1e9bd 100644 --- a/icann-rdap-common/src/check/string.rs +++ b/icann-rdap-common/src/check/string.rs @@ -31,7 +31,12 @@ impl StringCheck for T { fn is_unicode_domain_name(&self) -> bool { let s = self.to_string(); - s == "." || !s.is_whitespace_or_empty() + s == "." + || (!s.is_empty() + && s.split_terminator('.').all(|s| { + s.chars() + .all(|c| c == '-' || (!c.is_ascii_punctuation() && !c.is_whitespace())) + })) } } @@ -202,6 +207,8 @@ mod tests { #[case(".", true)] #[case("foo.bar", true)] #[case("foo.bar.", true)] + #[case("fo_o.bar.", false)] + #[case("fo o.bar.", false)] fn GIVEN_string_WHEN_is_unicode_domain_name_THEN_correct_result( #[case] test_string: &str, #[case] expected: bool, From 62fd5a0ab8b55f0d56c384bb096941e85cec21f7 Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Thu, 7 Mar 2024 14:28:56 -0500 Subject: [PATCH 07/11] optimization to save memory when using IDNs --- icann-rdap-srv/src/storage/mem/tx.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/icann-rdap-srv/src/storage/mem/tx.rs b/icann-rdap-srv/src/storage/mem/tx.rs index 2dc241c..405ee87 100644 --- a/icann-rdap-srv/src/storage/mem/tx.rs +++ b/icann-rdap-srv/src/storage/mem/tx.rs @@ -89,22 +89,19 @@ impl TxHandle for MemTx { } async fn add_domain(&mut self, domain: &Domain) -> Result<(), RdapServerError> { + let domain_response = Arc::new(RdapResponse::Domain(domain.clone())); + // add the domain as LDH, which is required. let ldh_name = domain .ldh_name .as_ref() .ok_or_else(|| RdapServerError::EmptyIndexData("ldhName".to_string()))?; - self.domains.insert( - ldh_name.to_owned(), - Arc::new(RdapResponse::Domain(domain.clone())), - ); + self.domains + .insert(ldh_name.to_owned(), domain_response.clone()); // add the domain by unicodeName if let Some(unicode_name) = domain.unicode_name.as_ref() { - self.idns.insert( - unicode_name.to_owned(), - Arc::new(RdapResponse::Domain(domain.clone())), - ); + self.idns.insert(unicode_name.to_owned(), domain_response); }; Ok(()) From af52b8c855644d863d85ee81eff0254424ec50f9 Mon Sep 17 00:00:00 2001 From: Adam Guyot Date: Mon, 11 Mar 2024 15:01:31 -0400 Subject: [PATCH 08/11] Adding redaction to RDAP server and display redaction table on client --- .github/workflows/release-rust.yml | 1 + icann-rdap-cli/src/query.rs | 36 +- icann-rdap-cli/tests/integration/cache.rs | 1 + icann-rdap-client/Cargo.toml | 2 +- icann-rdap-client/src/md/autnum.rs | 5 + icann-rdap-client/src/md/domain.rs | 5 + icann-rdap-client/src/md/entity.rs | 4 + icann-rdap-client/src/md/mod.rs | 1 + icann-rdap-client/src/md/nameserver.rs | 5 + icann-rdap-client/src/md/network.rs | 6 + icann-rdap-client/src/md/redacted.rs | 64 +++ icann-rdap-common/src/response/autnum.rs | 3 + icann-rdap-common/src/response/domain.rs | 2 + icann-rdap-common/src/response/entity.rs | 2 + icann-rdap-common/src/response/mod.rs | 30 ++ icann-rdap-common/src/response/nameserver.rs | 2 + icann-rdap-common/src/response/network.rs | 4 + icann-rdap-common/src/response/redacted.rs | 264 ++++++++++++ .../domain_search_with_redaction.json | 70 ++++ .../test_files/lookup_with_redaction.json | 393 ++++++++++++++++++ icann-rdap-common/src/response/types.rs | 15 +- 21 files changed, 897 insertions(+), 18 deletions(-) create mode 100644 icann-rdap-client/src/md/redacted.rs create mode 100644 icann-rdap-common/src/response/redacted.rs create mode 100644 icann-rdap-common/src/response/test_files/domain_search_with_redaction.json create mode 100644 icann-rdap-common/src/response/test_files/lookup_with_redaction.json diff --git a/.github/workflows/release-rust.yml b/.github/workflows/release-rust.yml index 1ab0696..c282419 100644 --- a/.github/workflows/release-rust.yml +++ b/.github/workflows/release-rust.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - cobenian-dev tags: - 'v*.*.*' diff --git a/icann-rdap-cli/src/query.rs b/icann-rdap-cli/src/query.rs index f32b23f..0f7caf7 100644 --- a/icann-rdap-cli/src/query.rs +++ b/icann-rdap-cli/src/query.rs @@ -213,29 +213,33 @@ fn do_output<'a, W: std::io::Write>( }), )?; } - OutputType::Markdown => writeln!( - write, - "{}", - response.rdap.to_md(MdParams { - heading_level: 1, - root: &response.rdap, - parent_type: response.rdap.get_type(), - check_types: &processing_params.check_types, - options: &MdOptions { - text_style_char: '_', - style_in_justify: true, - ..MdOptions::default() - }, - req_data, - }) - )?, + OutputType::Markdown => { + writeln!( + write, + "{}", + response.rdap.to_md(MdParams { + heading_level: 1, + root: &response.rdap, + parent_type: response.rdap.get_type(), + check_types: &processing_params.check_types, + options: &MdOptions { + text_style_char: '_', + style_in_justify: true, + ..MdOptions::default() + }, + req_data, + }) + )?; + } _ => {} // do nothing }; + let checks = response.rdap.get_checks(CheckParams { do_subchecks: true, root: &response.rdap, parent_type: response.rdap.get_type(), }); + let req_res = RequestResponse { checks, req_data, diff --git a/icann-rdap-cli/tests/integration/cache.rs b/icann-rdap-cli/tests/integration/cache.rs index 90f45f9..c8622dd 100644 --- a/icann-rdap-cli/tests/integration/cache.rs +++ b/icann-rdap-cli/tests/integration/cache.rs @@ -23,6 +23,7 @@ async fn GIVEN_domain_with_entity_WHEN_retreived_from_cache_THEN_is_domain() { test_jig.cmd.arg("foo.example"); let output = test_jig.cmd.output().expect("executing domain query"); + let responses: Vec = serde_json::from_slice(&output.stdout).expect("parsing stdout"); let rdap = &responses.first().expect("response is empty").res_data.rdap; diff --git a/icann-rdap-client/Cargo.toml b/icann-rdap-client/Cargo.toml index 5a12fd0..4cb40de 100644 --- a/icann-rdap-client/Cargo.toml +++ b/icann-rdap-client/Cargo.toml @@ -33,4 +33,4 @@ thiserror.workspace = true rstest = "0.17.0" # tokio async runtime -tokio = { version = "1.21", features = [ "full" ] } +tokio = { version = "1.21", features = [ "full" ] } \ No newline at end of file diff --git a/icann-rdap-client/src/md/autnum.rs b/icann-rdap-client/src/md/autnum.rs index aaf2706..8986a7d 100644 --- a/icann-rdap-client/src/md/autnum.rs +++ b/icann-rdap-client/src/md/autnum.rs @@ -76,6 +76,11 @@ impl ToMd for Autnum { .to_md(params.from_parent(typeid)), ); + // redacted + if let Some(redacted) = &self.object_common.redacted { + md.push_str(&redacted.as_slice().to_md(params.from_parent(typeid))); + } + md.push('\n'); md } diff --git a/icann-rdap-client/src/md/domain.rs b/icann-rdap-client/src/md/domain.rs index 976b80c..ee8e65b 100644 --- a/icann-rdap-client/src/md/domain.rs +++ b/icann-rdap-client/src/md/domain.rs @@ -94,6 +94,11 @@ impl ToMd for Domain { md.push_str(&network.to_md(params.next_level())); } + // redacted + if let Some(redacted) = &self.object_common.redacted { + md.push_str(&redacted.as_slice().to_md(params.from_parent(typeid))); + } + md.push('\n'); md } diff --git a/icann-rdap-client/src/md/entity.rs b/icann-rdap-client/src/md/entity.rs index f850a1b..37529ef 100644 --- a/icann-rdap-client/src/md/entity.rs +++ b/icann-rdap-client/src/md/entity.rs @@ -80,6 +80,10 @@ impl ToMd for Entity { .to_md(params.from_parent(typeid)), ); + // redacted + if let Some(redacted) = &self.object_common.redacted { + md.push_str(&redacted.as_slice().to_md(params.from_parent(typeid))); + } md.push('\n'); md } diff --git a/icann-rdap-client/src/md/mod.rs b/icann-rdap-client/src/md/mod.rs index 3ecaf85..8c68a62 100644 --- a/icann-rdap-client/src/md/mod.rs +++ b/icann-rdap-client/src/md/mod.rs @@ -15,6 +15,7 @@ pub mod error; pub mod help; pub mod nameserver; pub mod network; +pub mod redacted; pub mod search; pub mod string; pub mod table; diff --git a/icann-rdap-client/src/md/nameserver.rs b/icann-rdap-client/src/md/nameserver.rs index cc8cc30..42569de 100644 --- a/icann-rdap-client/src/md/nameserver.rs +++ b/icann-rdap-client/src/md/nameserver.rs @@ -75,6 +75,11 @@ impl ToMd for Nameserver { .entities .to_md(params.from_parent(typeid)), ); + + // redacted + if let Some(redacted) = &self.object_common.redacted { + md.push_str(&redacted.as_slice().to_md(params.from_parent(typeid))); + } md.push('\n'); md } diff --git a/icann-rdap-client/src/md/network.rs b/icann-rdap-client/src/md/network.rs index 210384d..b0d8e9a 100644 --- a/icann-rdap-client/src/md/network.rs +++ b/icann-rdap-client/src/md/network.rs @@ -75,6 +75,12 @@ impl ToMd for Network { .entities .to_md(params.from_parent(typeid)), ); + + // redacted + if let Some(redacted) = &self.object_common.redacted { + md.push_str(&redacted.as_slice().to_md(params.from_parent(typeid))); + } + md.push('\n'); md } diff --git a/icann-rdap-client/src/md/redacted.rs b/icann-rdap-client/src/md/redacted.rs new file mode 100644 index 0000000..97bb8be --- /dev/null +++ b/icann-rdap-client/src/md/redacted.rs @@ -0,0 +1,64 @@ +use icann_rdap_common::response::redacted::Redacted; + +use super::{string::StringUtil, table::MultiPartTable, MdOptions, MdParams, ToMd}; + +impl ToMd for &[Redacted] { + fn to_md(&self, params: MdParams) -> String { + let mut md = String::new(); + + // header + let header_text = "Redacted".to_string(); + md.push_str(&header_text.to_header(params.heading_level, params.options)); + + // multipart data + let mut table = MultiPartTable::new(); + table = table.header_ref(&"Fields"); + + for (index, redacted) in self.iter().enumerate() { + let options = MdOptions { + text_style_char: '*', + ..Default::default() + }; + + // make the name bold + let name = "Redaction"; + let b_name = name.to_bold(&options); + // build the table + table = table.and_data_ref(&b_name, &Some((index + 1).to_string())); + + // Get the data itself + let name_data = redacted + .name + .description + .clone() + .or(redacted.name.type_field.clone()); + let pre_path_data = redacted.pre_path.clone(); + let post_path_data = redacted.post_path.clone(); + let replacement_path_data = redacted.replacement_path.clone(); + let path_lang_data = redacted.path_lang.clone(); + let method_data = redacted.method.clone().map(|m| m.to_string()); + let reason_data = redacted.reason.clone().map(|m| m.to_string()); + + // Special case the 'column' fields + table = table + .and_data_ref(&"name".to_title_case(), &name_data) + .and_data_ref(&"prePath".to_title_case(), &pre_path_data) + .and_data_ref(&"postPath".to_title_case(), &post_path_data) + .and_data_ref(&"replacementPath".to_title_case(), &replacement_path_data) + .and_data_ref(&"pathLang".to_title_case(), &path_lang_data) + .and_data_ref(&"method".to_title_case(), &method_data) + .and_data_ref(&"reason".to_title_case(), &reason_data); + + // we don't have these right now but if we put them in later we will need them + // let check_params = CheckParams::from_md(params, typeid); + // let mut checks = redacted.object_common.get_sub_checks(check_params); + // checks.push(redacted.get_checks(check_params)); + // table = checks_to_table(checks, table, params); + } + + // render table + md.push_str(&table.to_md(params)); + md.push('\n'); + md + } +} diff --git a/icann-rdap-common/src/response/autnum.rs b/icann-rdap-common/src/response/autnum.rs index 8546987..4423503 100644 --- a/icann-rdap-common/src/response/autnum.rs +++ b/icann-rdap-common/src/response/autnum.rs @@ -49,6 +49,7 @@ impl Autnum { /// .build(); /// ``` #[builder(entry = "basic")] + pub fn new_autnum( autnum_range: std::ops::Range, handle: Option, @@ -59,6 +60,7 @@ impl Autnum { port_43: Option, entities: Vec, notices: Vec, + redacted: Option>, ) -> Self { let entities = (!entities.is_empty()).then_some(entities); let remarks = (!remarks.is_empty()).then_some(remarks); @@ -75,6 +77,7 @@ impl Autnum { .and_status(to_option_status(statuses)) .and_port_43(port_43) .and_entities(entities) + .and_redacted(redacted) .build(), start_autnum: Some(autnum_range.start), end_autnum: Some(autnum_range.end), diff --git a/icann-rdap-common/src/response/domain.rs b/icann-rdap-common/src/response/domain.rs index ea000f4..8b06a71 100644 --- a/icann-rdap-common/src/response/domain.rs +++ b/icann-rdap-common/src/response/domain.rs @@ -165,6 +165,7 @@ impl Domain { port_43: Option, entities: Vec, notices: Vec, + redacted: Option>, ) -> Self { let entities = (!entities.is_empty()).then_some(entities); let remarks = (!remarks.is_empty()).then_some(remarks); @@ -181,6 +182,7 @@ impl Domain { .and_status(to_option_status(statuses)) .and_port_43(port_43) .and_entities(entities) + .and_redacted(redacted) .build(), ldh_name: Some(ldh_name.into()), unicode_name, diff --git a/icann-rdap-common/src/response/entity.rs b/icann-rdap-common/src/response/entity.rs index 1bc98ab..ad6934a 100644 --- a/icann-rdap-common/src/response/entity.rs +++ b/icann-rdap-common/src/response/entity.rs @@ -75,6 +75,7 @@ impl Entity { roles: Vec, public_ids: Option, notices: Vec, + redacted: Option>, ) -> Self { let roles = (!roles.is_empty()).then_some(roles); let entities = (!entities.is_empty()).then_some(entities); @@ -92,6 +93,7 @@ impl Entity { .and_status(to_option_status(statuses)) .and_port_43(port_43) .and_entities(entities) + .and_redacted(redacted) .build(), vcard_array: contact.map(|c| c.to_vcard()), roles, diff --git a/icann-rdap-common/src/response/mod.rs b/icann-rdap-common/src/response/mod.rs index 00b0bd5..a1af13b 100644 --- a/icann-rdap-common/src/response/mod.rs +++ b/icann-rdap-common/src/response/mod.rs @@ -25,6 +25,7 @@ pub mod error; pub mod help; pub mod nameserver; pub mod network; +pub mod redacted; pub mod search; pub mod types; @@ -219,6 +220,7 @@ impl RdapResponse { RdapResponse::DomainSearchResults(_) => None, RdapResponse::EntitySearchResults(_) => None, RdapResponse::NameserverSearchResults(_) => None, + RdapResponse::ErrorResponse(_) => None, RdapResponse::Help(_) => None, } @@ -281,6 +283,34 @@ mod tests { use super::RdapResponse; + #[test] + fn GIVEN_redaction_response_when_try_from_THEN_response_is_lookup_with_redaction() { + // GIVEN + let expected: Value = + serde_json::from_str(include_str!("test_files/lookup_with_redaction.json")).unwrap(); + + // WHEN + let actual = RdapResponse::try_from(expected).unwrap(); + + // THEN + assert!(matches!(actual, RdapResponse::Domain(_))); + } + + #[test] + fn GIVEN_redaction_response_when_try_from_THEN_response_is_domain_search_results_with_redaction( + ) { + // GIVEN + let expected: Value = + serde_json::from_str(include_str!("test_files/domain_search_with_redaction.json")) + .unwrap(); + + // WHEN + let actual = RdapResponse::try_from(expected).unwrap(); + + // THEN + assert!(matches!(actual, RdapResponse::DomainSearchResults(_))); + } + #[test] fn GIVEN_domain_response_WHEN_try_from_THEN_response_is_domain() { // GIVEN diff --git a/icann-rdap-common/src/response/nameserver.rs b/icann-rdap-common/src/response/nameserver.rs index b4cfab6..ffe0d33 100644 --- a/icann-rdap-common/src/response/nameserver.rs +++ b/icann-rdap-common/src/response/nameserver.rs @@ -90,6 +90,7 @@ impl Nameserver { port_43: Option, entities: Vec, notices: Vec, + redacted: Option>, ) -> Result { let ip_addresses = if !addresses.is_empty() { Some(IpAddresses::basic().addresses(addresses).build()?) @@ -111,6 +112,7 @@ impl Nameserver { .and_status(to_option_status(statuses)) .and_port_43(port_43) .and_entities(entities) + .and_redacted(redacted) .build(), ldh_name: Some(ldh_name.into()), unicode_name: None, diff --git a/icann-rdap-common/src/response/network.rs b/icann-rdap-common/src/response/network.rs index 4fe8a7c..cca8cc1 100644 --- a/icann-rdap-common/src/response/network.rs +++ b/icann-rdap-common/src/response/network.rs @@ -117,6 +117,7 @@ impl Network { port_43: Option, entities: Vec, notices: Vec, + redacted: Option>, ) -> Result { let entities = (!entities.is_empty()).then_some(entities); let remarks = (!remarks.is_empty()).then_some(remarks); @@ -138,6 +139,7 @@ impl Network { port_43, entities, notices, + redacted, ) } @@ -156,6 +158,7 @@ impl Network { port_43: Option, entities: Option>, notices: Option>, + redacted: Option>, ) -> Result { let cidr = IpInet::from_str(&cidr)?; Ok(Self { @@ -171,6 +174,7 @@ impl Network { .and_status(status) .and_port_43(port_43) .and_entities(entities) + .and_redacted(redacted) .build(), start_address: Some(cidr.first_address().to_string()), end_address: Some(cidr.last_address().to_string()), diff --git a/icann-rdap-common/src/response/redacted.rs b/icann-rdap-common/src/response/redacted.rs new file mode 100644 index 0000000..d7a2dce --- /dev/null +++ b/icann-rdap-common/src/response/redacted.rs @@ -0,0 +1,264 @@ +use std::any::TypeId; + +use serde::{Deserialize, Serialize}; + +use crate::check::Checks; + +use std::fmt; + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct Name { + #[serde(rename = "description")] + pub description: Option, + + #[serde(rename = "type")] + pub type_field: Option, +} + +impl Name { + pub fn description(&self) -> Option<&String> { + self.description.as_ref() + } + + pub fn type_field(&self) -> Option<&String> { + self.type_field.as_ref() + } +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct Reason { + #[serde(rename = "description")] + pub description: Option, + + #[serde(rename = "type")] + pub type_field: Option, +} + +impl std::fmt::Display for Reason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let output = self + .description + .clone() + .unwrap_or_else(|| self.type_field.clone().unwrap_or_else(|| "".to_string())); + write!(f, "{}", output) + } +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub enum Method { + Removal, + EmptyValue, + PartialValue, + ReplacementValue, +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub struct Redacted { + #[serde[rename = "name"]] + pub name: Name, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "reason")] + pub reason: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "prePath")] + pub pre_path: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "postPath")] + pub post_path: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "pathLang")] + pub path_lang: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "replacementPath")] + pub replacement_path: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "method")] + pub method: Option, +} + +impl Default for Name { + fn default() -> Self { + Self { + description: Some(String::default()), + type_field: None, + } + } +} + +impl Default for Reason { + fn default() -> Self { + Self { + description: None, + type_field: None, + } + } +} + +impl Default for Method { + fn default() -> Self { + Self::Removal // according to IETF draft this is the default + } +} + +impl fmt::Display for Method { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Method::Removal => write!(f, "Removal"), + Method::EmptyValue => write!(f, "EmptyValue"), + Method::PartialValue => write!(f, "PartialValue"), + Method::ReplacementValue => write!(f, "ReplacementValue"), + } + } +} + +impl Redacted { + pub fn new() -> Self { + Self { + name: Name::default(), + reason: Some(Reason::default()), + pre_path: None, + post_path: None, + path_lang: None, + replacement_path: None, + method: Some(Method::default()), + } + } + + pub fn get_checks( + &self, + _check_params: crate::check::CheckParams<'_>, + ) -> crate::check::Checks<'_> { + Checks { + struct_name: "RDAP Conformance", + items: Vec::new(), + sub_checks: Vec::new(), + } + } + + pub fn get_type(&self) -> std::any::TypeId { + TypeId::of::() + } +} + +#[cfg(test)] +#[allow(non_snake_case)] +mod tests { + use super::*; + + #[test] + fn GIVEN_redaction_WHEN_set_THEN_success() { + // GIVEN + let mut name = Redacted::new(); + name.name = Name { + description: Some("Registry Domain ID".to_string()), + type_field: None, + }; + + // WHEN + let mut redacted = name; + redacted.reason = Some(Reason::default()); + redacted.pre_path = Some("$.handle".to_string()); + redacted.post_path = Some("$.entities[?(@.roles[0]=='registrant".to_string()); + redacted.path_lang = Some("jsonpath".to_string()); + redacted.replacement_path = Some( + "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='contact-uri')]" + .to_string(), + ); + redacted.method = Some(Method::Removal); + + // THEN + assert_eq!( + redacted.name.description, + Some("Registry Domain ID".to_string()) + ); + assert_eq!(redacted.pre_path, Some("$.handle".to_string())); + assert_eq!( + redacted.post_path, + Some("$.entities[?(@.roles[0]=='registrant".to_string()) + ); + assert_eq!(redacted.path_lang, Some("jsonpath".to_string())); + assert_eq!( + redacted.replacement_path, + Some( + "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='contact-uri')]" + .to_string() + ) + ); + assert_eq!(redacted.method, Some(Method::Removal)); + } + + #[test] + fn GIVEN_redaction_WHEN_deserialize_THEN_success() { + // GIVEN + let expected = r#" + { + "name": { + "type": "Registry Domain ID" + }, + "prePath": "$.handle", + "pathLang": "jsonpath", + "postPath": "$.entities[?(@.roles[0]=='registrant", + "replacementPath": "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='contact-uri')]", + "method": "removal", + "reason": { + "description": "Server policy" + } + } + "#; + + let mut name: Redacted = Redacted::new(); + name.name = Name { + type_field: Some("Registry Domain ID".to_string()), + description: None, + }; + + let reason: Reason = Reason { + description: Some("Server policy".to_string()), + type_field: None, + }; + + // WHEN + let mut sample_redact: Redacted = name; + sample_redact.pre_path = Some("$.handle".to_string()); + sample_redact.path_lang = Some("jsonpath".to_string()); + sample_redact.post_path = Some("$.entities[?(@.roles[0]=='registrant".to_string()); + sample_redact.replacement_path = Some( + "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='contact-uri')]" + .to_string(), + ); + sample_redact.method = Some(Method::Removal); + sample_redact.reason = Some(reason); + + let actual: Result = + serde_json::from_str::(expected); + + // THEN + let actual: Redacted = actual.unwrap(); + assert_eq!(actual, sample_redact); // sanity check + assert_eq!( + actual.name.type_field, + Some("Registry Domain ID".to_string()) + ); + assert_eq!(actual.pre_path, Some("$.handle".to_string())); + assert_eq!( + actual.post_path, + Some("$.entities[?(@.roles[0]=='registrant".to_string()) + ); + assert_eq!(actual.path_lang, Some("jsonpath".to_string())); + assert_eq!( + actual.replacement_path, + Some( + "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='contact-uri')]" + .to_string() + ) + ); + assert_eq!(actual.method, Some(Method::Removal)); + } +} diff --git a/icann-rdap-common/src/response/test_files/domain_search_with_redaction.json b/icann-rdap-common/src/response/test_files/domain_search_with_redaction.json new file mode 100644 index 0000000..345fec0 --- /dev/null +++ b/icann-rdap-common/src/response/test_files/domain_search_with_redaction.json @@ -0,0 +1,70 @@ +{ + "rdapConformance": [ + "rdap_level_0", + "redacted" + ], + "domainSearchResults":[ + { + "objectClassName": "domain", + "ldhName": "example1.com", + "links":[ + { + "value":"https://example.com/rdap/domain/example1.com", + "rel":"self", + "href":"https://example.com/rdap/domain/example1.com", + "type":"application/rdap+json" + }, + { + "value":"https://example.com/rdap/domain/example1.com", + "rel":"related", + "href":"https://example.com/rdap/domain/example1.com", + "type":"application/rdap+json" + } + ], + "redacted": [ + { + "name": { + "type": "Registry Domain ID" + }, + "prePath": "$.domainSearchResults[0].handle", + "pathLang": "jsonpath", + "method": "removal", + "reason": { + "type": "Server policy" + } + } + ] + }, + { + "objectClassName": "domain", + "ldhName": "example2.com", + "links":[ + { + "value":"https://example.com/rdap/domain/example2.com", + "rel":"self", + "href":"https://example.com/rdap/domain/example2.com", + "type":"application/rdap+json" + }, + { + "value":"https://example.com/rdap/domain/example2.com", + "rel":"related", + "href":"https://example.com/rdap/domain/example2.com", + "type":"application/rdap+json" + } + ], + "redacted": [ + { + "name": { + "description": "Registry Domain ID" + }, + "prePath": "$.domainSearchResults[1].handle", + "pathLang": "jsonpath", + "method": "removal", + "reason": { + "description": "Server policy" + } + } + ] + } + ] +} diff --git a/icann-rdap-common/src/response/test_files/lookup_with_redaction.json b/icann-rdap-common/src/response/test_files/lookup_with_redaction.json new file mode 100644 index 0000000..2408415 --- /dev/null +++ b/icann-rdap-common/src/response/test_files/lookup_with_redaction.json @@ -0,0 +1,393 @@ +{ + "rdapConformance": [ + "rdap_level_0", + "redacted" + ], + "objectClassName": "domain", + "ldhName": "example.com", + "secureDNS": { + "delegationSigned": false + }, + "notices": [ + { + "title": "Terms of Use", + "description": [ + "Service subject to Terms of Use." + ], + "links": [ + { + "rel": "self", + "href": "https://www.example.com/terms-of-use", + "type": "text/html", + "value": "https://www.example.com/terms-of-use" + } + ] + } + ], + "nameservers": [ + { + "objectClassName": "nameserver", + "ldhName": "ns1.example.com" + }, + { + "objectClassName": "nameserver", + "ldhName": "ns2.example.com" + } + ], + "entities": [ + { + "objectClassName": "entity", + "handle": "123", + "roles": [ + "registrar" + ], + "publicIds": [ + { + "type": "IANA Registrar ID", + "identifier": "1" + } + ], + "vcardArray": [ + "vcard", + [ + [ + "version", + {}, + "text", + "4.0" + ], + [ + "fn", + {}, + "text", + "Example Registrar Inc." + ], + [ + "adr", + {}, + "text", + [ + "", + "Suite 100", + "123 Example Dr.", + "Dulles", + "VA", + "20166-6503", + "US" + ] + ], + [ + "email", + {}, + "text", + "contact@organization.example" + ], + [ + "tel", + { + "type": "voice" + }, + "uri", + "tel:+1.7035555555" + ], + [ + "tel", + { + "type": "fax" + }, + "uri", + "tel:+1.7035555556" + ] + ] + ], + "entities": [ + { + "objectClassName": "entity", + "roles": [ + "abuse" + ], + "vcardArray": [ + "vcard", + [ + [ + "version", + {}, + "text", + "4.0" + ], + [ + "fn", + {}, + "text", + "Abuse Contact" + ], + [ + "email", + {}, + "text", + "abuse@organization.example" + ], + [ + "tel", + { + "type": "voice" + }, + "uri", + "tel:+1.7035555555" + ] + ] + ] + } + ] + }, + { + "objectClassName": "entity", + "handle": "XXXX", + "roles": [ + "registrant" + ], + "vcardArray": [ + "vcard", + [ + [ + "version", + {}, + "text", + "4.0" + ], + [ + "fn", + {}, + "text", + "" + ], + [ + "adr", + {}, + "text", + [ + "", + "", + "", + "", + "QC", + "", + "Canada" + ] + ] + ] + ] + }, + { + "objectClassName": "entity", + "handle": "YYYY", + "roles": [ + "technical" + ], + "vcardArray": [ + "vcard", + [ + [ + "version", + {}, + "text", + "4.0" + ], + [ + "fn", + {}, + "text", + "" + ], + [ + "org", + {}, + "text", + "Example Inc." + ], + [ + "adr", + {}, + "text", + [ + "", + "Suite 1234", + "4321 Rue Somewhere", + "Quebec", + "QC", + "G1V 2M2", + "Canada" + ] + ] + ] + ] + } + ], + "events": [ + { + "eventAction": "registration", + "eventDate": "1997-06-03T00:00:00Z" + }, + { + "eventAction": "last changed", + "eventDate": "2020-05-28T01:35:00Z" + }, + { + "eventAction": "expiration", + "eventDate": "2021-06-03T04:00:00Z" + } + ], + "status": [ + "server delete prohibited", + "server update prohibited", + "server transfer prohibited", + "client transfer prohibited" + ], + "redacted": [ + { + "name": { + "description": "Registry Domain ID" + }, + "prePath": "$.handle", + "pathLang": "jsonpath", + "method": "removal", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Registrant Name" + }, + "postPath": "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='fn')][3]", + "pathLang": "jsonpath", + "method": "emptyValue", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Registrant Organization" + }, + "prePath": "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='org')]", + "pathLang": "jsonpath", + "method": "removal", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Registrant Street" + }, + "postPath": "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='adr')][3][:3]", + "pathLang": "jsonpath", + "method": "emptyValue", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Registrant City" + }, + "postPath": "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='adr')][3][3]", + "pathLang": "jsonpath", + "method": "emptyValue", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Registrant Postal Code" + }, + "postPath": "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='adr')][3][5]", + "pathLang": "jsonpath", + "method": "emptyValue", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Registrant Email" + }, + "prePath": "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='email')]", + "method": "removal", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Registrant Phone" + }, + "prePath": "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[1].type=='voice')]", + "method": "removal", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Technical Name" + }, + "postPath": "$.entities[?(@.roles[0]=='technical')].vcardArray[1][?(@[0]=='fn')][3]", + "method": "emptyValue", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Technical Email" + }, + "prePath": "$.entities[?(@.roles[0]=='technical')].vcardArray[1][?(@[0]=='email')]", + "method": "removal", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Technical Phone" + }, + "prePath": "$.entities[?(@.roles[0]=='technical')].vcardArray[1][?(@[1].type=='voice')]", + "method": "removal", + "reason": { + "description": "Server policy" + } + }, + { + "name": { + "description": "Technical Fax" + }, + "prePath": "$.entities[?(@.roles[0]=='technical')].vcardArray[1][?(@[1].type=='fax')]", + "reason": { + "description": "Client request" + } + }, + { + "name": { + "description": "Administrative Contact" + }, + "prePath": "$.entities[?(@.roles[0]=='administrative')]", + "method": "removal", + "reason": { + "description": "Refer to the technical contact" + } + }, + { + "name": { + "description": "Billing Contact" + }, + "prePath": "$.entities[?(@.roles[0]=='billing')]", + "method": "removal", + "reason": { + "description": "Refer to the registrant contact" + } + } + ] +} \ No newline at end of file diff --git a/icann-rdap-common/src/response/types.rs b/icann-rdap-common/src/response/types.rs index 2b208bd..5cd64a4 100644 --- a/icann-rdap-common/src/response/types.rs +++ b/icann-rdap-common/src/response/types.rs @@ -1,7 +1,7 @@ use buildstructor::Builder; use serde::{Deserialize, Serialize}; -use super::entity::Entity; +use super::{entity::Entity, redacted::Redacted}; /// Represents an RDAP extension identifier. #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] @@ -234,6 +234,9 @@ pub struct ObjectCommon { #[serde(skip_serializing_if = "Option::is_none")] pub entities: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub redacted: Option>, } #[buildstructor::buildstructor] @@ -247,6 +250,7 @@ impl ObjectCommon { status: Option, port_43: Option, entities: Option>, + redacted: Option>, ) -> Self { Self { object_class_name: "domain".to_string(), @@ -257,6 +261,7 @@ impl ObjectCommon { status, port_43, entities, + redacted, } } @@ -269,6 +274,7 @@ impl ObjectCommon { status: Option, port_43: Option, entities: Option>, + redacted: Option>, ) -> Self { Self { object_class_name: "ip network".to_string(), @@ -279,6 +285,7 @@ impl ObjectCommon { status, port_43, entities, + redacted, } } @@ -291,6 +298,7 @@ impl ObjectCommon { status: Option, port_43: Option, entities: Option>, + redacted: Option>, ) -> Self { Self { object_class_name: "autnum".to_string(), @@ -301,6 +309,7 @@ impl ObjectCommon { status, port_43, entities, + redacted, } } @@ -313,6 +322,7 @@ impl ObjectCommon { status: Option, port_43: Option, entities: Option>, + redacted: Option>, ) -> Self { Self { object_class_name: "nameserver".to_string(), @@ -323,6 +333,7 @@ impl ObjectCommon { status, port_43, entities, + redacted, } } @@ -335,6 +346,7 @@ impl ObjectCommon { status: Option, port_43: Option, entities: Option>, + redacted: Option>, ) -> Self { Self { object_class_name: "entity".to_string(), @@ -345,6 +357,7 @@ impl ObjectCommon { status, port_43, entities, + redacted, } } From e0ef335b839c24576729a051e536995e44a15dff Mon Sep 17 00:00:00 2001 From: Adam Guyot Date: Thu, 14 Mar 2024 11:01:46 -0400 Subject: [PATCH 09/11] fixing items of note for PR --- .github/workflows/release-rust.yml | 1 - icann-rdap-cli/tests/integration/cache.rs | 1 - icann-rdap-client/src/md/entity.rs | 1 + icann-rdap-client/src/md/nameserver.rs | 1 + icann-rdap-client/src/md/redacted.rs | 19 +++-- icann-rdap-common/src/response/mod.rs | 1 - icann-rdap-common/src/response/redacted.rs | 93 +++++++++------------- 7 files changed, 48 insertions(+), 69 deletions(-) diff --git a/.github/workflows/release-rust.yml b/.github/workflows/release-rust.yml index c282419..1ab0696 100644 --- a/.github/workflows/release-rust.yml +++ b/.github/workflows/release-rust.yml @@ -5,7 +5,6 @@ on: push: branches: - main - - cobenian-dev tags: - 'v*.*.*' diff --git a/icann-rdap-cli/tests/integration/cache.rs b/icann-rdap-cli/tests/integration/cache.rs index c8622dd..90f45f9 100644 --- a/icann-rdap-cli/tests/integration/cache.rs +++ b/icann-rdap-cli/tests/integration/cache.rs @@ -23,7 +23,6 @@ async fn GIVEN_domain_with_entity_WHEN_retreived_from_cache_THEN_is_domain() { test_jig.cmd.arg("foo.example"); let output = test_jig.cmd.output().expect("executing domain query"); - let responses: Vec = serde_json::from_slice(&output.stdout).expect("parsing stdout"); let rdap = &responses.first().expect("response is empty").res_data.rdap; diff --git a/icann-rdap-client/src/md/entity.rs b/icann-rdap-client/src/md/entity.rs index 37529ef..94ac008 100644 --- a/icann-rdap-client/src/md/entity.rs +++ b/icann-rdap-client/src/md/entity.rs @@ -84,6 +84,7 @@ impl ToMd for Entity { if let Some(redacted) = &self.object_common.redacted { md.push_str(&redacted.as_slice().to_md(params.from_parent(typeid))); } + md.push('\n'); md } diff --git a/icann-rdap-client/src/md/nameserver.rs b/icann-rdap-client/src/md/nameserver.rs index 42569de..f5dd0c0 100644 --- a/icann-rdap-client/src/md/nameserver.rs +++ b/icann-rdap-client/src/md/nameserver.rs @@ -80,6 +80,7 @@ impl ToMd for Nameserver { if let Some(redacted) = &self.object_common.redacted { md.push_str(&redacted.as_slice().to_md(params.from_parent(typeid))); } + md.push('\n'); md } diff --git a/icann-rdap-client/src/md/redacted.rs b/icann-rdap-client/src/md/redacted.rs index 97bb8be..68d3561 100644 --- a/icann-rdap-client/src/md/redacted.rs +++ b/icann-rdap-client/src/md/redacted.rs @@ -32,20 +32,19 @@ impl ToMd for &[Redacted] { .description .clone() .or(redacted.name.type_field.clone()); - let pre_path_data = redacted.pre_path.clone(); - let post_path_data = redacted.post_path.clone(); - let replacement_path_data = redacted.replacement_path.clone(); - let path_lang_data = redacted.path_lang.clone(); - let method_data = redacted.method.clone().map(|m| m.to_string()); - let reason_data = redacted.reason.clone().map(|m| m.to_string()); + let method_data = redacted.method.as_ref().map(|m| m.to_string()); + let reason_data = redacted.reason.as_ref().map(|m| m.to_string()); // Special case the 'column' fields table = table .and_data_ref(&"name".to_title_case(), &name_data) - .and_data_ref(&"prePath".to_title_case(), &pre_path_data) - .and_data_ref(&"postPath".to_title_case(), &post_path_data) - .and_data_ref(&"replacementPath".to_title_case(), &replacement_path_data) - .and_data_ref(&"pathLang".to_title_case(), &path_lang_data) + .and_data_ref(&"prePath".to_title_case(), &redacted.pre_path) + .and_data_ref(&"postPath".to_title_case(), &redacted.post_path) + .and_data_ref( + &"replacementPath".to_title_case(), + &redacted.replacement_path, + ) + .and_data_ref(&"pathLang".to_title_case(), &redacted.path_lang) .and_data_ref(&"method".to_title_case(), &method_data) .and_data_ref(&"reason".to_title_case(), &reason_data); diff --git a/icann-rdap-common/src/response/mod.rs b/icann-rdap-common/src/response/mod.rs index a1af13b..54d5533 100644 --- a/icann-rdap-common/src/response/mod.rs +++ b/icann-rdap-common/src/response/mod.rs @@ -220,7 +220,6 @@ impl RdapResponse { RdapResponse::DomainSearchResults(_) => None, RdapResponse::EntitySearchResults(_) => None, RdapResponse::NameserverSearchResults(_) => None, - RdapResponse::ErrorResponse(_) => None, RdapResponse::Help(_) => None, } diff --git a/icann-rdap-common/src/response/redacted.rs b/icann-rdap-common/src/response/redacted.rs index d7a2dce..ecc8a48 100644 --- a/icann-rdap-common/src/response/redacted.rs +++ b/icann-rdap-common/src/response/redacted.rs @@ -1,11 +1,10 @@ -use std::any::TypeId; - +use buildstructor::Builder; use serde::{Deserialize, Serialize}; +use std::any::TypeId; +use std::fmt; use crate::check::Checks; -use std::fmt; - #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] pub struct Name { #[serde(rename = "description")] @@ -25,7 +24,7 @@ impl Name { } } -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Default)] pub struct Reason { #[serde(rename = "description")] pub description: Option, @@ -36,10 +35,7 @@ pub struct Reason { impl std::fmt::Display for Reason { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let output = self - .description - .clone() - .unwrap_or_else(|| self.type_field.clone().unwrap_or_else(|| "".to_string())); + let output = self.description.clone().unwrap_or_default(); write!(f, "{}", output) } } @@ -53,7 +49,7 @@ pub enum Method { ReplacementValue, } -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +#[derive(Builder, Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] pub struct Redacted { #[serde[rename = "name"]] pub name: Name, @@ -92,15 +88,6 @@ impl Default for Name { } } -impl Default for Reason { - fn default() -> Self { - Self { - description: None, - type_field: None, - } - } -} - impl Default for Method { fn default() -> Self { Self::Removal // according to IETF draft this is the default @@ -119,18 +106,6 @@ impl fmt::Display for Method { } impl Redacted { - pub fn new() -> Self { - Self { - name: Name::default(), - reason: Some(Reason::default()), - pre_path: None, - post_path: None, - path_lang: None, - replacement_path: None, - method: Some(Method::default()), - } - } - pub fn get_checks( &self, _check_params: crate::check::CheckParams<'_>, @@ -155,23 +130,24 @@ mod tests { #[test] fn GIVEN_redaction_WHEN_set_THEN_success() { // GIVEN - let mut name = Redacted::new(); - name.name = Name { + let name = Name { description: Some("Registry Domain ID".to_string()), type_field: None, }; // WHEN - let mut redacted = name; - redacted.reason = Some(Reason::default()); - redacted.pre_path = Some("$.handle".to_string()); - redacted.post_path = Some("$.entities[?(@.roles[0]=='registrant".to_string()); - redacted.path_lang = Some("jsonpath".to_string()); - redacted.replacement_path = Some( - "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='contact-uri')]" - .to_string(), - ); - redacted.method = Some(Method::Removal); + let redacted = Redacted::builder() + .name(name) + .reason(Reason::default()) + .pre_path("$.handle".to_string()) + .post_path("$.entities[?(@.roles[0]=='registrant'".to_string()) + .path_lang("jsonpath".to_string()) + .replacement_path( + "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='contact-uri')]" + .to_string(), + ) + .method(Method::Removal) + .build(); // THEN assert_eq!( @@ -181,7 +157,7 @@ mod tests { assert_eq!(redacted.pre_path, Some("$.handle".to_string())); assert_eq!( redacted.post_path, - Some("$.entities[?(@.roles[0]=='registrant".to_string()) + Some("$.entities[?(@.roles[0]=='registrant'".to_string()) ); assert_eq!(redacted.path_lang, Some("jsonpath".to_string())); assert_eq!( @@ -204,7 +180,7 @@ mod tests { }, "prePath": "$.handle", "pathLang": "jsonpath", - "postPath": "$.entities[?(@.roles[0]=='registrant", + "postPath": "$.entities[?(@.roles[0]=='registrant'", "replacementPath": "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='contact-uri')]", "method": "removal", "reason": { @@ -213,8 +189,8 @@ mod tests { } "#; - let mut name: Redacted = Redacted::new(); - name.name = Name { + // in this one we swap the two fields + let name = Name { type_field: Some("Registry Domain ID".to_string()), description: None, }; @@ -225,14 +201,19 @@ mod tests { }; // WHEN - let mut sample_redact: Redacted = name; - sample_redact.pre_path = Some("$.handle".to_string()); - sample_redact.path_lang = Some("jsonpath".to_string()); - sample_redact.post_path = Some("$.entities[?(@.roles[0]=='registrant".to_string()); - sample_redact.replacement_path = Some( - "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='contact-uri')]" - .to_string(), - ); + // use the builder for most of the fields but not all + let mut sample_redact: Redacted = Redacted::builder() + .name(name) + .pre_path("$.handle".to_string()) + .path_lang("jsonpath".to_string()) + .post_path("$.entities[?(@.roles[0]=='registrant'".to_string()) + .replacement_path( + "$.entities[?(@.roles[0]=='registrant')].vcardArray[1][?(@[0]=='contact-uri')]" + .to_string(), + ) + .build(); + + // also make sure we can set the rest sample_redact.method = Some(Method::Removal); sample_redact.reason = Some(reason); @@ -249,7 +230,7 @@ mod tests { assert_eq!(actual.pre_path, Some("$.handle".to_string())); assert_eq!( actual.post_path, - Some("$.entities[?(@.roles[0]=='registrant".to_string()) + Some("$.entities[?(@.roles[0]=='registrant'".to_string()) ); assert_eq!(actual.path_lang, Some("jsonpath".to_string())); assert_eq!( From 40da4fc3587499c16c4e25153c3f234e2977073f Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Thu, 14 Mar 2024 11:45:29 -0400 Subject: [PATCH 10/11] fix to issue #59 --- icann-rdap-common/src/check/types.rs | 190 ++++++++++++++++++++++++--- 1 file changed, 173 insertions(+), 17 deletions(-) diff --git a/icann-rdap-common/src/check/types.rs b/icann-rdap-common/src/check/types.rs index 1d51e40..735e780 100644 --- a/icann-rdap-common/src/check/types.rs +++ b/icann-rdap-common/src/check/types.rs @@ -84,13 +84,13 @@ impl GetChecks for Link { } else { items.push(CheckItem::self_link_has_no_type()) } - } else if RELATED_AND_SELF_LINK_PARENTS.contains(¶ms.parent_type) || + } else if RELATED_AND_SELF_LINK_PARENTS.contains(¶ms.parent_type) && // because some registries do not model nameservers directly, // they can be embedded in other objects but aren't first class // objects themself (see RIR example in RFC 9083). Therefore, // it only matters that a nameserver has no self link if it is // the top most object (i.e. a first class object). - params.root.get_type() == TypeId::of::() + params.root.get_type() != TypeId::of::() { items.push(CheckItem::object_class_has_no_self_link()) } @@ -269,12 +269,18 @@ impl GetSubChecks for ObjectCommon { mod tests { use rstest::rstest; - use crate::response::{ - domain::Domain, - entity::Entity, - nameserver::Nameserver, - types::{Common, Event, Extension, Link, ObjectCommon, StatusValue}, - RdapResponse, + use crate::{ + check::Checks, + response::{ + domain::Domain, + entity::Entity, + nameserver::Nameserver, + types::{ + Common, Event, Extension, Link, Notice, NoticeOrRemark, ObjectCommon, Remark, + StatusValue, + }, + RdapResponse, + }, }; use crate::check::{Check, CheckParams, GetChecks}; @@ -480,15 +486,151 @@ mod tests { }); // THEN - checks - .sub("Links") - .expect("Links not found") - .sub("Link") - .expect("Link not found") - .items - .iter() - .find(|c| c.check == Check::SelfLinkIsNotRdap) - .expect("link missing check"); + assert!(find_any_check(&checks, Check::SelfLinkIsNotRdap)); + } + + #[test] + fn GIVEN_domain_with_self_link_WHEN_checked_THEN_no_check_found() { + // GIVEN + let rdap = RdapResponse::Domain( + Domain::builder() + .common(Common::builder().build()) + .object_common( + ObjectCommon::domain() + .links(vec![Link::builder() + .href("https://foo") + .rel("self") + .media_type("application/rdap+json") + .build()]) + .build(), + ) + .build(), + ); + + // WHEN + let checks = rdap.get_checks(CheckParams { + do_subchecks: true, + root: &rdap, + parent_type: rdap.get_type(), + }); + + // THEN + dbg!(&checks); + assert!(!find_any_check(&checks, Check::ObjectClassHasNoSelfLink)); + } + + #[test] + fn GIVEN_nameserver_with_self_link_WHEN_checked_THEN_no_check_found() { + // GIVEN + let rdap = RdapResponse::Nameserver( + Nameserver::builder() + .common(Common::builder().build()) + .object_common( + ObjectCommon::domain() + .links(vec![Link::builder() + .href("https://foo") + .rel("self") + .media_type("application/rdap+json") + .build()]) + .build(), + ) + .build(), + ); + + // WHEN + let checks = rdap.get_checks(CheckParams { + do_subchecks: true, + root: &rdap, + parent_type: rdap.get_type(), + }); + + // THEN + assert!(!find_any_check(&checks, Check::ObjectClassHasNoSelfLink)); + } + + #[test] + /// Issue #59 + fn GIVEN_nameserver_with_self_link_and_notice_WHEN_checked_THEN_no_check_found() { + // GIVEN + let rdap = RdapResponse::Nameserver( + Nameserver::builder() + .common( + Common::builder() + .notices(vec![Notice( + NoticeOrRemark::builder() + .description_entry("a notice") + .links(vec![Link::builder() + .href("https://tos") + .rel("terms-of-service") + .media_type("text/html") + .build()]) + .build(), + )]) + .build(), + ) + .object_common( + ObjectCommon::domain() + .links(vec![Link::builder() + .href("https://foo") + .rel("self") + .media_type("application/rdap+json") + .build()]) + .build(), + ) + .build(), + ); + + // WHEN + let checks = rdap.get_checks(CheckParams { + do_subchecks: true, + root: &rdap, + parent_type: rdap.get_type(), + }); + + // THEN + dbg!(&checks); + assert!(!find_any_check(&checks, Check::ObjectClassHasNoSelfLink)); + } + + #[test] + /// Issue #59 + fn GIVEN_nameserver_with_self_link_and_remark_WHEN_checked_THEN_no_check_found() { + // GIVEN + let rdap = RdapResponse::Nameserver( + Nameserver::builder() + .common(Common::builder().build()) + .object_common( + ObjectCommon::domain() + .remarks(vec![Remark( + NoticeOrRemark::builder() + .description_entry("a notice") + .links(vec![Link::builder() + .href("https://tos") + .rel("terms-of-service") + .media_type("text/html") + .build()]) + .build(), + )]) + .links(vec![Link::builder() + .href("https://foo") + .rel("self") + .media_type("application/rdap+json") + .build()]) + .build(), + ) + .build(), + ); + + // WHEN + let checks = rdap.get_checks(CheckParams { + do_subchecks: true, + root: &rdap, + parent_type: rdap.get_type(), + }); + + // THEN + dbg!(&checks); + assert!(!find_any_check(&checks, Check::ObjectClassHasNoSelfLink)); } #[test] @@ -781,4 +923,18 @@ mod tests { .find(|c| c.check == Check::InvalidRdapConformanceParent) .expect("check missing"); } + + fn find_any_check(checks: &Checks, check_type: Check) -> bool { + if checks.items.iter().any(|c| c.check == check_type) { + return true; + } + if checks + .sub_checks + .iter() + .any(|c| find_any_check(c, check_type)) + { + return true; + } + false + } } From a621ebc37542fd5d170e2d664969bf9285187898 Mon Sep 17 00:00:00 2001 From: Andrew Newton Date: Fri, 15 Mar 2024 06:11:26 -0400 Subject: [PATCH 11/11] bumping to 0.0.16 --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- icann-rdap-cli/Cargo.toml | 4 ++-- icann-rdap-client/Cargo.toml | 4 ++-- icann-rdap-srv/Cargo.toml | 4 ++-- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9bb89ba..4f7c51e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1356,7 +1356,7 @@ dependencies = [ [[package]] name = "icann-rdap-cli" -version = "0.0.15" +version = "0.0.16" dependencies = [ "anyhow", "assert_cmd", @@ -1389,7 +1389,7 @@ dependencies = [ [[package]] name = "icann-rdap-client" -version = "0.0.15" +version = "0.0.16" dependencies = [ "buildstructor", "chrono", @@ -1412,7 +1412,7 @@ dependencies = [ [[package]] name = "icann-rdap-common" -version = "0.0.15" +version = "0.0.16" dependencies = [ "buildstructor", "chrono", @@ -1433,7 +1433,7 @@ dependencies = [ [[package]] name = "icann-rdap-srv" -version = "0.0.15" +version = "0.0.16" dependencies = [ "assert_cmd", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 3693b3e..4313d2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.0.15" +version = "0.0.16" edition = "2021" license = "MIT OR Apache-2.0" repository = "https://github.com/icann/icann-rdap" diff --git a/icann-rdap-cli/Cargo.toml b/icann-rdap-cli/Cargo.toml index a0fd921..50e04c9 100644 --- a/icann-rdap-cli/Cargo.toml +++ b/icann-rdap-cli/Cargo.toml @@ -14,8 +14,8 @@ path = "src/main.rs" [dependencies] -icann-rdap-client = { version = "0.0.15", path = "../icann-rdap-client" } -icann-rdap-common = { version = "0.0.15", path = "../icann-rdap-common" } +icann-rdap-client = { version = "0.0.16", path = "../icann-rdap-client" } +icann-rdap-common = { version = "0.0.16", path = "../icann-rdap-common" } anyhow.workspace = true clap.workspace = true diff --git a/icann-rdap-client/Cargo.toml b/icann-rdap-client/Cargo.toml index 4cb40de..bd15b61 100644 --- a/icann-rdap-client/Cargo.toml +++ b/icann-rdap-client/Cargo.toml @@ -10,7 +10,7 @@ An RDAP client library. [dependencies] -icann-rdap-common = { version = "0.0.15", path = "../icann-rdap-common" } +icann-rdap-common = { version = "0.0.16", path = "../icann-rdap-common" } buildstructor.workspace = true cidr-utils.workspace = true @@ -33,4 +33,4 @@ thiserror.workspace = true rstest = "0.17.0" # tokio async runtime -tokio = { version = "1.21", features = [ "full" ] } \ No newline at end of file +tokio = { version = "1.21", features = [ "full" ] } diff --git a/icann-rdap-srv/Cargo.toml b/icann-rdap-srv/Cargo.toml index 1c23497..02fe2be 100644 --- a/icann-rdap-srv/Cargo.toml +++ b/icann-rdap-srv/Cargo.toml @@ -10,8 +10,8 @@ An RDAP Server. [dependencies] -icann-rdap-client = { version = "0.0.15", path = "../icann-rdap-client" } -icann-rdap-common = { version = "0.0.15", path = "../icann-rdap-common" } +icann-rdap-client = { version = "0.0.16", path = "../icann-rdap-client" } +icann-rdap-common = { version = "0.0.16", path = "../icann-rdap-common" } async-trait.workspace = true axum.workspace = true