From 2197eae433fe3729eb6943fe008c7a3ddd9eb798 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Thu, 31 Oct 2024 03:03:40 -0700 Subject: [PATCH 1/2] env list of structs --- src/env.rs | 58 +++++++++++++++++++++++++++++++++++++++++----------- src/value.rs | 34 ++++++++++++++++++++++++++++++ tests/env.rs | 50 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 127 insertions(+), 15 deletions(-) diff --git a/src/env.rs b/src/env.rs index 9848d69a..83056c27 100644 --- a/src/env.rs +++ b/src/env.rs @@ -4,6 +4,7 @@ use crate::error::Result; use crate::map::Map; use crate::source::Source; use crate::value::{Value, ValueKind}; +use crate::ConfigError; #[cfg(feature = "convert-case")] use convert_case::{Case, Casing}; @@ -230,8 +231,8 @@ impl Source for Environment { } fn collect(&self) -> Result> { - let mut m = Map::new(); let uri: String = "the environment".into(); + let mut m = Value::new(Some(&uri), ValueKind::Table(Map::new())); let separator = self.separator.as_deref().unwrap_or(""); #[cfg(feature = "convert-case")] @@ -248,24 +249,25 @@ impl Source for Environment { .as_ref() .map(|prefix| format!("{prefix}{prefix_separator}").to_lowercase()); - let collector = |(key, value): (String, String)| { + let collector = |(key, value): (String, String)| -> Result<()> { // Treat empty environment variables as unset if self.ignore_empty && value.is_empty() { - return; + return Ok(()); } let mut key = key.to_lowercase(); // Check for prefix + let mut kept_prefix = String::new(); if let Some(ref prefix_pattern) = prefix_pattern { if key.starts_with(prefix_pattern) { - if !self.keep_prefix { - // Remove this prefix from the key - key = key[prefix_pattern.len()..].to_string(); + key = key[prefix_pattern.len()..].to_string(); + if self.keep_prefix { + kept_prefix = prefix_pattern.clone(); } } else { // Skip this key - return; + return Ok(()); } } @@ -277,9 +279,10 @@ impl Source for Environment { #[cfg(feature = "convert-case")] if let Some(convert_case) = convert_case { key = key.to_case(*convert_case); + kept_prefix = kept_prefix.to_case(*convert_case); } - let value = if self.try_parsing { + let value_kind = if self.try_parsing { // convert to lowercase because bool parsing expects all lowercase if let Ok(parsed) = value.to_lowercase().parse::() { ValueKind::Boolean(parsed) @@ -315,14 +318,45 @@ impl Source for Environment { ValueKind::String(value) }; - m.insert(key, Value::new(Some(&uri), value)); + let value = Value::new(Some(&uri), value_kind); + let mut key_parts = key.split('.'); + let key = key_parts + .next() + .ok_or(ConfigError::Message("No key after prefix".to_owned()))?; + + let value = key_parts + .rev() + .fold(value, |value, key| match key.parse::() { + Ok(index) => { + let nil = Value::new(Some(&uri), ValueKind::Nil); + let mut array = vec![nil; index + 1]; + array[index] = value; + Value::new(Some(&uri), ValueKind::Array(array)) + } + Err(_) => { + let mut map = Map::new(); + map.insert(key.to_owned(), value); + Value::new(Some(&uri), ValueKind::Table(map)) + } + }); + + let mut map = Map::new(); + kept_prefix.push_str(key); + map.insert(kept_prefix, value); + let table = Value::new(Some(&uri), ValueKind::Table(map)); + + m.merge(table); + Ok(()) }; match &self.source { - Some(source) => source.clone().into_iter().for_each(collector), - None => env::vars().for_each(collector), + Some(source) => source.clone().into_iter().try_for_each(collector)?, + None => env::vars().try_for_each(collector)?, } - Ok(m) + match m.kind { + ValueKind::Table(m) => Ok(m), + _ => unreachable!(), + } } } diff --git a/src/value.rs b/src/value.rs index c242c628..ce0ce55c 100644 --- a/src/value.rs +++ b/src/value.rs @@ -705,6 +705,40 @@ impl Value { )), } } + + /// Merge the value with another value recursively. + /// + /// In the case of a conflict, the other value (on the right) will take precedence. + pub(crate) fn merge(&mut self, other: Self) { + match (&mut self.kind, other.kind) { + (ValueKind::Table(ref mut table), ValueKind::Table(other_table)) => { + for (k, v) in other_table { + match table.entry(k) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + entry.get_mut().merge(v); + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(v); + } + } + } + } + (ValueKind::Array(ref mut array), ValueKind::Array(other)) => { + for (i, other) in other.into_iter().enumerate() { + if let Some(s) = array.get_mut(i) { + if other.kind != ValueKind::Nil { + s.merge(other); + } + } else { + array.push(other); + } + } + } + (s, other) => { + *s = other; + } + } + } } impl<'de> Deserialize<'de> for Value { diff --git a/tests/env.rs b/tests/env.rs index 4befa5bb..f79ceb07 100644 --- a/tests/env.rs +++ b/tests/env.rs @@ -48,7 +48,7 @@ fn test_separator_behavior() { temp_env::with_var("C_B_A", Some("abc"), || { let environment = Environment::with_prefix("C").separator("_"); - assert!(environment.collect().unwrap().contains_key("b.a")); + assert!(environment.collect().unwrap().contains_key("b")); }); } @@ -85,7 +85,7 @@ fn test_custom_separator_behavior() { temp_env::with_var("C.B.A", Some("abc"), || { let environment = Environment::with_prefix("C").separator("."); - assert!(environment.collect().unwrap().contains_key("b.a")); + assert!(environment.collect().unwrap().contains_key("b")); }); } @@ -96,7 +96,7 @@ fn test_custom_prefix_separator_behavior() { .separator(".") .prefix_separator("-"); - assert!(environment.collect().unwrap().contains_key("b.a")); + assert!(environment.collect().unwrap().contains_key("b")); }); } @@ -724,3 +724,47 @@ fn test_parse_uint_default() { let config: TestUint = config.try_deserialize().unwrap(); assert_eq!(config.int_val, 42); } + +#[test] +fn test_env_list_of_structs() { + #[derive(Deserialize, Debug, PartialEq)] + struct Struct { + a: Vec>, + b: u32, + } + + #[derive(Deserialize, Debug)] + struct ListOfStructs { + list: Vec, + } + + let values = vec![ + ("LIST_0_A_0".to_owned(), "1".to_owned()), + ("LIST_0_A_2".to_owned(), "2".to_owned()), + ("LIST_0_B".to_owned(), "3".to_owned()), + ("LIST_1_A_1".to_owned(), "4".to_owned()), + ("LIST_1_B".to_owned(), "5".to_owned()), + ]; + + let environment = Environment::default() + .separator("_") + .try_parsing(true) + .source(Some(values.into_iter().collect())); + + let config = Config::builder().add_source(environment).build().unwrap(); + + let config: ListOfStructs = config.try_deserialize().unwrap(); + assert_eq!( + config.list, + vec![ + Struct { + a: vec![Some(1), None, Some(2)], + b: 3 + }, + Struct { + a: vec![None, Some(4)], + b: 5 + }, + ] + ); +} From 65231a0da62f39395491a93ad1ac68186ba4f933 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Thu, 31 Oct 2024 04:22:38 -0700 Subject: [PATCH 2/2] tests and docs --- examples/env-list/main.rs | 29 +++++++++++- src/env.rs | 62 ++++++++++++++++++++++++-- tests/env.rs | 92 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 4 deletions(-) diff --git a/examples/env-list/main.rs b/examples/env-list/main.rs index b55f3a97..8040f7ca 100644 --- a/examples/env-list/main.rs +++ b/examples/env-list/main.rs @@ -1,18 +1,31 @@ use config::Config; + +#[derive(Debug, Default, serde_derive::Deserialize, PartialEq, Eq)] +struct ListOfStructs { + a: String, + b: String, +} + #[derive(Debug, Default, serde_derive::Deserialize, PartialEq, Eq)] struct AppConfig { list: Vec, + structs: Vec>, } fn main() { std::env::set_var("APP_LIST", "Hello World"); + std::env::set_var("APP_STRUCTS_0_A", "Hello"); + std::env::set_var("APP_STRUCTS_0_B", "World"); + std::env::set_var("APP_STRUCTS_2_A", "foo"); + std::env::set_var("APP_STRUCTS_2_B", "bar"); let config = Config::builder() .add_source( config::Environment::with_prefix("APP") .try_parsing(true) .separator("_") - .list_separator(" "), + .list_separator(" ") + .with_list_parse_key("list"), ) .build() .unwrap(); @@ -20,6 +33,20 @@ fn main() { let app: AppConfig = config.try_deserialize().unwrap(); assert_eq!(app.list, vec![String::from("Hello"), String::from("World")]); + assert_eq!( + app.structs, + vec![ + Some(ListOfStructs { + a: String::from("Hello"), + b: String::from("World") + }), + None, + Some(ListOfStructs { + a: String::from("foo"), + b: String::from("bar") + }), + ] + ); std::env::remove_var("APP_LIST"); } diff --git a/src/env.rs b/src/env.rs index 83056c27..b90107da 100644 --- a/src/env.rs +++ b/src/env.rs @@ -13,6 +13,63 @@ use convert_case::{Case, Casing}; /// config Value type. We have to be aware how the config tree is created from the environment /// dictionary, therefore we are mindful about prefixes for the environment keys, level separators, /// encoding form (kebab, snake case) etc. +/// +/// Environment variables are specified using the path to the key they correspond to. When +/// specifying a value within a list, the index should be specified in the key immediately after +/// the list type. +/// +/// ## Example +/// +/// ```rust +/// # use config::{Environment, Config}; +/// # use serde::Deserialize; +/// # use std::collections::HashMap; +/// # use std::convert::TryInto; +/// # +/// # #[test] +/// # fn test_env_list_of_structs() { +/// #[derive(Deserialize, Debug, PartialEq)] +/// struct Struct { +/// a: Vec>, +/// b: u32, +/// } +/// +/// #[derive(Deserialize, Debug)] +/// struct ListOfStructs { +/// list: Vec, +/// } +/// +/// let values = vec![ +/// ("LIST_0_A_0".to_owned(), "1".to_owned()), +/// ("LIST_0_A_2".to_owned(), "2".to_owned()), +/// ("LIST_0_B".to_owned(), "3".to_owned()), +/// ("LIST_1_A_1".to_owned(), "4".to_owned()), +/// ("LIST_1_B".to_owned(), "5".to_owned()), +/// ]; +/// +/// let environment = Environment::default() +/// .separator("_") +/// .try_parsing(true) +/// .source(Some(values.into_iter().collect())); +/// +/// let config = Config::builder().add_source(environment).build().unwrap(); +/// +/// let config: ListOfStructs = config.try_deserialize().unwrap(); +/// assert_eq!( +/// config.list, +/// vec![ +/// Struct { +/// a: vec![Some(1), None, Some(2)], +/// b: 3 +/// }, +/// Struct { +/// a: vec![None, Some(4)], +/// b: 5 +/// }, +/// ] +/// ); +/// # } +/// ``` #[must_use] #[derive(Clone, Debug, Default)] pub struct Environment { @@ -291,11 +348,10 @@ impl Source for Environment { } else if let Ok(parsed) = value.parse::() { ValueKind::Float(parsed) } else if let Some(separator) = &self.list_separator { - if let Some(keys) = &self.list_parse_keys { + if let Some(parse_keys) = &self.list_parse_keys { #[cfg(feature = "convert-case")] let key = key.to_lowercase(); - - if keys.contains(&key) { + if parse_keys.contains(&key) { let v: Vec = value .split(separator) .map(|s| Value::new(Some(&uri), ValueKind::String(s.to_owned()))) diff --git a/tests/env.rs b/tests/env.rs index f79ceb07..8173605e 100644 --- a/tests/env.rs +++ b/tests/env.rs @@ -768,3 +768,95 @@ fn test_env_list_of_structs() { ] ); } + +#[test] +fn test_env_list_of_structs_list_sep() { + #[derive(Deserialize, Debug, PartialEq)] + struct Struct { + a: Vec, + b: Vec, + } + + #[derive(Deserialize, Debug)] + struct ListOfStructs { + list: Vec>, + } + + let values = vec![ + ("LIST_0_A".to_owned(), "hello,darkness".to_owned()), + ("LIST_0_B".to_owned(), "hello,world".to_owned()), + ("LIST_2_A".to_owned(), "strange".to_owned()), + ("LIST_2_B".to_owned(), "charm".to_owned()), + ]; + + let environment = Environment::default() + .separator("_") + .list_separator(",") + .try_parsing(true) + .source(Some(values.into_iter().collect())); + + let config = Config::builder().add_source(environment).build().unwrap(); + + let config: ListOfStructs = config.try_deserialize().unwrap(); + assert_eq!( + config.list, + vec![ + Some(Struct { + a: vec!["hello".to_owned(), "darkness".to_owned()], + b: vec!["hello".to_owned(), "world".to_owned()], + }), + None, + Some(Struct { + a: vec!["strange".to_owned()], + b: vec!["charm".to_owned()], + }), + ] + ); +} + +#[test] +fn test_env_list_of_structs_list_parse_keys() { + #[derive(Deserialize, Debug, PartialEq)] + struct Struct { + a: Vec, + b: String, + } + + #[derive(Deserialize, Debug)] + struct ListOfStructs { + list: Vec>, + } + + let values = vec![ + ("LIST_0_A".to_owned(), "hello,darkness".to_owned()), + ("LIST_0_B".to_owned(), "hello".to_owned()), + ("LIST_2_A".to_owned(), "strange".to_owned()), + ("LIST_2_B".to_owned(), "charm".to_owned()), + ]; + + let environment = Environment::default() + .separator("_") + .list_separator(",") + .with_list_parse_key("list.0.a") + .with_list_parse_key("list.2.a") + .try_parsing(true) + .source(Some(values.into_iter().collect())); + + let config = Config::builder().add_source(environment).build().unwrap(); + + let config: ListOfStructs = config.try_deserialize().unwrap(); + assert_eq!( + config.list, + vec![ + Some(Struct { + a: vec!["hello".to_owned(), "darkness".to_owned()], + b: "hello".to_owned(), + }), + None, + Some(Struct { + a: vec!["strange".to_owned()], + b: "charm".to_owned(), + }), + ] + ); +}