diff --git a/src/rust/engine/Cargo.lock b/src/rust/engine/Cargo.lock index 9f2a03b9c4d..70e8a2dcdc8 100644 --- a/src/rust/engine/Cargo.lock +++ b/src/rust/engine/Cargo.lock @@ -157,7 +157,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.51", ] [[package]] @@ -168,7 +168,7 @@ checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.51", ] [[package]] @@ -567,7 +567,7 @@ dependencies = [ "bitflags 1.3.2", "clap_derive", "clap_lex", - "indexmap", + "indexmap 1.9.3", "once_cell", "strsim", "termcolor", @@ -1044,7 +1044,7 @@ dependencies = [ "grpc_util", "hashing", "humansize", - "indexmap", + "indexmap 1.9.3", "internment", "itertools", "lazy_static", @@ -1100,6 +1100,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.2.8" @@ -1506,7 +1512,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util 0.7.10", @@ -1528,6 +1534,12 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + [[package]] name = "hashing" version = "0.0.1" @@ -1753,10 +1765,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.0", "serde", ] +[[package]] +name = "indexmap" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", +] + [[package]] name = "indicatif" version = "0.17.7" @@ -1810,7 +1832,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73bd1419c48e9f496d5830cc2f7fff35cab112193f289fd7266ab0679bb97237" dependencies = [ - "hashbrown", + "hashbrown 0.12.0", "parking_lot 0.12.1", ] @@ -2319,6 +2341,9 @@ dependencies = [ "maplit", "peg", "regex", + "serde", + "serde_json", + "serde_yaml", "shellexpand", "tempfile", "toml", @@ -2483,7 +2508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 1.9.3", ] [[package]] @@ -2577,7 +2602,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.32", + "syn 2.0.51", ] [[package]] @@ -2616,9 +2641,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -2741,7 +2766,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.32", + "syn 2.0.51", "tempfile", "which", ] @@ -2756,7 +2781,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.51", ] [[package]] @@ -2837,7 +2862,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.32", + "syn 2.0.51", ] [[package]] @@ -2849,7 +2874,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.51", ] [[package]] @@ -2864,9 +2889,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.32" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -3211,7 +3236,7 @@ dependencies = [ "deepsize", "env_logger", "fnv", - "indexmap", + "indexmap 1.9.3", "internment", "itertools", "log", @@ -3396,29 +3421,29 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.51", ] [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ "itoa", "ryu", @@ -3443,7 +3468,7 @@ checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.51", ] [[package]] @@ -3476,12 +3501,25 @@ dependencies = [ "base64 0.13.0", "chrono", "hex", - "indexmap", + "indexmap 1.9.3", "serde", "serde_json", "time", ] +[[package]] +name = "serde_yaml" +version = "0.9.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd075d994154d4a774f95b51fb96bdc2832b0ea48425c92546073816cda1f2f" +dependencies = [ + "indexmap 2.2.3", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha2" version = "0.10.7" @@ -3611,7 +3649,7 @@ dependencies = [ "hashing", "http", "http-body", - "indexmap", + "indexmap 1.9.3", "itertools", "lmdb-rkv", "log", @@ -3679,9 +3717,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.32" +version = "2.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" +checksum = "6ab617d94515e94ae53b8406c628598680aa0c9587474ecbe58188f7b345d66c" dependencies = [ "proc-macro2", "quote", @@ -3935,7 +3973,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.51", ] [[package]] @@ -4048,7 +4086,7 @@ dependencies = [ "proc-macro2", "prost-build", "quote", - "syn 2.0.32", + "syn 2.0.51", ] [[package]] @@ -4059,7 +4097,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand", @@ -4171,7 +4209,7 @@ version = "0.0.1" dependencies = [ "console", "futures", - "indexmap", + "indexmap 1.9.3", "indicatif", "logging", "parking_lot 0.12.1", @@ -4225,6 +4263,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" +[[package]] +name = "unsafe-libyaml" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" + [[package]] name = "untrusted" version = "0.7.1" @@ -4730,5 +4774,5 @@ checksum = "855e0f6af9cd72b87d8a6c586f3cb583f5cdcc62c2c80869d8cd7e96fdf7ee20" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.51", ] diff --git a/src/rust/engine/Cargo.toml b/src/rust/engine/Cargo.toml index dc83342b527..0adc2eda122 100644 --- a/src/rust/engine/Cargo.toml +++ b/src/rust/engine/Cargo.toml @@ -300,6 +300,7 @@ serde = "1.0.160" serde_derive = "1.0.98" serde_json = "1.0" serde_test = "1.0" +serde_yaml = "0.9" sha2 = "0.10" shell-quote = "0.3.0" shellexpand = "2.1" diff --git a/src/rust/engine/options/Cargo.toml b/src/rust/engine/options/Cargo.toml index ddbd6be562e..ad1389dca8a 100644 --- a/src/rust/engine/options/Cargo.toml +++ b/src/rust/engine/options/Cargo.toml @@ -14,6 +14,9 @@ shellexpand = { workspace = true } toml = { workspace = true } regex = { workspace = true } whoami = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +serde_yaml = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/src/rust/engine/options/src/args.rs b/src/rust/engine/options/src/args.rs index 47d9fee7ff1..efffb3c010f 100644 --- a/src/rust/engine/options/src/args.rs +++ b/src/rust/engine/options/src/args.rs @@ -4,11 +4,8 @@ use std::env; use super::id::{NameTransform, OptionId, Scope}; -use super::parse::{ - parse_bool, parse_bool_list, parse_dict, parse_float_list, parse_int_list, ParseError, -}; use super::{DictEdit, OptionsSource}; -use crate::parse::parse_string_list; +use crate::parse::{expand, expand_to_dict, expand_to_list, Parseable}; use crate::ListEdit; use std::collections::HashMap; @@ -80,11 +77,7 @@ impl Args { Ok(None) } - fn get_list( - &self, - id: &OptionId, - parse_list: fn(&str) -> Result>, ParseError>, - ) -> Result>>, String> { + fn get_list(&self, id: &OptionId) -> Result>>, String> { let arg_names = Self::arg_names(id, Negate::False); let mut edits = vec![]; for arg in &self.args { @@ -94,7 +87,11 @@ impl Args { let value = components.next().ok_or_else(|| { format!("Expected string list option {name} to have a value.") })?; - edits.extend(parse_list(value).map_err(|e| e.render(name))?) + if let Some(es) = + expand_to_list::(value.to_string()).map_err(|e| e.render(name))? + { + edits.extend(es); + } } } } @@ -112,40 +109,45 @@ impl OptionsSource for Args { } fn get_string(&self, id: &OptionId) -> Result, String> { - self.find_flag(Self::arg_names(id, Negate::False)) - .map(|value| value.map(|(_, v, _)| v)) + match self.find_flag(Self::arg_names(id, Negate::False))? { + Some((name, value, _)) => expand(value).map_err(|e| e.render(name)), + _ => Ok(None), + } } fn get_bool(&self, id: &OptionId) -> Result, String> { let arg_names = Self::arg_names(id, Negate::True); match self.find_flag(arg_names)? { Some((_, s, negated)) if s.as_str() == "" => Ok(Some(!negated)), - Some((name, ref value, negated)) => parse_bool(value) - .map(|b| Some(b ^ negated)) - .map_err(|e| e.render(name)), - None => Ok(None), + Some((name, value, negated)) => match expand(value).map_err(|e| e.render(&name))? { + Some(value) => bool::parse(&value) + .map(|b| Some(b ^ negated)) + .map_err(|e| e.render(&name)), + _ => Ok(None), + }, + _ => Ok(None), } } fn get_bool_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_bool_list) + self.get_list::(id) } fn get_int_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_int_list) + self.get_list::(id) } fn get_float_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_float_list) + self.get_list::(id) } fn get_string_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_string_list) + self.get_list::(id) } fn get_dict(&self, id: &OptionId) -> Result, String> { match self.find_flag(Self::arg_names(id, Negate::False))? { - Some((name, ref value, _)) => parse_dict(value).map(Some).map_err(|e| e.render(name)), + Some((name, value, _)) => expand_to_dict(value).map_err(|e| e.render(name)), None => Ok(None), } } diff --git a/src/rust/engine/options/src/args_tests.rs b/src/rust/engine/options/src/args_tests.rs index 740796c97e7..758124ac8a3 100644 --- a/src/rust/engine/options/src/args_tests.rs +++ b/src/rust/engine/options/src/args_tests.rs @@ -1,8 +1,12 @@ // Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). // Licensed under the Apache License, Version 2.0 (see LICENSE). +use core::fmt::Debug; +use maplit::hashmap; + use crate::args::Args; -use crate::option_id; +use crate::parse::test_util::write_fromfile; +use crate::{option_id, DictEdit, DictEditAction, Val}; use crate::{ListEdit, ListEditAction, OptionId, OptionsSource}; fn args>(args: I) -> Args { @@ -148,3 +152,141 @@ Expected \",\" or the end of a list indicated by ']' at line 1 column 18" args.get_string_list(&option_id!("bad")).unwrap_err() ); } + +#[test] +fn test_scalar_fromfile() { + fn do_test( + content: &str, + expected: T, + getter: fn(&Args, &OptionId) -> Result, String>, + negate: bool, + ) { + let (_tmpdir, fromfile_path) = write_fromfile("fromfile.txt", content); + let args = Args { + args: vec![format!( + "--{}foo=@{}", + if negate { "no-" } else { "" }, + fromfile_path.display() + )], + }; + let actual = getter(&args, &option_id!("foo")).unwrap().unwrap(); + assert_eq!(expected, actual) + } + + do_test("true", true, Args::get_bool, false); + do_test("false", false, Args::get_bool, false); + do_test("true", false, Args::get_bool, true); + do_test("false", true, Args::get_bool, true); + do_test("-42", -42, Args::get_int, false); + do_test("3.14", 3.14, Args::get_float, false); + do_test("EXPANDED", "EXPANDED".to_owned(), Args::get_string, false); + + let (_tmpdir, fromfile_path) = write_fromfile("fromfile.txt", "BAD INT"); + let args = Args { + args: vec![format!("--foo=@{}", fromfile_path.display())], + }; + assert_eq!( + args.get_int(&option_id!("foo")).unwrap_err(), + "Problem parsing --foo int value:\n1:BAD INT\n ^\n\ + Expected \"+\", \"-\" or ['0'..='9'] at line 1 column 1" + ); +} + +#[test] +fn test_list_fromfile() { + fn do_test(content: &str, expected: &[ListEdit], filename: &str) { + let (_tmpdir, fromfile_path) = write_fromfile(filename, content); + let args = Args { + args: vec![format!("--foo=@{}", &fromfile_path.display())], + }; + let actual = args.get_int_list(&option_id!("foo")).unwrap().unwrap(); + assert_eq!(expected.to_vec(), actual) + } + + do_test( + "-42", + &[ListEdit { + action: ListEditAction::Add, + items: vec![-42], + }], + "fromfile.txt", + ); + do_test( + "[10, 12]", + &[ListEdit { + action: ListEditAction::Replace, + items: vec![10, 12], + }], + "fromfile.json", + ); + do_test( + "- 22\n- 44\n", + &[ListEdit { + action: ListEditAction::Replace, + items: vec![22, 44], + }], + "fromfile.yaml", + ); +} + +#[test] +fn test_dict_fromfile() { + fn do_test(content: &str, filename: &str) { + let expected = DictEdit { + action: DictEditAction::Replace, + items: hashmap! { + "FOO".to_string() => Val::Dict(hashmap! { + "BAR".to_string() => Val::Float(3.14), + "BAZ".to_string() => Val::Dict(hashmap! { + "QUX".to_string() => Val::Bool(true), + "QUUX".to_string() => Val::List(vec![ Val::Int(1), Val::Int(2)]) + }) + }),}, + }; + + let (_tmpdir, fromfile_path) = write_fromfile(filename, content); + let args = Args { + args: vec![format!("--foo=@{}", &fromfile_path.display())], + }; + let actual = args.get_dict(&option_id!("foo")).unwrap().unwrap(); + assert_eq!(expected, actual) + } + + do_test( + "{'FOO': {'BAR': 3.14, 'BAZ': {'QUX': True, 'QUUX': [1, 2]}}}", + "fromfile.txt", + ); + do_test( + "{\"FOO\": {\"BAR\": 3.14, \"BAZ\": {\"QUX\": true, \"QUUX\": [1, 2]}}}", + "fromfile.json", + ); + do_test( + r#" + FOO: + BAR: 3.14 + BAZ: + QUX: true + QUUX: + - 1 + - 2 + "#, + "fromfile.yaml", + ); +} + +#[test] +fn test_nonexistent_required_fromfile() { + let args = Args { + args: vec!["--foo=@/does/not/exist".to_string()], + }; + let err = args.get_string(&option_id!("foo")).unwrap_err(); + assert!(err.starts_with("Problem reading /does/not/exist for --foo: No such file or directory")); +} + +#[test] +fn test_nonexistent_optional_fromfile() { + let args = Args { + args: vec!["--foo=@?/does/not/exist".to_string()], + }; + assert!(args.get_string(&option_id!("foo")).unwrap().is_none()); +} diff --git a/src/rust/engine/options/src/config.rs b/src/rust/engine/options/src/config.rs index f304975dcfd..d1f4e96c062 100644 --- a/src/rust/engine/options/src/config.rs +++ b/src/rust/engine/options/src/config.rs @@ -11,9 +11,7 @@ use toml::value::Table; use toml::Value; use super::id::{NameTransform, OptionId}; -use super::parse::{ - parse_bool_list, parse_dict, parse_float_list, parse_int_list, parse_string_list, ParseError, -}; +use super::parse::{expand, expand_to_dict, expand_to_list, Parseable}; use super::{DictEdit, DictEditAction, ListEdit, ListEditAction, OptionsSource, Val}; type InterpolationMap = HashMap; @@ -99,17 +97,28 @@ struct ValueConversionError<'a> { given_value: &'a Value, } -trait FromValue: Sized { +trait FromValue: Parseable { fn from_value(value: &Value) -> Result; fn from_config(config: &Config, id: &OptionId) -> Result, String> { if let Some(value) = config.get_value(id) { - match Self::from_value(value) { - Ok(x) => Ok(Some(x)), - Err(verr) => Err(format!( - "Expected {id} to be a {} but given {}", - verr.expected_type, verr.given_value - )), + if value.is_str() { + match expand(value.as_str().unwrap().to_owned()) + .map_err(|e| e.render(config.display(id)))? + { + Some(expanded_value) => Ok(Some( + Self::parse(&expanded_value).map_err(|e| e.render(config.display(id)))?, + )), + _ => Ok(None), + } + } else { + match Self::from_value(value) { + Ok(x) => Ok(Some(x)), + Err(verr) => Err(format!( + "Expected {id} to be a {} but given {}", + verr.expected_type, verr.given_value + )), + } } } else { Ok(None) @@ -316,10 +325,9 @@ impl Config { .and_then(|table| table.get(Self::option_name(id))) } - fn get_list( + fn get_list( &self, id: &OptionId, - parse_list: fn(&str) -> Result>, ParseError>, ) -> Result>>, String> { let mut list_edits = vec![]; if let Some(table) = self.value.get(id.scope()) { @@ -352,7 +360,11 @@ impl Config { } } Value::String(v) => { - list_edits.extend(parse_list(v).map_err(|e| e.render(option_name))?); + if let Some(es) = expand_to_list::(v.to_string()) + .map_err(|e| e.render(self.display(id)))? + { + list_edits.extend(es); + } } value => list_edits.push(ListEdit { action: ListEditAction::Replace, @@ -387,19 +399,19 @@ impl OptionsSource for Config { } fn get_bool_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_bool_list) + self.get_list::(id) } fn get_int_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_int_list) + self.get_list::(id) } fn get_float_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_float_list) + self.get_list::(id) } fn get_string_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_string_list) + self.get_list::(id) } fn get_dict(&self, id: &OptionId) -> Result, String> { @@ -422,7 +434,8 @@ impl OptionsSource for Config { })); } Value::String(v) => { - return Ok(Some(parse_dict(v).map_err(|e| e.render(option_name))?)); + return expand_to_dict(v.to_owned()) + .map_err(|e| e.render(self.display(id))); } _ => { return Err(format!( diff --git a/src/rust/engine/options/src/config_tests.rs b/src/rust/engine/options/src/config_tests.rs index 2d6ab5de017..2465c97a17e 100644 --- a/src/rust/engine/options/src/config_tests.rs +++ b/src/rust/engine/options/src/config_tests.rs @@ -1,17 +1,23 @@ // Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). // Licensed under the Apache License, Version 2.0 (see LICENSE). +use maplit::hashmap; use regex::Regex; use std::collections::HashMap; +use std::fmt::Debug; use std::fs::File; use std::io::Write; -use crate::config::{interpolate_string, Config}; -use crate::{option_id, DictEdit, DictEditAction, ListEdit, ListEditAction, OptionsSource, Val}; +use crate::config::interpolate_string; +use crate::{ + option_id, DictEdit, DictEditAction, ListEdit, ListEditAction, OptionId, OptionsSource, Val, +}; +use crate::config::Config; +use crate::parse::test_util::write_fromfile; use tempfile::TempDir; -fn maybe_config(file_content: &'static str) -> Result { +fn maybe_config(file_content: &str) -> Result { let dir = TempDir::new().unwrap(); let path = dir.path().join("pants.toml"); File::create(&path) @@ -27,7 +33,7 @@ fn maybe_config(file_content: &'static str) -> Result { ) } -fn config(file_content: &'static str) -> Config { +fn config(file_content: &str) -> Config { maybe_config(file_content).unwrap() } @@ -166,3 +172,115 @@ fn test_interpolate_config() { pat ); } + +#[test] +fn test_scalar_fromfile() { + fn do_test( + content: &str, + expected: T, + getter: fn(&Config, &OptionId) -> Result, String>, + ) { + let (_tmpdir, fromfile_path) = write_fromfile("fromfile.txt", content); + let conf = config(format!("[GLOBAL]\nfoo = '@{}'\n", fromfile_path.display()).as_str()); + let actual = getter(&conf, &option_id!("foo")).unwrap().unwrap(); + assert_eq!(expected, actual) + } + + do_test("true", true, Config::get_bool); + do_test("-42", -42, Config::get_int); + do_test("3.14", 3.14, Config::get_float); + do_test("EXPANDED", "EXPANDED".to_owned(), Config::get_string); +} + +#[test] +fn test_list_fromfile() { + fn do_test(content: &str, expected: &[ListEdit], filename: &str) { + let (_tmpdir, fromfile_path) = write_fromfile(filename, content); + let conf = config(format!("[GLOBAL]\nfoo = '@{}'\n", fromfile_path.display()).as_str()); + let actual = conf.get_int_list(&option_id!("foo")).unwrap().unwrap(); + assert_eq!(expected.to_vec(), actual) + } + + do_test( + "-42", + &[ListEdit { + action: ListEditAction::Add, + items: vec![-42], + }], + "fromfile.txt", + ); + do_test( + "[10, 12]", + &[ListEdit { + action: ListEditAction::Replace, + items: vec![10, 12], + }], + "fromfile.json", + ); + do_test( + "- 22\n- 44\n", + &[ListEdit { + action: ListEditAction::Replace, + items: vec![22, 44], + }], + "fromfile.yaml", + ); +} + +#[test] +fn test_dict_fromfile() { + fn do_test(content: &str, filename: &str) { + let expected = DictEdit { + action: DictEditAction::Replace, + items: hashmap! { + "FOO".to_string() => Val::Dict(hashmap! { + "BAR".to_string() => Val::Float(3.14), + "BAZ".to_string() => Val::Dict(hashmap! { + "QUX".to_string() => Val::Bool(true), + "QUUX".to_string() => Val::List(vec![ Val::Int(1), Val::Int(2)]) + }) + }),}, + }; + + let (_tmpdir, fromfile_path) = write_fromfile(filename, content); + let conf = config(format!("[GLOBAL]\nfoo = '@{}'\n", fromfile_path.display()).as_str()); + let actual = conf.get_dict(&option_id!("foo")).unwrap().unwrap(); + assert_eq!(expected, actual) + } + + do_test( + "{'FOO': {'BAR': 3.14, 'BAZ': {'QUX': True, 'QUUX': [1, 2]}}}", + "fromfile.txt", + ); + do_test( + "{\"FOO\": {\"BAR\": 3.14, \"BAZ\": {\"QUX\": true, \"QUUX\": [1, 2]}}}", + "fromfile.json", + ); + do_test( + r#" + FOO: + BAR: 3.14 + BAZ: + QUX: true + QUUX: + - 1 + - 2 + "#, + "fromfile.yaml", + ); +} + +#[test] +fn test_nonexistent_required_fromfile() { + let conf = config("[GLOBAL]\nfoo = '@/does/not/exist'\n"); + let err = conf.get_string(&option_id!("foo")).unwrap_err(); + assert!(err.starts_with( + "Problem reading /does/not/exist for [GLOBAL] foo: No such file or directory" + )); +} + +#[test] +fn test_nonexistent_optional_fromfile() { + let conf = config("[GLOBAL]\nfoo = '@?/does/not/exist'\n"); + assert!(conf.get_string(&option_id!("foo")).unwrap().is_none()); +} diff --git a/src/rust/engine/options/src/env.rs b/src/rust/engine/options/src/env.rs index 0e817db751b..6e7cafd8a6d 100644 --- a/src/rust/engine/options/src/env.rs +++ b/src/rust/engine/options/src/env.rs @@ -7,10 +7,7 @@ use std::ffi::OsString; use super::id::{NameTransform, OptionId, Scope}; use super::{DictEdit, OptionsSource}; -use crate::parse::{ - parse_bool, parse_bool_list, parse_dict, parse_float_list, parse_int_list, parse_string_list, - ParseError, -}; +use crate::parse::{expand, expand_to_dict, expand_to_list, Parseable}; use crate::ListEdit; #[derive(Debug)] @@ -70,18 +67,14 @@ impl Env { names } - fn get_list( - &self, - id: &OptionId, - parse_list: fn(&str) -> Result>, ParseError>, - ) -> Result>>, String> { - if let Some(value) = self.get_string(id)? { - parse_list(&value) - .map(Some) - .map_err(|e| e.render(self.display(id))) - } else { - Ok(None) + fn get_list(&self, id: &OptionId) -> Result>>, String> { + for env_var_name in &Self::env_var_names(id) { + if let Some(value) = self.env.get(env_var_name) { + return expand_to_list::(value.to_owned()) + .map_err(|e| e.render(self.display(id))); + } } + Ok(None) } } @@ -100,10 +93,9 @@ impl OptionsSource for Env { } fn get_string(&self, id: &OptionId) -> Result, String> { - let env_var_names = Self::env_var_names(id); - for env_var_name in &env_var_names { + for env_var_name in &Self::env_var_names(id) { if let Some(value) = self.env.get(env_var_name) { - return Ok(Some(value.to_owned())); + return expand(value.to_owned()).map_err(|e| e.render(self.display(id))); } } Ok(None) @@ -111,7 +103,7 @@ impl OptionsSource for Env { fn get_bool(&self, id: &OptionId) -> Result, String> { if let Some(value) = self.get_string(id)? { - parse_bool(&value) + bool::parse(&value) .map(Some) .map_err(|e| e.render(self.display(id))) } else { @@ -120,28 +112,27 @@ impl OptionsSource for Env { } fn get_bool_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_bool_list) + self.get_list::(id) } fn get_int_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_int_list) + self.get_list::(id) } fn get_float_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_float_list) + self.get_list::(id) } fn get_string_list(&self, id: &OptionId) -> Result>>, String> { - self.get_list(id, parse_string_list) + self.get_list::(id) } fn get_dict(&self, id: &OptionId) -> Result, String> { - if let Some(value) = self.get_string(id)? { - parse_dict(&value) - .map(Some) - .map_err(|e| e.render(self.display(id))) - } else { - Ok(None) + for env_var_name in &Self::env_var_names(id) { + if let Some(value) = self.env.get(env_var_name) { + return expand_to_dict(value.to_owned()).map_err(|e| e.render(self.display(id))); + } } + Ok(None) } } diff --git a/src/rust/engine/options/src/env_tests.rs b/src/rust/engine/options/src/env_tests.rs index 27cbdb97498..2487005eb4f 100644 --- a/src/rust/engine/options/src/env_tests.rs +++ b/src/rust/engine/options/src/env_tests.rs @@ -2,12 +2,15 @@ // Licensed under the Apache License, Version 2.0 (see LICENSE). use crate::env::Env; -use crate::option_id; -use crate::{ListEdit, ListEditAction, OptionId, OptionsSource}; +use crate::parse::test_util::write_fromfile; +use crate::{option_id, DictEdit, DictEditAction}; +use crate::{ListEdit, ListEditAction, OptionId, OptionsSource, Val}; +use maplit::hashmap; use std::collections::HashMap; use std::ffi::OsString; +use std::fmt::Debug; -fn env>(vars: I) -> Env { +fn env<'a, I: IntoIterator>(vars: I) -> Env { Env { env: vars .into_iter() @@ -199,3 +202,124 @@ Expected \",\" or the end of a tuple indicated by ')' at line 1 column 18" env.get_string_list(&option_id!("bad")).unwrap_err() ); } + +#[test] +fn test_scalar_fromfile() { + fn do_test( + content: &str, + expected: T, + getter: fn(&Env, &OptionId) -> Result, String>, + ) { + let (_tmpdir, fromfile_path) = write_fromfile("fromfile.txt", content); + let env = env([( + "PANTS_FOO", + format!("@{}", fromfile_path.display()).as_str(), + )]); + let actual = getter(&env, &option_id!("foo")).unwrap().unwrap(); + assert_eq!(expected, actual) + } + + do_test("true", true, Env::get_bool); + do_test("-42", -42, Env::get_int); + do_test("3.14", 3.14, Env::get_float); + do_test("EXPANDED", "EXPANDED".to_owned(), Env::get_string); +} + +#[test] +fn test_list_fromfile() { + fn do_test(content: &str, expected: &[ListEdit], filename: &str) { + let (_tmpdir, fromfile_path) = write_fromfile(filename, content); + let env = env([( + "PANTS_FOO", + format!("@{}", fromfile_path.display()).as_str(), + )]); + let actual = env.get_int_list(&option_id!("foo")).unwrap().unwrap(); + assert_eq!(expected.to_vec(), actual) + } + + do_test( + "-42", + &[ListEdit { + action: ListEditAction::Add, + items: vec![-42], + }], + "fromfile.txt", + ); + do_test( + "[10, 12]", + &[ListEdit { + action: ListEditAction::Replace, + items: vec![10, 12], + }], + "fromfile.json", + ); + do_test( + "- 22\n- 44\n", + &[ListEdit { + action: ListEditAction::Replace, + items: vec![22, 44], + }], + "fromfile.yaml", + ); +} + +#[test] +fn test_dict_fromfile() { + fn do_test(content: &str, filename: &str) { + let expected = DictEdit { + action: DictEditAction::Replace, + items: hashmap! { + "FOO".to_string() => Val::Dict(hashmap! { + "BAR".to_string() => Val::Float(3.14), + "BAZ".to_string() => Val::Dict(hashmap! { + "QUX".to_string() => Val::Bool(true), + "QUUX".to_string() => Val::List(vec![ Val::Int(1), Val::Int(2)]) + }) + }),}, + }; + + let (_tmpdir, fromfile_path) = write_fromfile(filename, content); + let env = env([( + "PANTS_FOO", + format!("@{}", fromfile_path.display()).as_str(), + )]); + let actual = env.get_dict(&option_id!("foo")).unwrap().unwrap(); + assert_eq!(expected, actual) + } + + do_test( + "{'FOO': {'BAR': 3.14, 'BAZ': {'QUX': True, 'QUUX': [1, 2]}}}", + "fromfile.txt", + ); + do_test( + "{\"FOO\": {\"BAR\": 3.14, \"BAZ\": {\"QUX\": true, \"QUUX\": [1, 2]}}}", + "fromfile.json", + ); + do_test( + r#" + FOO: + BAR: 3.14 + BAZ: + QUX: true + QUUX: + - 1 + - 2 + "#, + "fromfile.yaml", + ); +} + +#[test] +fn test_nonexistent_required_fromfile() { + let env = env([("PANTS_FOO", "@/does/not/exist")]); + let err = env.get_string(&option_id!("foo")).unwrap_err(); + assert!( + err.starts_with("Problem reading /does/not/exist for PANTS_FOO: No such file or directory") + ); +} + +#[test] +fn test_nonexistent_optional_fromfile() { + let env = env([("PANTS_FOO", "@?/does/not/exist")]); + assert!(env.get_string(&option_id!("foo")).unwrap().is_none()); +} diff --git a/src/rust/engine/options/src/lib.rs b/src/rust/engine/options/src/lib.rs index 6583114cd1a..fbdaae8ebf3 100644 --- a/src/rust/engine/options/src/lib.rs +++ b/src/rust/engine/options/src/lib.rs @@ -37,10 +37,12 @@ use std::os::unix::ffi::OsStrExt; use std::path::Path; use std::rc::Rc; +use serde::Deserialize; + pub use self::args::Args; use self::config::Config; pub use self::env::Env; -use crate::parse::{parse_float, parse_int}; +use crate::parse::Parseable; pub use build_root::BuildRoot; pub use id::{OptionId, Scope}; pub use types::OptionType; @@ -54,7 +56,8 @@ pub use types::OptionType; // We only use this for parsing values in dicts, as in other cases we know that the type must // be some scalar or string, or a uniform list of one type of scalar or string, so we can // parse as such. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(untagged)] pub enum Val { Bool(bool), Int(i64), @@ -89,12 +92,6 @@ pub struct DictEdit { pub items: HashMap, } -/// -/// A source of option values. -/// -/// This is currently a subset of the types of options the Pants python option system handles. -/// Implementations should mimic the behavior of the equivalent python source. -/// pub(crate) trait OptionsSource { /// /// Get a display version of the option `id` that most closely matches the syntax used to supply @@ -124,7 +121,7 @@ pub(crate) trait OptionsSource { /// fn get_int(&self, id: &OptionId) -> Result, String> { if let Some(value) = self.get_string(id)? { - parse_int(&value) + i64::parse(&value) .map(Some) .map_err(|e| e.render(self.display(id))) } else { @@ -142,12 +139,12 @@ pub(crate) trait OptionsSource { /// fn get_float(&self, id: &OptionId) -> Result, String> { if let Some(value) = self.get_string(id)? { - let parsed_as_float = parse_float(&value) + let parsed_as_float = f64::parse(&value) .map(Some) .map_err(|e| e.render(self.display(id))); if parsed_as_float.is_err() { // See if we can parse as an int and coerce it to a float. - if let Ok(i) = parse_int(&value) { + if let Ok(i) = i64::parse(&value) { return Ok(Some(i as f64)); } } @@ -199,8 +196,6 @@ pub enum Source { #[derive(Debug)] pub struct OptionValue { pub derivation: Option>, - // Scalar options are always set from a single source, so we provide that - // here, as it can be useful in user-facing messages. pub source: Source, pub value: T, } @@ -208,12 +203,16 @@ pub struct OptionValue { #[derive(Debug)] pub struct ListOptionValue { pub derivation: Option>)>>, + // The highest-priority source that provided edits for this value. + pub source: Source, pub value: Vec, } #[derive(Debug)] pub struct DictOptionValue { pub derivation: Option>, + // The highest-priority source that provided edits for this value. + pub source: Source, pub value: HashMap, } @@ -434,8 +433,10 @@ impl OptionParser { } derivation = Some(derivations); } - for (_source_type, source) in self.sources.iter() { + let mut highest_priority_source = Source::Default; + for (source_type, source) in self.sources.iter() { if let Some(list_edits) = getter(source, id)? { + highest_priority_source = source_type.clone(); for list_edit in list_edits { match list_edit.action { ListEditAction::Replace => list = list_edit.items, @@ -447,6 +448,7 @@ impl OptionParser { } Ok(ListOptionValue { derivation, + source: highest_priority_source, value: list, }) } @@ -536,8 +538,10 @@ impl OptionParser { } derivation = Some(derivations); } - for (_, source) in self.sources.iter() { + let mut highest_priority_source = Source::Default; + for (source_type, source) in self.sources.iter() { if let Some(dict_edit) = source.get_dict(id)? { + highest_priority_source = source_type.clone(); match dict_edit.action { DictEditAction::Replace => dict = dict_edit.items, DictEditAction::Add => dict.extend(dict_edit.items), @@ -546,6 +550,7 @@ impl OptionParser { } Ok(DictOptionValue { derivation, + source: highest_priority_source, value: dict, }) } diff --git a/src/rust/engine/options/src/parse.rs b/src/rust/engine/options/src/parse.rs index a9a2ca9cb5b..0b15398be6f 100644 --- a/src/rust/engine/options/src/parse.rs +++ b/src/rust/engine/options/src/parse.rs @@ -4,7 +4,12 @@ use super::{DictEdit, DictEditAction, ListEdit, ListEditAction, Val}; use crate::render_choice; +use log::warn; +use serde::de::{Deserialize, DeserializeOwned}; use std::collections::HashMap; +use std::fmt::Display; +use std::path::{Path, PathBuf}; +use std::{fs, io}; peg::parser! { grammar option_value_parser() for str { @@ -274,49 +279,191 @@ fn format_parse_error( )) } -#[allow(dead_code)] -pub(crate) fn parse_bool(value: &str) -> Result { - option_value_parser::bool(value).map_err(|e| format_parse_error("bool", value, e)) +pub(crate) fn parse_dict(value: &str) -> Result { + option_value_parser::dict_edit(value).map_err(|e| format_parse_error("dict", value, e)) } -#[allow(dead_code)] -pub(crate) fn parse_int(value: &str) -> Result { - option_value_parser::int(value).map_err(|e| format_parse_error("int", value, e)) +pub(crate) trait Parseable: Sized + DeserializeOwned { + fn parse(value: &str) -> Result; + fn parse_list(value: &str) -> Result>, ParseError>; } -#[allow(dead_code)] -pub(crate) fn parse_float(value: &str) -> Result { - option_value_parser::float(value).map_err(|e| format_parse_error("float", value, e)) +impl Parseable for bool { + fn parse(value: &str) -> Result { + option_value_parser::bool(value).map_err(|e| format_parse_error("bool", value, e)) + } + + fn parse_list(value: &str) -> Result>, ParseError> { + option_value_parser::bool_list_edits(value) + .map_err(|e| format_parse_error("bool list", value, e)) + } } -#[allow(dead_code)] -pub(crate) fn parse_quoted_string(value: &str) -> Result { - option_value_parser::quoted_string(value).map_err(|e| format_parse_error("string", value, e)) +impl Parseable for i64 { + fn parse(value: &str) -> Result { + option_value_parser::int(value).map_err(|e| format_parse_error("int", value, e)) + } + + fn parse_list(value: &str) -> Result>, ParseError> { + option_value_parser::int_list_edits(value) + .map_err(|e| format_parse_error("int list", value, e)) + } } -#[allow(dead_code)] -pub(crate) fn parse_bool_list(value: &str) -> Result>, ParseError> { - option_value_parser::bool_list_edits(value) - .map_err(|e| format_parse_error("bool list", value, e)) +impl Parseable for f64 { + fn parse(value: &str) -> Result { + option_value_parser::float(value).map_err(|e| format_parse_error("float", value, e)) + } + + fn parse_list(value: &str) -> Result>, ParseError> { + option_value_parser::float_list_edits(value) + .map_err(|e| format_parse_error("float list", value, e)) + } } -#[allow(dead_code)] -pub(crate) fn parse_int_list(value: &str) -> Result>, ParseError> { - option_value_parser::int_list_edits(value).map_err(|e| format_parse_error("int list", value, e)) +impl Parseable for String { + fn parse(value: &str) -> Result { + Ok(value.to_owned()) + } + + fn parse_list(value: &str) -> Result>, ParseError> { + option_value_parser::string_list_edits(value) + .map_err(|e| format_parse_error("string list", value, e)) + } } -#[allow(dead_code)] -pub(crate) fn parse_float_list(value: &str) -> Result>, ParseError> { - option_value_parser::float_list_edits(value) - .map_err(|e| format_parse_error("float list", value, e)) +// If the corresponding unexpanded value points to a @fromfile, then the +// first component is the path to that file, and the second is the value from the file, +// or None if the file doesn't exist and the @?fromfile syntax was used. +// +// Otherwise, the first component is None and the second is the original value. +type ExpandedValue = (Option, Option); + +fn mk_parse_err(err: impl Display, path: &Path) -> ParseError { + ParseError::new(format!( + "Problem reading {path} for {{name}}: {err}", + path = path.display() + )) } -pub(crate) fn parse_string_list(value: &str) -> Result>, ParseError> { - option_value_parser::string_list_edits(value) - .map_err(|e| format_parse_error("string list", value, e)) +fn maybe_expand(value: String) -> Result { + if let Some(suffix) = value.strip_prefix('@') { + if suffix.starts_with('@') { + // @@ escapes the initial @. + Ok((None, Some(suffix.to_owned()))) + } else { + match suffix.strip_prefix('?') { + Some(subsuffix) => { + // @? means the path is allowed to not exist. + let path = PathBuf::from(subsuffix); + match fs::read_to_string(&path) { + Ok(content) => Ok((Some(path), Some(content))), + Err(err) if err.kind() == io::ErrorKind::NotFound => { + warn!("Optional file config '{}' does not exist.", path.display()); + Ok((Some(path), None)) + } + Err(err) => Err(mk_parse_err(err, &path)), + } + } + _ => { + let path = PathBuf::from(suffix); + let content = fs::read_to_string(&path).map_err(|e| mk_parse_err(e, &path))?; + Ok((Some(path), Some(content))) + } + } + } + } else { + Ok((None, Some(value))) + } } -#[allow(dead_code)] -pub(crate) fn parse_dict(value: &str) -> Result { - option_value_parser::dict_edit(value).map_err(|e| format_parse_error("dict", value, e)) +pub(crate) fn expand(value: String) -> Result, ParseError> { + let (_, expanded_value) = maybe_expand(value)?; + Ok(expanded_value) +} + +#[derive(Debug)] +enum FromfileType { + Json, + Yaml, + Unknown, +} + +impl FromfileType { + fn detect(path: &Path) -> FromfileType { + if let Some(ext) = path.extension() { + if ext == "json" { + return FromfileType::Json; + } else if ext == "yml" || ext == "yaml" { + return FromfileType::Yaml; + }; + } + FromfileType::Unknown + } +} + +fn try_deserialize<'a, DE: Deserialize<'a>>( + value: &'a str, + path_opt: Option, +) -> Result, ParseError> { + if let Some(path) = path_opt { + match FromfileType::detect(&path) { + FromfileType::Json => serde_json::from_str(value).map_err(|e| mk_parse_err(e, &path)), + FromfileType::Yaml => serde_yaml::from_str(value).map_err(|e| mk_parse_err(e, &path)), + _ => Ok(None), + } + } else { + Ok(None) + } +} + +pub(crate) fn expand_to_list( + value: String, +) -> Result>>, ParseError> { + let (path_opt, value_opt) = maybe_expand(value)?; + if let Some(value) = value_opt { + if let Some(items) = try_deserialize(&value, path_opt)? { + Ok(Some(vec![ListEdit { + action: ListEditAction::Replace, + items, + }])) + } else { + T::parse_list(&value).map(Some) + } + } else { + Ok(None) + } +} + +pub(crate) fn expand_to_dict(value: String) -> Result, ParseError> { + let (path_opt, value_opt) = maybe_expand(value)?; + if let Some(value) = value_opt { + if let Some(items) = try_deserialize(&value, path_opt)? { + Ok(Some(DictEdit { + action: DictEditAction::Replace, + items, + })) + } else { + parse_dict(&value).map(Some) + } + } else { + Ok(None) + } +} + +#[cfg(test)] +pub(crate) mod test_util { + use std::fs::File; + use std::io::Write; + use std::path::PathBuf; + use tempfile::{tempdir, TempDir}; + + pub(crate) fn write_fromfile(filename: &str, content: &str) -> (TempDir, PathBuf) { + let tmpdir = tempdir().unwrap(); + let fromfile_path = tmpdir.path().join(filename); + let mut fromfile = File::create(&fromfile_path).unwrap(); + fromfile.write_all(content.as_bytes()).unwrap(); + fromfile.flush().unwrap(); + (tmpdir, fromfile_path) + } } diff --git a/src/rust/engine/options/src/parse_tests.rs b/src/rust/engine/options/src/parse_tests.rs index 29a7a902523..c2123b48b8d 100644 --- a/src/rust/engine/options/src/parse_tests.rs +++ b/src/rust/engine/options/src/parse_tests.rs @@ -1,8 +1,10 @@ // Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). // Licensed under the Apache License, Version 2.0 (see LICENSE). +use crate::parse::test_util::write_fromfile; use crate::parse::*; use crate::{DictEdit, DictEditAction, ListEdit, ListEditAction, Val}; +use maplit::hashmap; use std::collections::HashMap; use std::fmt::Debug; @@ -17,6 +19,18 @@ macro_rules! check { ($left:expr, $right:expr, $($arg:tt)+) => { check_with_arg($left, $right, $($arg)+); }; } +macro_rules! check_err { + ($res:expr, $expected_suffix:expr $(,)?) => { + let actual_msg = $res.unwrap_err().render("XXX"); + assert!( + actual_msg.ends_with($expected_suffix), + "Error message does not have expected suffix:\n{actual_msg}\nvs\n{:>width$}", + $expected_suffix, + width = actual_msg.len(), + ) + }; +} + fn check(expected: T, res: Result) { match res { Ok(actual) => assert_eq!(expected, actual), @@ -38,7 +52,18 @@ fn check_with_arg( #[test] fn test_parse_quoted_string() { fn check_str(expected: &str, input: &str) { - check!(expected.to_string(), parse_quoted_string(input)); + // This is slightly convoluted: quoted strings appear as list items, + // so we generate a list, and then extract the parsed string out of + // the Result>, ...> returned by parse_list(). + let parsed = String::parse_list(format!("[{}]", input).as_str()) + .unwrap() + .first() + .unwrap() + .items + .first() + .unwrap() + .to_string(); + check!(expected.to_string(), Ok(parsed)); } check_str("", "''"); @@ -54,7 +79,7 @@ fn test_parse_quoted_string() { #[test] fn test_parse_bool() { fn check_bool(expected: bool, input: &str) { - check!(expected, parse_bool(input)); + check!(expected, bool::parse(input)); } check_bool(true, "true"); @@ -69,14 +94,14 @@ fn test_parse_bool() { "Problem parsing foo bool value:\n1:1\n ^\nExpected 'true' or 'false' \ at line 1 column 1" .to_owned(), - parse_bool("1").unwrap_err().render("foo") + bool::parse("1").unwrap_err().render("foo") ) } #[test] fn test_parse_int() { fn check_int(expected: i64, input: &str) { - check!(expected, parse_int(input)); + check!(expected, i64::parse(input)); } check_int(0, "0"); check_int(1, "1"); @@ -92,20 +117,20 @@ fn test_parse_int() { "Problem parsing foo int value:\n1:badint\n ^\nExpected \"+\", \"-\" or ['0'..='9'] \ at line 1 column 1" .to_owned(), - parse_int("badint").unwrap_err().render("foo") + i64::parse("badint").unwrap_err().render("foo") ); assert_eq!( "Problem parsing foo int value:\n1:12badint\n --^\nExpected \"_\", EOF or ['0'..='9'] \ at line 1 column 3" .to_owned(), - parse_int("12badint").unwrap_err().render("foo") + i64::parse("12badint").unwrap_err().render("foo") ); } #[test] fn test_parse_float() { fn check_float(expected: f64, input: &str) { - check!(expected, parse_float(input)); + check!(expected, f64::parse(input)); } check_float(0.0, "0.0"); check_float(0.0, "-0.0"); @@ -122,10 +147,10 @@ fn test_parse_float() { #[test] fn test_parse_list_from_empty_string() { - assert!(parse_string_list("").unwrap().is_empty()); - assert!(parse_bool_list("").unwrap().is_empty()); - assert!(parse_int_list("").unwrap().is_empty()); - assert!(parse_float_list("").unwrap().is_empty()); + assert!(String::parse_list("").unwrap().is_empty()); + assert!(bool::parse_list("").unwrap().is_empty()); + assert!(i64::parse_list("").unwrap().is_empty()); + assert!(f64::parse_list("").unwrap().is_empty()); } fn string_list_edit>( @@ -154,15 +179,15 @@ const EMPTY_FLOAT_LIST: [f64; 0] = []; fn test_parse_string_list_replace() { check!( vec![string_list_edit(ListEditAction::Replace, EMPTY_STRING_LIST)], - parse_string_list("[]") + String::parse_list("[]") ); check!( vec![string_list_edit(ListEditAction::Replace, ["foo"])], - parse_string_list("['foo']") + String::parse_list("['foo']") ); check!( vec![string_list_edit(ListEditAction::Replace, ["foo", "bar"])], - parse_string_list("['foo','bar']") + String::parse_list("['foo','bar']") ); } @@ -170,15 +195,15 @@ fn test_parse_string_list_replace() { fn test_parse_bool_list_replace() { check!( vec![scalar_list_edit(ListEditAction::Replace, EMPTY_BOOL_LIST)], - parse_bool_list("[]") + bool::parse_list("[]") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [true])], - parse_bool_list("[True]") + bool::parse_list("[True]") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [true, false])], - parse_bool_list("[True,FALSE]") + bool::parse_list("[True,FALSE]") ); } @@ -186,15 +211,15 @@ fn test_parse_bool_list_replace() { fn test_parse_int_list_replace() { check!( vec![scalar_list_edit(ListEditAction::Replace, EMPTY_INT_LIST)], - parse_int_list("[]") + i64::parse_list("[]") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [42])], - parse_int_list("[42]") + i64::parse_list("[42]") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [42, -127])], - parse_int_list("[42,-127]") + i64::parse_list("[42,-127]") ); } @@ -202,15 +227,15 @@ fn test_parse_int_list_replace() { fn test_parse_float_list_replace() { check!( vec![scalar_list_edit(ListEditAction::Replace, EMPTY_FLOAT_LIST)], - parse_float_list("[]") + f64::parse_list("[]") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [123456.78])], - parse_float_list("[123_456.78]") + f64::parse_list("[123_456.78]") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [42.0, -1.27e+7])], - parse_float_list("[42.0,-127.0e+5]") + f64::parse_list("[42.0,-127.0e+5]") ); } @@ -218,7 +243,7 @@ fn test_parse_float_list_replace() { fn test_parse_string_list_add() { check!( vec![string_list_edit(ListEditAction::Add, EMPTY_STRING_LIST)], - parse_string_list("+[]") + String::parse_list("+[]") ); } @@ -226,7 +251,7 @@ fn test_parse_string_list_add() { fn test_parse_scalar_list_add() { check!( vec![scalar_list_edit(ListEditAction::Add, EMPTY_INT_LIST)], - parse_int_list("+[]") + i64::parse_list("+[]") ); } @@ -234,7 +259,7 @@ fn test_parse_scalar_list_add() { fn test_parse_string_list_remove() { check!( vec![string_list_edit(ListEditAction::Remove, EMPTY_STRING_LIST)], - parse_string_list("-[]") + String::parse_list("-[]") ); } @@ -242,7 +267,7 @@ fn test_parse_string_list_remove() { fn test_parse_scalar_list_remove() { check!( vec![scalar_list_edit(ListEditAction::Remove, EMPTY_BOOL_LIST)], - parse_bool_list("-[]") + bool::parse_list("-[]") ); } @@ -254,7 +279,7 @@ fn test_parse_string_list_edits() { string_list_edit(ListEditAction::Add, ["baz"]), string_list_edit(ListEditAction::Remove, EMPTY_STRING_LIST), ], - parse_string_list("-['foo', 'bar'],+['baz'],-[]") + String::parse_list("-['foo', 'bar'],+['baz'],-[]") ); } @@ -266,7 +291,7 @@ fn test_parse_bool_list_edits() { scalar_list_edit(ListEditAction::Add, [false]), scalar_list_edit(ListEditAction::Remove, EMPTY_BOOL_LIST), ], - parse_bool_list("-[True, FALSE],+[false],-[]") + bool::parse_list("-[True, FALSE],+[false],-[]") ); } @@ -278,7 +303,7 @@ fn test_parse_int_list_edits() { scalar_list_edit(ListEditAction::Add, [42]), scalar_list_edit(ListEditAction::Remove, EMPTY_INT_LIST), ], - parse_int_list("-[-3, 4],+[42],-[]") + i64::parse_list("-[-3, 4],+[42],-[]") ); } @@ -290,7 +315,7 @@ fn test_parse_float_list_edits() { scalar_list_edit(ListEditAction::Add, [42.7]), scalar_list_edit(ListEditAction::Remove, EMPTY_FLOAT_LIST), ], - parse_float_list("-[-3.0, 4.1],+[42.7],-[]") + f64::parse_list("-[-3.0, 4.1],+[42.7],-[]") ); } @@ -301,7 +326,7 @@ fn test_parse_string_list_edits_whitespace() { string_list_edit(ListEditAction::Remove, ["foo"]), string_list_edit(ListEditAction::Add, ["bar"]), ], - parse_string_list(" - [ 'foo' , ] ,\n + [ 'bar' ] ") + String::parse_list(" - [ 'foo' , ] ,\n + [ 'bar' ] ") ); } @@ -312,7 +337,7 @@ fn test_parse_scalar_list_edits_whitespace() { scalar_list_edit(ListEditAction::Remove, [42.0]), scalar_list_edit(ListEditAction::Add, [-127.1, 0.0]), ], - parse_float_list(" - [ 42.0 , ] , + [ -127.1 ,0. ] ") + f64::parse_list(" - [ 42.0 , ] , + [ -127.1 ,0. ] ") ); } @@ -320,15 +345,15 @@ fn test_parse_scalar_list_edits_whitespace() { fn test_parse_string_list_implicit_add() { check!( vec![string_list_edit(ListEditAction::Add, vec!["foo"])], - parse_string_list("foo") + String::parse_list("foo") ); check!( vec![string_list_edit(ListEditAction::Add, vec!["foo bar"])], - parse_string_list("foo bar") + String::parse_list("foo bar") ); check!( vec![string_list_edit(ListEditAction::Add, ["--bar"])], - parse_string_list("--bar") + String::parse_list("--bar") ); } @@ -336,15 +361,15 @@ fn test_parse_string_list_implicit_add() { fn test_parse_scalar_list_implicit_add() { check!( vec![scalar_list_edit(ListEditAction::Add, vec![true])], - parse_bool_list("True") + bool::parse_list("True") ); check!( vec![scalar_list_edit(ListEditAction::Add, vec![-127])], - parse_int_list("-127") + i64::parse_list("-127") ); check!( vec![scalar_list_edit(ListEditAction::Add, vec![0.7])], - parse_float_list("0.7") + f64::parse_list("0.7") ); } @@ -352,22 +377,22 @@ fn test_parse_scalar_list_implicit_add() { fn test_parse_string_list_quoted_chars() { check!( vec![string_list_edit(ListEditAction::Add, vec!["[]"])], - parse_string_list(r"\[]"), + String::parse_list(r"\[]"), "Expected an implicit add of the literal string `[]` via an escaped opening `[`." ); check!( vec![string_list_edit(ListEditAction::Add, vec![" "])], - parse_string_list(r"\ "), + String::parse_list(r"\ "), "Expected an implicit add of the literal string ` `." ); check!( vec![string_list_edit(ListEditAction::Add, vec!["+"])], - parse_string_list(r"\+"), + String::parse_list(r"\+"), "Expected an implicit add of the literal string `+`." ); check!( vec![string_list_edit(ListEditAction::Add, vec!["-"])], - parse_string_list(r"\-"), + String::parse_list(r"\-"), "Expected an implicit add of the literal string `-`." ); check!( @@ -375,7 +400,7 @@ fn test_parse_string_list_quoted_chars() { ListEditAction::Replace, vec!["'foo", r"\"] )], - parse_string_list(r"['\'foo', '\\']") + String::parse_list(r"['\'foo', '\\']") ); } @@ -383,12 +408,12 @@ fn test_parse_string_list_quoted_chars() { fn test_parse_string_list_quote_forms() { check!( vec![string_list_edit(ListEditAction::Replace, ["foo"])], - parse_string_list(r#"["foo"]"#), + String::parse_list(r#"["foo"]"#), "Expected double quotes to work." ); check!( vec![string_list_edit(ListEditAction::Replace, ["foo", "bar"])], - parse_string_list(r#"["foo", 'bar']"#), + String::parse_list(r#"["foo", 'bar']"#), "Expected mixed quote forms to work." ); } @@ -397,11 +422,11 @@ fn test_parse_string_list_quote_forms() { fn test_parse_string_list_trailing_comma() { check!( vec![string_list_edit(ListEditAction::Replace, ["foo"])], - parse_string_list("['foo',]") + String::parse_list("['foo',]") ); check!( vec![string_list_edit(ListEditAction::Replace, ["foo", "bar"])], - parse_string_list("['foo','bar',]") + String::parse_list("['foo','bar',]") ); } @@ -409,15 +434,15 @@ fn test_parse_string_list_trailing_comma() { fn test_parse_scalar_list_trailing_comma() { check!( vec![scalar_list_edit(ListEditAction::Replace, [false, true])], - parse_bool_list("[false,true,]") + bool::parse_list("[false,true,]") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [42])], - parse_int_list("[42,]") + i64::parse_list("[42,]") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [42.0, -127.1])], - parse_float_list("[42.0,-127.1,]") + f64::parse_list("[42.0,-127.1,]") ); } @@ -425,11 +450,11 @@ fn test_parse_scalar_list_trailing_comma() { fn test_parse_string_list_whitespace() { check!( vec![string_list_edit(ListEditAction::Replace, ["foo"])], - parse_string_list(" [ 'foo' ] ") + String::parse_list(" [ 'foo' ] ") ); check!( vec![string_list_edit(ListEditAction::Replace, ["foo", "bar"])], - parse_string_list(" [ 'foo' , 'bar' , ] ") + String::parse_list(" [ 'foo' , 'bar' , ] ") ); } @@ -437,15 +462,15 @@ fn test_parse_string_list_whitespace() { fn test_parse_scalar_list_whitespace() { check!( vec![scalar_list_edit(ListEditAction::Replace, [true, false])], - parse_bool_list(" [ True, False ] ") + bool::parse_list(" [ True, False ] ") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [42])], - parse_int_list(" [ 42 ] ") + i64::parse_list(" [ 42 ] ") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [42.0, -127.1])], - parse_float_list(" [ 42.0 , -127.1 , ] ") + f64::parse_list(" [ 42.0 , -127.1 , ] ") ); } @@ -453,15 +478,15 @@ fn test_parse_scalar_list_whitespace() { fn test_parse_string_list_tuple() { check!( vec![string_list_edit(ListEditAction::Replace, EMPTY_STRING_LIST)], - parse_string_list("()") + String::parse_list("()") ); check!( vec![string_list_edit(ListEditAction::Replace, ["foo"])], - parse_string_list(r#"("foo")"#) + String::parse_list(r#"("foo")"#) ); check!( vec![string_list_edit(ListEditAction::Replace, ["foo", "bar"])], - parse_string_list(r#" ('foo', "bar",)"#) + String::parse_list(r#" ('foo', "bar",)"#) ); } @@ -469,15 +494,15 @@ fn test_parse_string_list_tuple() { fn test_parse_scalar_list_tuple() { check!( vec![scalar_list_edit(ListEditAction::Replace, EMPTY_INT_LIST)], - parse_int_list("()") + i64::parse_list("()") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [true])], - parse_bool_list("(True)") + bool::parse_list("(True)") ); check!( vec![scalar_list_edit(ListEditAction::Replace, [42.0, -127.1])], - parse_float_list(r#" (42.0, -127.1,)"#) + f64::parse_list(r#" (42.0, -127.1,)"#) ); } @@ -499,7 +524,7 @@ or '-' indicating `remove` at line 2 column 10" .to_owned(); assert_eq!( expected_error_msg, - parse_string_list(bad_input).unwrap_err().render("foo") + String::parse_list(bad_input).unwrap_err().render("foo") ) } @@ -521,7 +546,7 @@ or '-' indicating `remove` at line 2 column 10" .to_owned(); assert_eq!( expected_error_msg, - parse_int_list(bad_input).unwrap_err().render("foo") + i64::parse_list(bad_input).unwrap_err().render("foo") ) } @@ -648,3 +673,268 @@ fn test_parse_heterogeneous_dict() { ) ); } + +#[test] +fn test_expand_fromfile() { + let (_tmpdir, fromfile_pathbuf) = write_fromfile("fromfile.txt", "FOO"); + let fromfile_path_str = format!("{}", fromfile_pathbuf.display()); + assert_eq!( + Ok(Some(fromfile_path_str.clone())), + expand(fromfile_path_str.clone()) + ); + assert_eq!( + Ok(Some("FOO".to_string())), + expand(format!("@{}", fromfile_path_str)) + ); + assert_eq!(Ok(None), expand("@?/does/not/exist".to_string())); + let err = expand("@/does/not/exist".to_string()).unwrap_err(); + assert!(err + .render("XXX") + .starts_with("Problem reading /does/not/exist for XXX: No such file or directory")) +} + +#[test] +fn test_expand_fromfile_to_list() { + fn expand_fromfile( + content: &str, + prefix: &str, + filename: &str, + ) -> Result>>, ParseError> { + let (_tmpdir, _) = write_fromfile(filename, content); + expand_to_list(format!( + "{prefix}{}", + _tmpdir.path().join(filename).display() + )) + } + + fn do_test( + content: &str, + expected: &[ListEdit], + filename: &str, + ) { + let res = expand_fromfile(content, "@", filename); + assert_eq!(expected.to_vec(), res.unwrap().unwrap()); + } + + fn add(items: Vec) -> ListEdit { + return ListEdit { + action: ListEditAction::Add, + items: items, + }; + } + + fn remove(items: Vec) -> ListEdit { + return ListEdit { + action: ListEditAction::Remove, + items: items, + }; + } + + fn replace(items: Vec) -> ListEdit { + return ListEdit { + action: ListEditAction::Replace, + items: items, + }; + } + + do_test( + "EXPANDED", + &[add(vec!["EXPANDED".to_string()])], + "fromfile.txt", + ); + do_test( + "['FOO', 'BAR']", + &[replace(vec!["FOO".to_string(), "BAR".to_string()])], + "fromfile.txt", + ); + do_test( + "+['FOO', 'BAR'],-['BAZ']", + &[ + add(vec!["FOO".to_string(), "BAR".to_string()]), + remove(vec!["BAZ".to_string()]), + ], + "fromfile.txt", + ); + do_test( + "[\"FOO\", \"BAR\"]", + &[replace(vec!["FOO".to_string(), "BAR".to_string()])], + "fromfile.json", + ); + do_test( + "- FOO\n- BAR\n", + &[replace(vec!["FOO".to_string(), "BAR".to_string()])], + "fromfile.yaml", + ); + + do_test("true", &[add(vec![true])], "fromfile.txt"); + do_test( + "[true, false]", + &[replace(vec![true, false])], + "fromfile.json", + ); + do_test( + "- true\n- false\n", + &[replace(vec![true, false])], + "fromfile.yaml", + ); + + do_test("-42", &[add(vec![-42])], "fromfile.txt"); + do_test("[10, 12]", &[replace(vec![10, 12])], "fromfile.json"); + do_test("- 22\n- 44\n", &[replace(vec![22, 44])], "fromfile.yaml"); + + do_test("-5.6", &[add(vec![-5.6])], "fromfile.txt"); + do_test("-[3.14]", &[remove(vec![3.14])], "fromfile.txt"); + do_test("[3.14]", &[replace(vec![3.14])], "fromfile.json"); + do_test( + "- 11.22\n- 33.44\n", + &[replace(vec![11.22, 33.44])], + "fromfile.yaml", + ); + + check_err!( + expand_fromfile::("THIS IS NOT JSON", "@", "invalid.json"), + "expected value at line 1 column 1", + ); + + check_err!( + expand_fromfile::("{}", "@", "wrong_type.json"), + "invalid type: map, expected a sequence at line 1 column 0", + ); + + check_err!( + expand_fromfile::("[1, \"FOO\"]", "@", "wrong_type.json"), + "invalid type: string \"FOO\", expected i64 at line 1 column 9", + ); + + check_err!( + expand_fromfile::("THIS IS NOT YAML", "@", "invalid.yml"), + "invalid type: string \"THIS IS NOT YAML\", expected a sequence", + ); + + check_err!( + expand_fromfile::("- 1\n- true", "@", "wrong_type.yaml"), + "invalid type: boolean `true`, expected i64 at line 2 column 3", + ); + + check_err!( + expand_to_list::("@/does/not/exist".to_string()), + "Problem reading /does/not/exist for XXX: No such file or directory (os error 2)", + ); + + assert_eq!( + Ok(None), + expand_to_list::("@?/does/not/exist".to_string()) + ); + + // Test an optional fromfile that does exist, to ensure we handle the `?` in this case. + let res = expand_fromfile::("[1, 2]", "@?", "fromfile.json"); + assert_eq!(vec![replace(vec![1, 2])], res.unwrap().unwrap()); +} + +#[test] +fn test_expand_fromfile_to_dict() { + fn expand_fromfile( + content: &str, + prefix: &str, + filename: &str, + ) -> Result, ParseError> { + let (_tmpdir, _) = write_fromfile(filename, content); + expand_to_dict(format!( + "{prefix}{}", + _tmpdir.path().join(filename).display() + )) + } + + fn do_test(content: &str, expected: &DictEdit, filename: &str) { + let res = expand_fromfile(content, "@", filename); + assert_eq!(*expected, res.unwrap().unwrap()) + } + + fn add(items: HashMap) -> DictEdit { + return DictEdit { + action: DictEditAction::Add, + items, + }; + } + + fn replace(items: HashMap) -> DictEdit { + return DictEdit { + action: DictEditAction::Replace, + items, + }; + } + + do_test( + "{'FOO': 42}", + &replace(hashmap! {"FOO".to_string() => Val::Int(42),}), + "fromfile.txt", + ); + + do_test( + "+{'FOO': [True, False]}", + &add(hashmap! {"FOO".to_string() => Val::List(vec![Val::Bool(true), Val::Bool(false)]),}), + "fromfile.txt", + ); + + let complex_obj = replace(hashmap! { + "FOO".to_string() => Val::Dict(hashmap! { + "BAR".to_string() => Val::Float(3.14), + "BAZ".to_string() => Val::Dict(hashmap! { + "QUX".to_string() => Val::Bool(true), + "QUUX".to_string() => Val::List(vec![ Val::Int(1), Val::Int(2)]) + }) + }),}); + + do_test( + "{\"FOO\": {\"BAR\": 3.14, \"BAZ\": {\"QUX\": true, \"QUUX\": [1, 2]}}}", + &complex_obj, + "fromfile.json", + ); + do_test( + r#" + FOO: + BAR: 3.14 + BAZ: + QUX: true + QUUX: + - 1 + - 2 + "#, + &complex_obj, + "fromfile.yaml", + ); + + check_err!( + expand_fromfile("THIS IS NOT JSON", "@", "invalid.json"), + "expected value at line 1 column 1", + ); + + check_err!( + expand_fromfile("[1, 2]", "@", "wrong_type.json"), + "invalid type: sequence, expected a map at line 1 column 0", + ); + + check_err!( + expand_fromfile("THIS IS NOT YAML", "@", "invalid.yml"), + "invalid type: string \"THIS IS NOT YAML\", expected a map", + ); + + check_err!( + expand_fromfile("- 1\n- 2", "@", "wrong_type.yaml"), + "invalid type: sequence, expected a map", + ); + + check_err!( + expand_to_dict("@/does/not/exist".to_string()), + "Problem reading /does/not/exist for XXX: No such file or directory (os error 2)", + ); + + assert_eq!(Ok(None), expand_to_dict("@?/does/not/exist".to_string())); + + // Test an optional fromfile that does exist, to ensure we handle the `?` in this case. + let res = expand_fromfile("{'FOO': 42}", "@?", "fromfile.txt"); + assert_eq!( + replace(hashmap! {"FOO".to_string() => Val::Int(42),}), + res.unwrap().unwrap() + ); +}