new: nested objects flattening feature (#229)

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Build
         run: cargo build --verbose --benches
       - name: Run tests

.github/workflows/coverage.yml

@@ -10,7 +10,7 @@ jobs:
       options: --security-opt seccomp=unconfined
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Generate code coverage
         run: |

.github/workflows/publish.yml

@@ -87,7 +87,7 @@ jobs:
             asset: hl-windows.zip
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:

Cargo.toml

@@ -4,7 +4,7 @@ members = [".", "crate/encstr"]
 [workspace.package]
 repository = "https://github.com/pamburus/hl"
 authors = ["Pavel Ivanov <mr.pavel.ivanov@gmail.com>"]
-version = "0.28.2-alpha.2"
+version = "0.29.0-alpha.1"
 edition = "2021"
 license = "MIT"
 

README.md

@@ -488,6 +488,7 @@ Output Options:
       --no-raw                Disable raw source messages output, overrides --raw option
       --raw-fields            Output field values as is, without unescaping or prettifying
   -h, --hide <KEY>            Hide or reveal fields with the specified keys, prefix with ! to reveal, specify '!*' to reveal all
+      --flatten <WHEN>        Whether to flatten objects [env: HL_FLATTEN=] [default: always] [possible values: never, always]
   -t, --time-format <FORMAT>  Time format, see https://man7.org/linux/man-pages/man1/date.1.html [env: HL_TIME_FORMAT=] [default: "%b %d %T.%3N"]
   -Z, --time-zone <TZ>        Time zone name, see column "TZ identifier" at https://en.wikipedia.org/wiki/List_of_tz_database_time_zones [env: HL_TIME_ZONE=] [default: UTC]
   -L, --local                 Use local time zone, overrides --time-zone option

etc/defaults/config.yaml

@@ -52,6 +52,7 @@ fields:
 
 # Formatting settings.
 formatting:
+  flatten: always
   punctuation:
     logger-name-separator: ':'
     field-key-value-separator: '='

src/app.rs

@@ -74,6 +74,7 @@ pub struct Options {
     pub tail: u64,
     pub delimiter: Delimiter,
     pub unix_ts_unit: Option<UnixTimestampUnit>,
+    pub flatten: bool,
 }
 
 impl Options {
@@ -85,8 +86,24 @@ impl Options {
             (false, Some(query)) => Box::new((&self.filter).and(query)),
         }
     }
+
+    #[cfg(test)]
+    fn with_theme(self, theme: Arc<Theme>) -> Self {
+        Self { theme, ..self }
+    }
+
+    #[cfg(test)]
+    fn with_fields(self, fields: FieldOptions) -> Self {
+        Self { fields, ..self }
+    }
+
+    #[cfg(test)]
+    fn with_raw_fields(self, raw_fields: bool) -> Self {
+        Self { raw_fields, ..self }
+    }
 }
 
+#[derive(Default)]
 pub struct FieldOptions {
     pub filter: Arc<IncludeExcludeKeyFilter>,
     pub settings: Fields,
@@ -639,7 +656,8 @@ impl App {
                     self.options.fields.filter.clone(),
                     self.options.formatting.clone(),
                 )
-                .with_field_unescaping(!self.options.raw_fields),
+                .with_field_unescaping(!self.options.raw_fields)
+                .with_flatten(self.options.flatten),
             )
         }
     }
@@ -997,3 +1015,160 @@ where
         i += 1;
     }
 }
+
+// ---
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::io::Cursor;
+
+    use chrono_tz::UTC;
+
+    use crate::{filtering::MatchOptions, themecfg::testing, LinuxDateFormat};
+
+    #[test]
+    fn test_common_prefix_len() {
+        let items = vec!["abc", "abcd", "ab", "ab"];
+        assert_eq!(common_prefix_len(&items), 2);
+    }
+
+    #[test]
+    fn test_cat_empty() {
+        let input = input("");
+        let mut output = Vec::new();
+        let app = App::new(options());
+        app.run(vec![input], &mut output).unwrap();
+        assert_eq!(std::str::from_utf8(&output).unwrap(), "");
+    }
+
+    #[test]
+    fn test_cat_one_line() {
+        let input = input(
+            r#"{"caller":"main.go:539","duration":"15d","level":"info","msg":"No time or size retention was set so using the default time retention","ts":"2023-12-07T20:07:05.949Z"}"#,
+        );
+        let mut output = Vec::new();
+        let app = App::new(options());
+        app.run(vec![input], &mut output).unwrap();
+        assert_eq!(
+            std::str::from_utf8(&output).unwrap(),
+            "2023-12-07 20:07:05.949 |INF| No time or size retention was set so using the default time retention duration=15d @ main.go:539\n",
+        );
+    }
+
+    #[test]
+    fn test_cat_with_theme() {
+        let input = input(
+            r#"{"caller":"main.go:539","duration":"15d","level":"info","msg":"No time or size retention was set so using the default time retention","ts":"2023-12-07T20:07:05.949Z"}"#,
+        );
+        let mut output = Vec::new();
+        let app = App::new(options().with_theme(theme()));
+        app.run(vec![input], &mut output).unwrap();
+        assert_eq!(
+            std::str::from_utf8(&output).unwrap(),
+            "\u{1b}[0;2;3m2023-12-07 20:07:05.949 \u{1b}[0;36m|INF| \u{1b}[0;1;39mNo time or size retention was set so using the default time retention \u{1b}[0;32mduration\u{1b}[0;2m=\u{1b}[0;39m15d\u{1b}[0;2;3m @ main.go:539\u{1b}[0m\n",
+        );
+    }
+
+    #[test]
+    fn test_cat_no_msg() {
+        let input =
+            input(r#"{"caller":"main.go:539","duration":"15d","level":"info","ts":"2023-12-07T20:07:05.949Z"}"#);
+        let mut output = Vec::new();
+        let app = App::new(options().with_theme(theme()));
+        app.run(vec![input], &mut output).unwrap();
+        assert_eq!(
+            std::str::from_utf8(&output).unwrap(),
+            "\u{1b}[0;2;3m2023-12-07 20:07:05.949 \u{1b}[0;36m|INF|\u{1b}[0m \u{1b}[0;32mduration\u{1b}[0;2m=\u{1b}[0;39m15d\u{1b}[0;2;3m @ main.go:539\u{1b}[0m\n",
+        );
+    }
+
+    #[test]
+    fn test_cat_msg_array() {
+        let input = input(
+            r#"{"caller":"main.go:539","duration":"15d","level":"info","ts":"2023-12-07T20:07:05.949Z","msg":["x","y"]}"#,
+        );
+        let mut output = Vec::new();
+        let app = App::new(options().with_theme(theme()));
+        app.run(vec![input], &mut output).unwrap();
+        assert_eq!(
+            std::str::from_utf8(&output).unwrap(),
+            "\u{1b}[0;2;3m2023-12-07 20:07:05.949 \u{1b}[0;36m|INF| \u{1b}[0;32mmsg\u{1b}[0;2m=\u{1b}[0;93m[\u{1b}[0;39mx\u{1b}[0;93m \u{1b}[0;39my\u{1b}[0;93m] \u{1b}[0;32mduration\u{1b}[0;2m=\u{1b}[0;39m15d\u{1b}[0;2;3m @ main.go:539\u{1b}[0m\n",
+        );
+    }
+
+    #[test]
+    fn test_cat_field_exclude() {
+        let input = input(
+            r#"{"caller":"main.go:539","duration":"15d","level":"info","ts":"2023-12-07T20:07:05.949Z","msg":"xy"}"#,
+        );
+        let mut output = Vec::new();
+        let mut ff = IncludeExcludeKeyFilter::new(MatchOptions::default());
+        ff.entry("duration").exclude();
+        let app = App::new(options().with_fields(FieldOptions {
+            filter: Arc::new(ff),
+            ..FieldOptions::default()
+        }));
+        app.run(vec![input], &mut output).unwrap();
+        assert_eq!(
+            std::str::from_utf8(&output).unwrap(),
+            "2023-12-07 20:07:05.949 |INF| xy ... @ main.go:539\n",
+        );
+    }
+
+    #[test]
+    fn test_cat_raw_fields() {
+        let input = input(
+            r#"{"caller":"main.go:539","duration":"15d","level":"info","ts":"2023-12-07T20:07:05.949Z","msg":"xy"}"#,
+        );
+        let mut output = Vec::new();
+        let mut ff = IncludeExcludeKeyFilter::new(MatchOptions::default());
+        ff.entry("duration").exclude();
+        let app = App::new(options().with_raw_fields(true));
+        app.run(vec![input], &mut output).unwrap();
+        assert_eq!(
+            std::str::from_utf8(&output).unwrap(),
+            "2023-12-07 20:07:05.949 |INF| xy duration=\"15d\" @ main.go:539\n",
+        );
+    }
+
+    fn input<S: Into<String>>(s: S) -> InputHolder {
+        InputHolder::new(InputReference::File("-".into()), Some(Box::new(Cursor::new(s.into()))))
+    }
+
+    fn options() -> Options {
+        Options {
+            theme: Arc::new(Theme::none()),
+            time_format: LinuxDateFormat::new("%Y-%m-%d %T.%3N").compile(),
+            raw: false,
+            raw_fields: false,
+            allow_prefix: false,
+            buffer_size: NonZeroUsize::new(4096).unwrap(),
+            max_message_size: NonZeroUsize::new(4096 * 1024).unwrap(),
+            concurrency: 1,
+            filter: Filter::default(),
+            query: None,
+            fields: FieldOptions::default(),
+            formatting: Formatting::default(),
+            time_zone: Tz::IANA(UTC),
+            hide_empty_fields: false,
+            sort: false,
+            follow: false,
+            sync_interval: Duration::from_secs(1),
+            input_info: None,
+            input_format: None,
+            dump_index: false,
+            debug: false,
+            app_dirs: None,
+            tail: 0,
+            delimiter: Delimiter::default(),
+            unix_ts_unit: None,
+            flatten: false,
+        }
+    }
+
+    fn theme() -> Arc<Theme> {
+        Arc::new(Theme::from(testing::theme().unwrap()))
+    }
+}

src/cli.rs

@@ -11,6 +11,7 @@ use crate::{
     config,
     error::*,
     level::{LevelValueParser, RelaxedLevel},
+    settings,
 };
 
 // ---
@@ -153,6 +154,21 @@ pub struct Opt {
     )]
     pub hide: Vec<String>,
 
+    /// Whether to flatten objects.
+    #[arg(
+        long,
+        env = "HL_FLATTEN",
+        value_name = "WHEN",
+        value_enum,
+        default_value_t = config::get().formatting.flatten.as_ref().map(|x| match x{
+            settings::FlattenOption::Never => FlattenOption::Never,
+            settings::FlattenOption::Always => FlattenOption::Always,
+        }).unwrap_or(FlattenOption::Always),
+        overrides_with = "flatten",
+        help_heading = heading::OUTPUT
+    )]
+    pub flatten: FlattenOption,
+
     /// Time format, see https://man7.org/linux/man-pages/man1/date.1.html.
     #[arg(
         short,
@@ -360,6 +376,12 @@ pub enum UnixTimestampUnit {
     Ns,
 }
 
+#[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FlattenOption {
+    Never,
+    Always,
+}
+
 mod heading {
     pub const FILTERING: &str = "Filtering Options";
     pub const INPUT: &str = "Input Options";