new: configuration options to ignore and hide fields

Cargo.toml

@@ -4,7 +4,7 @@ categories = ["command-line-utilities"]
 description = "Utility for viewing json-formatted log files."
 keywords = ["cli", "human", "log"]
 name = "hl"
-version = "0.9.6"
+version = "0.9.7"
 edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -39,6 +39,7 @@ shellwords = "1"
 signal-hook = "0"
 structopt = "0"
 thiserror = "1"
+wildmatch = "2"
 
 [dependencies.itoa]
 version = "0"
@@ -54,19 +55,29 @@ features = ["handleapi"]
 
 [dev-dependencies]
 criterion = "0"
+stats_alloc = "0"
 diligent-date-parser = "0"
 regex = "1"
+wildmatch = "2"
 
 [profile.release]
 debug = false
 opt-level = 3
 codegen-units = 1
 lto = true
 
+[[bench]]
+name = "re_match"
+harness = false
+
 [[bench]]
 name = "ts_parse"
 harness = false
 
 [[bench]]
 name = "ts_format"
 harness = false
+
+[[bench]]
+name = "wild_match"
+harness = false

benches/re_match.rs

@@ -0,0 +1,52 @@
+// std imports
+use std::alloc::System;
+
+// third-party imports
+use criterion::{criterion_group, criterion_main, Criterion};
+use regex::Regex;
+use stats_alloc::{Region, StatsAlloc, INSTRUMENTED_SYSTEM};
+
+#[global_allocator]
+static GLOBAL: &StatsAlloc<System> = &INSTRUMENTED_SYSTEM;
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let re = Regex::new(r"^_").unwrap();
+
+    let mut c1 = None;
+    let mut n1 = 0;
+    c.bench_function("regex-short-match", |b| {
+        let reg = Region::new(&GLOBAL);
+        b.iter(|| {
+            assert_eq!(re.is_match("_TEST"), true);
+            n1 += 1;
+        });
+        c1 = Some(reg.change());
+    });
+    println!("allocations at 1 ({:?} iterations): {:#?}", n1, c1);
+
+    let mut c2 = None;
+    let mut n2 = 0;
+    c.bench_function("regex-long-match", |b| {
+        let reg = Region::new(&GLOBAL);
+        b.iter(|| {
+            assert_eq!(re.is_match("_TEST_SOME_VERY_VERY_LONG_NAME"), true);
+            n2 += 1;
+        });
+        c2 = Some(reg.change());
+    });
+    println!("allocations at 2 ({:?} iterations): {:#?}", n2, c2);
+
+    c.bench_function("regex-short-non-match", |b| {
+        b.iter(|| {
+            assert_eq!(re.is_match("TEST"), false);
+        });
+    });
+    c.bench_function("regex-long-non-match", |b| {
+        b.iter(|| {
+            assert_eq!(re.is_match("TEST_SOME_VERY_VERY_LONG_NAME"), false);
+        });
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/ts_format.rs

@@ -7,7 +7,7 @@ use hl::datefmt::{DateTimeFormatter, LinuxDateFormat};
 use hl::timestamp::Timestamp;
 
 fn criterion_benchmark(c: &mut Criterion) {
-    let tsr = Timestamp::new("2020-06-27T00:48:30.466249792+00:00");
+    let tsr = Timestamp::new("2020-06-27T00:48:30.466249792+00:00", None);
     let ts = tsr.parse().unwrap();
     let tsn = ts.naive_local();
     c.bench_function("chrono conversion to naive local", |b| {

benches/ts_parse.rs

@@ -13,19 +13,19 @@ fn criterion_benchmark(c: &mut Criterion) {
         b.iter(|| assert!(re.is_match(rfc3339)));
     });
     c.bench_function("as_rfc3339", |b| {
-        let ts = Timestamp::new(rfc3339);
+        let ts = Timestamp::new(rfc3339, None);
         b.iter(|| assert!(ts.as_rfc3339().is_some()));
     });
     c.bench_function("parse unix", |b| {
-        let ts = Timestamp::new(unix);
+        let ts = Timestamp::new(unix, None);
         b.iter(|| assert!(ts.parse().is_some()))
     });
     c.bench_function("parse unix microseconds", |b| {
-        let ts = Timestamp::new(unix_us);
+        let ts = Timestamp::new(unix_us, None);
         b.iter(|| assert!(ts.parse().is_some()))
     });
     c.bench_function("parse rfc3339", |b| {
-        let ts = Timestamp::new(rfc3339);
+        let ts = Timestamp::new(rfc3339, None);
         b.iter(|| assert!(ts.parse().is_some()))
     });
 }

benches/wild_match.rs

@@ -0,0 +1,77 @@
+// std imports
+use std::alloc::System;
+
+// third-party imports
+use criterion::{criterion_group, criterion_main, Criterion};
+use stats_alloc::{Region, StatsAlloc, INSTRUMENTED_SYSTEM};
+use wildmatch::WildMatch;
+
+#[global_allocator]
+static GLOBAL: &StatsAlloc<System> = &INSTRUMENTED_SYSTEM;
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let pattern = WildMatch::new(r"_*");
+    let prefix = String::from("_");
+
+    let mut c1 = None;
+    let mut n1 = 0;
+    c.bench_function("wild-short-match", |b| {
+        let reg = Region::new(&GLOBAL);
+        b.iter(|| {
+            assert_eq!(pattern.matches("_TEST"), true);
+            n1 += 1;
+        });
+        c1 = Some(reg.change());
+    });
+    println!("allocations at 1 ({:?} iterations): {:#?}", n1, c1);
+
+    let mut c2 = None;
+    let mut n2 = 0;
+    c.bench_function("wild-long-match", |b| {
+        let reg = Region::new(&GLOBAL);
+        b.iter(|| {
+            assert_eq!(pattern.matches("_TEST_SOME_VERY_VERY_LONG_NAME"), true);
+            n2 += 1;
+        });
+        c2 = Some(reg.change());
+    });
+    println!("allocations at 2 ({:?} iterations): {:#?}", n2, c2);
+
+    c.bench_function("wild-short-non-match", |b| {
+        b.iter(|| {
+            assert_eq!(pattern.matches("TEST"), false);
+        });
+    });
+    c.bench_function("wild-long-non-match", |b| {
+        b.iter(|| {
+            assert_eq!(pattern.matches("TEST_SOME_VERY_VERY_LONG_NAME"), false);
+        });
+    });
+    c.bench_function("compare-short-match", |b| {
+        let what = String::from("_TEST");
+        b.iter(|| {
+            assert_eq!(what.starts_with(&prefix), true);
+        });
+    });
+    c.bench_function("compare-long-match", |b| {
+        let what = String::from("_TEST_SOME_VERY_VERY_LONG_NAME");
+        b.iter(|| {
+            assert_eq!(what.starts_with(&prefix), true);
+        });
+    });
+    c.bench_function("compare-short-non-match", |b| {
+        let what = String::from("TEST");
+        b.iter(|| {
+            assert_eq!(what.starts_with(&prefix), false);
+        });
+    });
+    c.bench_function("compare-long-non-match", |b| {
+        let what = String::from("TEST_SOME_VERY_VERY_LONG_NAME");
+        b.iter(|| {
+            assert_eq!(what.starts_with(&prefix), false);
+        });
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

etc/defaults/config.yaml

@@ -5,30 +5,36 @@ time-format: '%b %d %T.%3N'
 # https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 time-zone: UTC
 
-# Configuration of the predefined set of fields
+# Settings for fields processing
 fields:
-  time:
-    names: [ts, TS, time, TIME, Time, _SOURCE_REALTIME_TIMESTAMP, __REALTIME_TIMESTAMP]
-  logger:
-    names: [logger, LOGGER, Logger]
-  level:
-    variants:
-      - names: [level, LEVEL, Level]
-        values:
-          debug: [debug]
-          info: [info, information]
-          warning: [warning, warn]
-          error: [error, err, fatal, critical, panic]
-      - names: [PRIORITY]
-        values:
-          debug: [7]
-          info: [6]
-          warning: [5, 4]
-          error: [3, 2, 1]
-  message:
-    names: [msg, message, MESSAGE, Message]
-  caller:
-    names: [caller, CALLER, Caller]
+  # Configuration of the predefined set of fields
+  predefined:
+    time:
+      names: [ts, TS, time, TIME, Time, _SOURCE_REALTIME_TIMESTAMP, __REALTIME_TIMESTAMP]
+    logger:
+      names: [logger, LOGGER, Logger]
+    level:
+      variants:
+        - names: [level, LEVEL, Level]
+          values:
+            debug: [debug]
+            info: [info, information]
+            warning: [warning, warn]
+            error: [error, err, fatal, critical, panic]
+        - names: [PRIORITY]
+          values:
+            debug: [7]
+            info: [6]
+            warning: [5, 4]
+            error: [3, 2, 1]
+    message:
+      names: [msg, message, MESSAGE, Message]
+    caller:
+      names: [caller, CALLER, Caller]
+  # List of wildcard field names to ignore
+  ignore: ['_*']
+  # List of exact field names to hide
+  hide: []
 
 # Number of processing threads, configured automatically based on CPU count if not specified
 concurrency: ~

src/main.rs

@@ -302,6 +302,9 @@ fn run() -> Result<()> {
     for key in opt.show {
         fields.entry(&key).include();
     }
+    for key in &CONFIG.fields.hide {
+        fields.entry(&key).exclude();
+    }
 
     let max_message_size = opt.max_message_size;
     let buffer_size = std::cmp::min(max_message_size, opt.buffer_size);