Alter the parser and introduce new rule (#1066)

Co-authored-by: Loris Moulin <45130584+l0r1s@users.noreply.github.com>

crates/parser/src/ast.rs

@@ -12,6 +12,12 @@ pub enum Operator {
     IsAtMost,
 }
 
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Copy, Clone)]
+pub enum MathOps {
+    Minus,
+    Plus,
+}
+
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub struct Comparison {
     pub op: Operator,
@@ -43,6 +49,16 @@ pub enum AssertionKind {
         #[serde(with = "optional_timeout")]
         timeout: Option<Duration>,
     },
+    CalcMetrics {
+        node_name: NodeName,
+        metric_name_a: String,
+        math_ops: MathOps,
+        metric_name_b: String,
+        op: Operator,
+        target_value: u64,
+        #[serde(with = "optional_timeout")]
+        timeout: Option<Duration>,
+    },
     ParaRuntimeUpgrade {
         node_name: NodeName,
         para_id: ParaId,

crates/parser/src/lib.rs

@@ -61,6 +61,18 @@ fn parse_taget_value(pair: Pair<Rule>) -> Result<u64, ParserError> {
         .map_err(|_| ParserError::ParseError(format!("Can't parse {target_str} as u64")))
 }
 
+fn parse_math_ops(pair: Pair<Rule>) -> Result<ast::MathOps, ParserError> {
+    let mut pairs = pair.into_inner();
+    let math_ops = get_pair(&mut pairs, "math_ops")?;
+    let sign = match math_ops.as_rule() {
+        Rule::plus => ast::MathOps::Plus,
+        Rule::minus => ast::MathOps::Minus,
+        _ => return Err(ParserError::UnreachableRule(format!("{math_ops:?}"))),
+    };
+
+    Ok(sign)
+}
+
 fn parse_comparison(pair: Pair<Rule>) -> Result<ast::Comparison, ParserError> {
     let mut inner_pairs = pair.into_inner();
     let op_rule = get_pair(&mut inner_pairs, "op_rule")?;
@@ -333,6 +345,36 @@ pub fn parse(unparsed_file: &str) -> Result<ast::TestDefinition, errors::ParserE
 
                 assertions.push(assertion);
             }
+            Rule::calc_metrics => {
+                // Pairs should be in order:
+                // name, para_id, block_height, math_ops, finalized_height, comparison [timeout]
+                let mut pairs = record.into_inner();
+                let name = parse_name(get_pair(&mut pairs, "name")?)?;
+                let metric_name_a = get_pair(&mut pairs, "metric_name")?.as_str().to_string();
+                let math_ops = parse_math_ops(get_pair(&mut pairs, "math_ops")?)?;
+                let metric_name_b = get_pair(&mut pairs, "metric_name")?.as_str().to_string();
+                let comparison = parse_comparison(get_pair(&mut pairs, "comparison")?)?;
+                let timeout: Option<Duration> = if let Some(within_rule) = pairs.next() {
+                    Some(parse_within(within_rule)?)
+                } else {
+                    None
+                };
+
+                let assertion = Assertion {
+                    parsed: AssertionKind::CalcMetrics {
+                        node_name: name.to_owned(),
+                        metric_name_a,
+                        math_ops,
+                        metric_name_b,
+                        op: comparison.op,
+                        target_value: comparison.target_value,
+                        timeout,
+                    },
+                    original_line,
+                };
+
+                assertions.push(assertion);
+            }
             Rule::para_runtime_upgrade => {
                 // Pairs should be in order:
                 // name, para_id, file_or_uri, [timeout]
@@ -655,7 +697,7 @@ fn get_pair<'a>(
     match pairs.next() {
         Some(p) => Ok(p),
         None => Err(ParserError::Unexpected(format!(
-            "Pair {rule_name} should exists"
+            "Pair {rule_name} should exist"
         ))),
     }
 }

crates/parser/src/tests.rs

@@ -167,6 +167,38 @@ fn report_parse_ok() {
     assert_eq!(result, t);
 }
 
+#[test]
+fn report_parse_calc_ok() {
+    let line: &str =
+        r#"alice: reports block height minus finalised block is at least 10 within 200 seconds"#;
+    let data = r#"{
+      "description": null,
+      "network": "./a.toml",
+      "creds": "config",
+      "assertions": [
+        {
+          "original_line": "alice: reports block height minus finalised block is at least 10 within 200 seconds",
+          "parsed": {
+            "fn": "CalcMetrics",
+            "args": {
+              "node_name": "alice",
+              "metric_name_a": "block height",
+              "math_ops":  "Minus",
+              "metric_name_b": "finalised block",
+              "op": "IsAtLeast",
+              "target_value": 10,
+              "timeout": 200
+            }
+          }
+        }
+      ]
+    }"#;
+    let t: TestDefinition = serde_json::from_str(data).unwrap();
+
+    let result = parse(&[NETWORK, CREDS, line].join("\n")).unwrap();
+    assert_eq!(result, t);
+}
+
 #[test]
 fn para_dummy_upgrade_parse_ok() {
     let line: &str = r#"alice: parachain 100 perform dummy upgrade within 200 seconds"#;

crates/parser/src/zombienet.pest

@@ -41,7 +41,12 @@ op_lt        = { "<" | "is lower than" }
 op_gt        = { ">" | "is greater than" | "greater than" }
 op_eq        = { "==" | "=" | "equals" | "is equal to" | "is" }
 op_ineq      = { "!=" }
-comparison = { (op_lte | op_gte | op_gt | op_lt | op_eq | op_ineq) ~ int+ }
+comparison   = { (op_lte | op_gte | op_gt | op_lt | op_eq | op_ineq) ~ int+ }
+
+// Math ops
+minus       = { "-" | "minus" }
+plus        = { "+" | "plus" }
+math_ops    = { plus | minus }
 
 // commons
 node_name = { name ~ colon }
@@ -59,6 +64,7 @@ creds = { "Creds:" ~ ("config" | file_path) }
 is_up = { node_name ~ "is up" ~ within? }
 para_is_registered = { node_name ~ parachain ~ "is registered" ~ within? }
 para_block_height = { node_name ~ parachain ~ "block height" ~ comparison ~ within? }
+calc_metrics = { node_name ~ "reports" ~ metric_name ~ math_ops ~ metric_name ~ comparison ~ within? }
 para_runtime_upgrade = { node_name ~ parachain ~ "perform upgrade with" ~ ( uri | file_path ) ~ within? }
 para_runtime_dummy_upgrade = { node_name ~ parachain ~ "perform dummy upgrade" ~ within? }
 histogram = { node_name ~ "reports histogram" ~ metric_name ~ "has" ~ (comparison | int+) ~ "samples in buckets" ~ square_brackets_strings ~ within? }
@@ -90,6 +96,7 @@ file = { SOI ~ (
     is_up |
     para_is_registered |
     para_block_height |
+    calc_metrics |
     para_runtime_upgrade |
     para_runtime_dummy_upgrade |
     histogram |

examples/0001-small-network.zndsl

@@ -22,4 +22,8 @@ alice: system event matches glob "*was backed*" within 10 seconds
 
 #parachain tests
 alice: parachain 100 is registered within 225 seconds
-alice: parachain 100 block height is at least 10 within 200 seconds
+alice: parachain 100 block height is at least 10 within 200 seconds
+
+## test the block height - (or minus) finalised block
+alice: reports block height minus finalised block is lower than 10 within 20 seconds
+alice: reports block height - finalised block is lower than 10 within 20 seconds

examples/0003-big-network.zndsl

@@ -5,3 +5,7 @@ Creds: config
 # metrics
 a: reports node_roles is 4
 b: reports sub_libp2p_is_major_syncing is 0
+
+## test the block height - (or minus) finalised block
+a: reports block height minus finalised block is lower than 10 within 20 seconds
+b: reports block height - finalised block is lower than 10 within 20 seconds

javascript/packages/orchestrator/src/test-runner/assertions.ts

@@ -79,6 +79,49 @@ const Report = ({
   };
 };
 
+const CalcMetrics = ({
+  node_name,
+  metric_name_a,
+  math_ops,
+  metric_name_b,
+  target_value,
+  op,
+  timeout,
+}: FnArgs) => {
+  const comparatorFn = comparators[op!];
+  const mathFn = (a: number, b: number): number => {
+    return math_ops === "Minus" ? a - b : a + b;
+  };
+
+  return async (network: Network) => {
+    const nodes = network.getNodes(node_name!);
+    const promise_a = nodes.map((node: any) =>
+      node.getMetric(
+        metric_name_a!,
+        toChaiComparator(op!),
+        target_value!,
+        timeout || DEFAULT_INDIVIDUAL_TEST_TIMEOUT,
+      ),
+    );
+
+    const promise_b = nodes.map((node: any) =>
+      node.getMetric(
+        metric_name_b!,
+        toChaiComparator(op!),
+        target_value!,
+        timeout || DEFAULT_INDIVIDUAL_TEST_TIMEOUT,
+      ),
+    );
+
+    const values = await Promise.all([...promise_a, ...promise_b]);
+
+    for (let i = 0; i++; i < nodes.length) {
+      const value = mathFn(values[i], values[i + nodes.length]);
+      comparatorFn(value as number, target_value as number);
+    }
+  };
+};
+
 const Histogram = ({
   node_name,
   metric_name,
@@ -462,6 +505,7 @@ export default {
   SystemEvent,
   CustomJs,
   CustomSh,
+  CalcMetrics,
   ParaBlockHeight,
   ParaIsRegistered,
   ParaRuntimeUpgrade,

javascript/packages/orchestrator/src/types.ts

@@ -318,6 +318,9 @@ export interface FnArgs {
   timeout?: number;
   target_value?: number;
   metric_name?: string;
+  metric_name_a?: string;
+  metric_name_b?: string;
+  math_ops?: string;
   buckets?: string[];
   span_id?: string;
   op?: string;