Mimir query engine: add tool to find disabled but supported test cases (

#9018)

* Add tool to find supported but disabled test cases.

* Enable supported test cases.

* Add a readme.

* Add changelog entry.

CHANGELOG.md

@@ -25,7 +25,7 @@
 * [CHANGE] Distributor: Promote replying with `Retry-After` header on retryable errors to stable and set `-distributor.retry-after-header.enabled=true` by default. #8694
 * [CHANGE] Distributor: Replace `-distributor.retry-after-header.max-backoff-exponent` and `-distributor.retry-after-header.base-seconds` with `-distributor.retry-after-header.min-backoff` and `-distributor.retry-after-header.max-backoff` for easier configuration. #8694
 * [CHANGE] Ingester: increase the default inactivity timeout of active series (`-ingester.active-series-metrics-idle-timeout`) from `10m` to `20m`. #8975
-* [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.query-engine=mimir`. #8422 #8430 #8454 #8455 #8360 #8490 #8508 #8577 #8660 #8671 #8677 #8747 #8850 #8872 #8838 #8911 #8909 #8923 #8924 #8925 #8932 #8933 #8934 #8962 #8986 #8993 #8995 #9017
+* [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.query-engine=mimir`. #8422 #8430 #8454 #8455 #8360 #8490 #8508 #8577 #8660 #8671 #8677 #8747 #8850 #8872 #8838 #8911 #8909 #8923 #8924 #8925 #8932 #8933 #8934 #8962 #8986 #8993 #8995 #9017 #9018
 * [FEATURE] Experimental Kafka-based ingest storage. #6888 #6894 #6929 #6940 #6951 #6974 #6982 #7029 #7030 #7091 #7142 #7147 #7148 #7153 #7160 #7193 #7349 #7376 #7388 #7391 #7393 #7394 #7402 #7404 #7423 #7424 #7437 #7486 #7503 #7508 #7540 #7621 #7682 #7685 #7694 #7695 #7696 #7697 #7701 #7733 #7734 #7741 #7752 #7838 #7851 #7871 #7877 #7880 #7882 #7887 #7891 #7925 #7955 #7967 #8031 #8063 #8077 #8088 #8135 #8176 #8184 #8194 #8216 #8217 #8222 #8233 #8503 #8542 #8579 #8657 #8686 #8688 #8703 #8706 #8708 #8738 #8750 #8778 #8808 #8809 #8841 #8842 #8845 #8853 #8886 #8988
   * What it is:
     * When the new ingest storage architecture is enabled, distributors write incoming write requests to a Kafka-compatible backend, and the ingesters asynchronously replay ingested data from Kafka. In this architecture, the write and read path are de-coupled through a Kafka-compatible backend. The write path and Kafka load is a function of the incoming write traffic, the read path load is a function of received queries. Whatever the load on the read path, it doesn't affect the write path.

pkg/streamingpromql/testdata/upstream/aggregators.test

@@ -123,7 +123,6 @@ eval instant at 50m SUM(http_requests) BY (job, group)
 # 	{job="api-server"} 100
 # 	{job="app-server"} 500
 
-# Unsupported by streaming engine.
 # Unsupported by streaming engine.
 # eval instant at 50m MAX(http_requests) BY (job)
 # 	{job="api-server"} 400
@@ -250,10 +249,9 @@ load 5m
   label_grouping_test{a="aa", b="bb"} 0+10x10
   label_grouping_test{a="a", b="abb"} 0+20x10
 
-# Unsupported by streaming engine.
-# eval instant at 50m sum(label_grouping_test) by (a, b)
-#   {a="a", b="abb"} 200
-#   {a="aa", b="bb"} 100
+eval instant at 50m sum(label_grouping_test) by (a, b)
+  {a="a", b="abb"} 200
+  {a="aa", b="bb"} 100
 
 
 

pkg/streamingpromql/testdata/upstream/at_modifier.test

@@ -94,9 +94,8 @@ eval instant at 25s sum_over_time(metric{job="1"}[100] offset 50s @ 100)
   {job="1"} 15
 
 # Different timestamps.
-# Unsupported by streaming engine.
-# eval instant at 25s metric{job="1"} @ 50 + metric{job="1"} @ 100
-#   {job="1"} 15
+eval instant at 25s metric{job="1"} @ 50 + metric{job="1"} @ 100
+  {job="1"} 15
 
 # Unsupported by streaming engine.
 # eval instant at 25s rate(metric{job="1"}[100s] @ 100) + label_replace(rate(metric{job="2"}[123s] @ 200), "job", "1", "", "")

pkg/streamingpromql/testdata/upstream/functions.test

@@ -242,9 +242,8 @@ load 5m
 	http_requests{job="app-server", instance="1", group="canary"}		0+80x10
 
 # deriv should return the same as rate in simple cases.
-# Unsupported by streaming engine.
-# eval instant at 50m rate(http_requests{group="canary", instance="1", job="app-server"}[50m])
-# 	{group="canary", instance="1", job="app-server"} 0.26666666666666666
+eval instant at 50m rate(http_requests{group="canary", instance="1", job="app-server"}[50m])
+	{group="canary", instance="1", job="app-server"} 0.26666666666666666
 
 # Unsupported by streaming engine.
 # eval instant at 50m deriv(http_requests{group="canary", instance="1", job="app-server"}[50m])

pkg/streamingpromql/testdata/upstream/subquery.test.disabled → pkg/streamingpromql/testdata/upstream/subquery.test

@@ -7,26 +7,32 @@ load 10s
   metric 1 2
 
 # Evaluation before 0s gets no sample.
-eval instant at 10s sum_over_time(metric[50s:10s])
-  {} 3
+# Unsupported by streaming engine.
+# eval instant at 10s sum_over_time(metric[50s:10s])
+#   {} 3
 
-eval instant at 10s sum_over_time(metric[50s:5s])
-  {} 4
+# Unsupported by streaming engine.
+# eval instant at 10s sum_over_time(metric[50s:5s])
+#   {} 4
 
 # Every evaluation yields the last value, i.e. 2
-eval instant at 5m sum_over_time(metric[50s:10s])
-  {} 12
+# Unsupported by streaming engine.
+# eval instant at 5m sum_over_time(metric[50s:10s])
+#   {} 12
 
 # Series becomes stale at 5m10s (5m after last sample)
 # Hence subquery gets a single sample at 6m-50s=5m10s.
-eval instant at 6m sum_over_time(metric[50s:10s])
-  {} 2
+# Unsupported by streaming engine.
+# eval instant at 6m sum_over_time(metric[50s:10s])
+#   {} 2
 
-eval instant at 10s rate(metric[20s:10s])
-  {} 0.1
+# Unsupported by streaming engine.
+# eval instant at 10s rate(metric[20s:10s])
+#   {} 0.1
 
-eval instant at 20s rate(metric[20s:5s])
-  {} 0.05
+# Unsupported by streaming engine.
+# eval instant at 20s rate(metric[20s:5s])
+#   {} 0.05
 
 clear
 
@@ -36,15 +42,17 @@ load 10s
   http_requests{job="api-server", instance="0", group="canary"}  0+30x1000 300+80x1000
   http_requests{job="api-server", instance="1", group="canary"}  0+40x2000
 
-eval instant at 8000s rate(http_requests{group=~"pro.*"}[1m:10s])
-  {job="api-server", instance="0", group="production"} 1
-  {job="api-server", instance="1", group="production"} 2
+# Unsupported by streaming engine.
+# eval instant at 8000s rate(http_requests{group=~"pro.*"}[1m:10s])
+#   {job="api-server", instance="0", group="production"} 1
+#   {job="api-server", instance="1", group="production"} 2
 
-eval instant at 20000s avg_over_time(rate(http_requests[1m])[1m:1s])
-  {job="api-server", instance="0", group="canary"}     8
-  {job="api-server", instance="1", group="canary"}     4
-  {job="api-server", instance="1", group="production"} 3
-  {job="api-server", instance="0", group="production"} 3
+# Unsupported by streaming engine.
+# eval instant at 20000s avg_over_time(rate(http_requests[1m])[1m:1s])
+#   {job="api-server", instance="0", group="canary"}     8
+#   {job="api-server", instance="1", group="canary"}     4
+#   {job="api-server", instance="1", group="production"} 3
+#   {job="api-server", instance="0", group="production"} 3
 
 clear
 
@@ -53,61 +61,78 @@ load 10s
   metric2 0+2x1000
   metric3 0+3x1000
 
-eval instant at 1000s sum_over_time(metric1[30s:10s])
-  {} 394
+# Unsupported by streaming engine.
+# eval instant at 1000s sum_over_time(metric1[30s:10s])
+#   {} 394
 
 # This is (394*2 - 100), because other than the last 100 at 1000s,
 # everything else is repeated with the 5s step.
-eval instant at 1000s sum_over_time(metric1[30s:5s])
-  {} 688
+# Unsupported by streaming engine.
+# eval instant at 1000s sum_over_time(metric1[30s:5s])
+#   {} 688
 
 # Offset is aligned with the step.
-eval instant at 1010s sum_over_time(metric1[30s:10s] offset 10s)
-  {} 394
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time(metric1[30s:10s] offset 10s)
+#   {} 394
 
 # Same result for different offsets due to step alignment.
-eval instant at 1010s sum_over_time(metric1[30s:10s] offset 9s)
-  {} 297
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time(metric1[30s:10s] offset 9s)
+#   {} 297
 
-eval instant at 1010s sum_over_time(metric1[30s:10s] offset 7s)
-  {} 297
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time(metric1[30s:10s] offset 7s)
+#   {} 297
 
-eval instant at 1010s sum_over_time(metric1[30s:10s] offset 5s)
-  {} 297
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time(metric1[30s:10s] offset 5s)
+#   {} 297
 
-eval instant at 1010s sum_over_time(metric1[30s:10s] offset 3s)
-  {} 297
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time(metric1[30s:10s] offset 3s)
+#   {} 297
 
-eval instant at 1010s sum_over_time((metric1)[30s:10s] offset 3s)
-  {} 297
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time((metric1)[30s:10s] offset 3s)
+#   {} 297
 
-eval instant at 1010s sum_over_time(metric1[30:10] offset 3)
-  {} 297
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time(metric1[30:10] offset 3)
+#   {} 297
 
-eval instant at 1010s sum_over_time((metric1)[30:10s] offset 3s)
-  {} 297
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time((metric1)[30:10s] offset 3s)
+#   {} 297
 
-eval instant at 1010s sum_over_time((metric1)[30:10s] offset 3s)
-  {} 297
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time((metric1)[30:10s] offset 3s)
+#   {} 297
 
-eval instant at 1010s sum_over_time((metric1)[30:10] offset 3s)
-  {} 297
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time((metric1)[30:10] offset 3s)
+#   {} 297
 
-eval instant at 1010s sum_over_time((metric1)[30:10] offset 3)
-  {} 297
+# Unsupported by streaming engine.
+# eval instant at 1010s sum_over_time((metric1)[30:10] offset 3)
+#   {} 297
 
 # Nested subqueries
-eval instant at 1000s rate(sum_over_time(metric1[30s:10s])[50s:10s])
-  {} 0.4
+# Unsupported by streaming engine.
+# eval instant at 1000s rate(sum_over_time(metric1[30s:10s])[50s:10s])
+#   {} 0.4
 
-eval instant at 1000s rate(sum_over_time(metric2[30s:10s])[50s:10s])
-  {} 0.8
+# Unsupported by streaming engine.
+# eval instant at 1000s rate(sum_over_time(metric2[30s:10s])[50s:10s])
+#   {} 0.8
 
-eval instant at 1000s rate(sum_over_time(metric3[30s:10s])[50s:10s])
-  {} 1.2
+# Unsupported by streaming engine.
+# eval instant at 1000s rate(sum_over_time(metric3[30s:10s])[50s:10s])
+#   {} 1.2
 
-eval instant at 1000s rate(sum_over_time((metric1+metric2+metric3)[30s:10s])[30s:10s])
-  {} 2.4
+# Unsupported by streaming engine.
+# eval instant at 1000s rate(sum_over_time((metric1+metric2+metric3)[30s:10s])[30s:10s])
+#   {} 2.4
 
 clear
 
@@ -121,17 +146,20 @@ eval instant at 80s rate(metric[1m])
   {} 2.517857143
 
 # No extrapolation, [2@20, 144@80]: (144 - 2) / 60
-eval instant at 80s rate(metric[1m:10s])
-  {} 2.366666667
+# Unsupported by streaming engine.
+# eval instant at 80s rate(metric[1m:10s])
+#   {} 2.366666667
 
 # Only one value between 10s and 20s, 2@14
 eval instant at 20s min_over_time(metric[10s])
   {} 2
 
 # min(1@10, 2@20)
-eval instant at 20s min_over_time(metric[10s:10s])
-  {} 1
+# Unsupported by streaming engine.
+# eval instant at 20s min_over_time(metric[10s:10s])
+#   {} 1
 
-eval instant at 20m min_over_time(rate(metric[5m])[20m:1m])
-  {} 0.12119047619047618
+# Unsupported by streaming engine.
+# eval instant at 20m min_over_time(rate(metric[5m])[20m:1m])
+#   {} 0.12119047619047618
 

tools/check-for-disabled-but-supported-mqe-test-cases/README.md

@@ -0,0 +1,5 @@
+This tool checks for upstream test cases that have been disabled but are now supported by MQE.
+
+See [these docs](../../pkg/streamingpromql/testdata/upstream/README.md) for more information.
+
+Run this tool with `go run .` in the `check-for-disabled-but-supported-mqe-test-cases` directory.