optimize : enhance fail/timeout status metric and log level (apache#4517

)
pengten · Apr 6, 2022 · 660166a · 660166a
1 parent abe196f
commit 660166a
Show file tree

Hide file tree

Showing 6 changed files with 58 additions and 24 deletions.
diff --git a/changes/1.5.0.md b/changes/1.5.0.md
@@ -209,6 +209,7 @@ Seata 是一款开源的分布式事务解决方案，提供高性能和简单
   - [[#4469](https://github.com/seata/seata/pull/4469)] 优化控制台db模式下获取配置的方式
   - [[#4478](https://github.com/seata/seata/pull/4478)] 优化 Nacos 配置和注册元数据属性
   - [[#4522](https://github.com/seata/seata/pull/4522)] 优化 GC 参数
+  - [[#4517](https://github.com/seata/seata/pull/4517)] 增强失败/超时状态的监控
 
 
 ### test：

diff --git a/changes/en-us/1.5.0.md b/changes/en-us/1.5.0.md
@@ -214,7 +214,8 @@
   - [[#4469](https://github.com/seata/seata/pull/4469)] optimize the way to get configuration in DB mode of console
   - [[#4478](https://github.com/seata/seata/pull/4478)] optimize Nacos config and naming properties
   - [[#4522](https://github.com/seata/seata/pull/4522)] optimize GC parameters in JVM
-
+  - [[#4517](https://github.com/seata/seata/pull/4517)] enhance fail/timeout status metric and log level
+
 
   ### test:	
 

diff --git a/metrics/seata-metrics-api/src/main/java/io/seata/metrics/IdConstants.java b/metrics/seata-metrics-api/src/main/java/io/seata/metrics/IdConstants.java
@@ -66,4 +66,9 @@ public interface IdConstants {
     String STATUS_VALUE_COMMITTED = "committed";
 
     String STATUS_VALUE_ROLLBACKED = "rollbacked";
+
+    String STATUS_VALUE_FAILED = "failed";
+
+    String STATUS_VALUE_TWO_PHASE_TIMEOUT = "2phaseTimeout";
+
 }
diff --git a/server/src/main/java/io/seata/server/coordinator/DefaultCoordinator.java b/server/src/main/java/io/seata/server/coordinator/DefaultCoordinator.java
@@ -436,7 +436,7 @@ protected void handleRetryRollbacking(Collection<GlobalSession> rollbackingSessi
                     }
                     // Prevent thread safety issues
                     SessionHolder.getRetryRollbackingSessionManager().removeGlobalSession(rollbackingSession);
-                    LOGGER.info("Global transaction rollback retry timeout and has removed [{}]", rollbackingSession.getXid());
+                    LOGGER.error("Global transaction rollback retry timeout and has removed [{}]", rollbackingSession.getXid());
 
                     SessionHelper.endRollbackFailed(rollbackingSession);
 

diff --git a/server/src/main/java/io/seata/server/metrics/MeterIdConstants.java b/server/src/main/java/io/seata/server/metrics/MeterIdConstants.java
@@ -39,6 +39,7 @@ public interface MeterIdConstants {
         .withTag(IdConstants.METER_KEY, IdConstants.METER_VALUE_COUNTER)
         .withTag(IdConstants.STATUS_KEY, IdConstants.STATUS_VALUE_ROLLBACKED);
 
+
     Id SUMMARY_COMMITTED = new Id(IdConstants.SEATA_TRANSACTION)
         .withTag(IdConstants.ROLE_KEY, IdConstants.ROLE_VALUE_TC)
         .withTag(IdConstants.METER_KEY, IdConstants.METER_VALUE_SUMMARY)
@@ -49,6 +50,16 @@ public interface MeterIdConstants {
         .withTag(IdConstants.METER_KEY, IdConstants.METER_VALUE_SUMMARY)
         .withTag(IdConstants.STATUS_KEY, IdConstants.STATUS_VALUE_ROLLBACKED);
 
+    Id SUMMARY_FAILED = new Id(IdConstants.SEATA_TRANSACTION)
+        .withTag(IdConstants.ROLE_KEY, IdConstants.ROLE_VALUE_TC)
+        .withTag(IdConstants.METER_KEY, IdConstants.METER_VALUE_SUMMARY)
+        .withTag(IdConstants.STATUS_KEY, IdConstants.STATUS_VALUE_FAILED);
+
+    Id SUMMARY_TWO_PHASE_TIMEOUT = new Id(IdConstants.SEATA_TRANSACTION)
+         .withTag(IdConstants.ROLE_KEY, IdConstants.ROLE_VALUE_TC)
+         .withTag(IdConstants.METER_KEY, IdConstants.METER_VALUE_SUMMARY)
+         .withTag(IdConstants.STATUS_KEY, IdConstants.STATUS_VALUE_TWO_PHASE_TIMEOUT);
+
     Id TIMER_COMMITTED = new Id(IdConstants.SEATA_TRANSACTION)
         .withTag(IdConstants.ROLE_KEY, IdConstants.ROLE_VALUE_TC)
         .withTag(IdConstants.METER_KEY, IdConstants.METER_VALUE_TIMER)
@@ -58,4 +69,10 @@ public interface MeterIdConstants {
         .withTag(IdConstants.ROLE_KEY, IdConstants.ROLE_VALUE_TC)
         .withTag(IdConstants.METER_KEY, IdConstants.METER_VALUE_TIMER)
         .withTag(IdConstants.STATUS_KEY, IdConstants.STATUS_VALUE_ROLLBACKED);
+
+    Id TIMER_FAILED = new Id(IdConstants.SEATA_TRANSACTION)
+        .withTag(IdConstants.ROLE_KEY, IdConstants.ROLE_VALUE_TC)
+        .withTag(IdConstants.METER_KEY, IdConstants.METER_VALUE_TIMER)
+        .withTag(IdConstants.STATUS_KEY, IdConstants.STATUS_VALUE_FAILED);
+
 }
diff --git a/server/src/main/java/io/seata/server/metrics/MetricsSubscriber.java b/server/src/main/java/io/seata/server/metrics/MetricsSubscriber.java
@@ -71,9 +71,7 @@ private void processGlobalStatusBegin(GlobalTransactionEvent event) {
     }
 
     private void processGlobalStatusCommitted(GlobalTransactionEvent event) {
-        registry.getCounter(MeterIdConstants.COUNTER_ACTIVE
-                .withTag(APP_ID_KEY, event.getApplicationId())
-                .withTag(GROUP_KEY, event.getGroup())).decrease(1);
+        decreaseActive(event);
         registry.getCounter(MeterIdConstants.COUNTER_COMMITTED
                 .withTag(APP_ID_KEY, event.getApplicationId())
                 .withTag(GROUP_KEY, event.getGroup())).increase(1);
@@ -87,9 +85,7 @@ private void processGlobalStatusCommitted(GlobalTransactionEvent event) {
     }
 
     private void processGlobalStatusRollbacked(GlobalTransactionEvent event) {
-        registry.getCounter(MeterIdConstants.COUNTER_ACTIVE
-                .withTag(APP_ID_KEY, event.getApplicationId())
-                .withTag(GROUP_KEY, event.getGroup())).decrease(1);
+        decreaseActive(event);
         registry.getCounter(MeterIdConstants.COUNTER_ROLLBACKED
                 .withTag(APP_ID_KEY, event.getApplicationId())
                 .withTag(GROUP_KEY, event.getGroup())).increase(1);
@@ -103,42 +99,56 @@ private void processGlobalStatusRollbacked(GlobalTransactionEvent event) {
     }
 
     private void processGlobalStatusCommitFailed(GlobalTransactionEvent event) {
-        registry.getCounter(MeterIdConstants.COUNTER_ACTIVE
-                .withTag(APP_ID_KEY, event.getApplicationId())
-                .withTag(GROUP_KEY, event.getGroup())).decrease(1);
+        decreaseActive(event);
+        reportFailed(event);
     }
 
     private void processGlobalStatusRollbackFailed(GlobalTransactionEvent event) {
-        registry.getCounter(MeterIdConstants.COUNTER_ACTIVE
-                .withTag(APP_ID_KEY, event.getApplicationId())
-                .withTag(GROUP_KEY, event.getGroup())).decrease(1);
+        decreaseActive(event);
+        reportFailed(event);
     }
 
     private void processGlobalStatusTimeoutRollbacked(GlobalTransactionEvent event) {
-        registry.getCounter(MeterIdConstants.COUNTER_ACTIVE
-                .withTag(APP_ID_KEY, event.getApplicationId())
-                .withTag(GROUP_KEY, event.getGroup())).decrease(1);
+        decreaseActive(event);
     }
 
     private void processGlobalStatusTimeoutRollbackFailed(GlobalTransactionEvent event) {
-        registry.getCounter(MeterIdConstants.COUNTER_ACTIVE
-                .withTag(APP_ID_KEY, event.getApplicationId())
-                .withTag(GROUP_KEY, event.getGroup())).decrease(1);
+        decreaseActive(event);
+        reportTwoPhaseTimeout(event);
     }
 
-
     private void processGlobalStatusCommitRetryTimeout(GlobalTransactionEvent event) {
-        registry.getCounter(MeterIdConstants.COUNTER_ACTIVE
-                .withTag(APP_ID_KEY, event.getApplicationId())
-                .withTag(GROUP_KEY, event.getGroup())).decrease(1);
+        decreaseActive(event);
+        reportTwoPhaseTimeout(event);
     }
 
     private void processGlobalStatusTimeoutRollbackRetryTimeout(GlobalTransactionEvent event) {
+        decreaseActive(event);
+    }
+
+    private void decreaseActive(GlobalTransactionEvent event) {
         registry.getCounter(MeterIdConstants.COUNTER_ACTIVE
                 .withTag(APP_ID_KEY, event.getApplicationId())
                 .withTag(GROUP_KEY, event.getGroup())).decrease(1);
     }
 
+    private void reportFailed(GlobalTransactionEvent event) {
+        registry.getSummary(MeterIdConstants.SUMMARY_FAILED
+                .withTag(APP_ID_KEY, event.getApplicationId())
+                .withTag(GROUP_KEY, event.getGroup())).increase(1);
+        registry.getTimer(MeterIdConstants.TIMER_FAILED
+                .withTag(APP_ID_KEY, event.getApplicationId())
+                .withTag(GROUP_KEY, event.getGroup()))
+                .record(event.getEndTime() - event.getBeginTime(), TimeUnit.MILLISECONDS);
+    }
+
+    private void reportTwoPhaseTimeout(GlobalTransactionEvent event) {
+        registry.getSummary(MeterIdConstants.SUMMARY_TWO_PHASE_TIMEOUT
+                .withTag(APP_ID_KEY, event.getApplicationId())
+                .withTag(GROUP_KEY, event.getGroup())).increase(1);
+    }
+
+
 
     @Subscribe
     public void recordGlobalTransactionEventForMetrics(GlobalTransactionEvent event) {