Support different types of MiddleManagers by Autoscaler apache#8695

- Extended and refactored unit tests
VladimirIordanov · Dec 4, 2019 · ba88a3c · ba88a3c
1 parent 66d3ebe
commit ba88a3c
Show file tree

Hide file tree

Showing 3 changed files with 331 additions and 47 deletions.
diff --git a/...n/java/org/apache/druid/indexing/overlord/autoscaling/CategoriedProvisioningStrategy.java b/...n/java/org/apache/druid/indexing/overlord/autoscaling/CategoriedProvisioningStrategy.java
@@ -324,17 +324,23 @@ public boolean doProvision()
       allCategories.addAll(workersByCategories.keySet());
 
       log.debug(
-          "Pending Tasks of %d categories, Workers of %d categories (%d common categories: %s)",
+          "Pending Tasks of %d categories (%s), Workers of %d categories (%s). %d common categories: %s",
           tasksByCategories.size(),
+          tasksByCategories.keySet(),
           workersByCategories.size(),
+          workersByCategories.keySet(),
           allCategories.size(),
           allCategories
       );
 
       if (allCategories.isEmpty()) {
         // Likely empty categories means initialization. Just try to spinup required amount of workers of each non empty autoscalers
         if (workerConfig.getDefaultAutoScaler() != null) {
-          didProvision = initAutoscaler(workerConfig.getDefaultAutoScaler(), CategoriedWorkerBehaviorConfig.DEFAULT_AUTOSCALER_CATEGORY, workerConfig);
+          didProvision = initAutoscaler(
+              workerConfig.getDefaultAutoScaler(),
+              CategoriedWorkerBehaviorConfig.DEFAULT_AUTOSCALER_CATEGORY,
+              workerConfig
+          );
         }
         for (Map.Entry<String, AutoScaler> autoscalerInfo : workerConfig.getAutoScalers().entrySet()) {
           String category = autoscalerInfo.getKey();
@@ -639,6 +645,9 @@ private boolean initAutoscaler(AutoScaler autoScaler, String category, Categorie
     @Nullable
     private AutoScaler getCategoryAutoscaler(String category, CategoriedWorkerBehaviorConfig workerConfig)
     {
+      if (CategoriedWorkerBehaviorConfig.DEFAULT_AUTOSCALER_CATEGORY.equals(category)) {
+        return workerConfig.getDefaultAutoScaler();
+      }
       AutoScaler autoScaler = workerConfig.getAutoScalers().get(category);
       if (autoScaler == null && workerConfig.isStrong()) {
         log.warn(

diff --git a/...rc/main/java/org/apache/druid/indexing/overlord/setup/CategoriedWorkerBehaviorConfig.java b/...rc/main/java/org/apache/druid/indexing/overlord/setup/CategoriedWorkerBehaviorConfig.java
@@ -22,13 +22,15 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.druid.indexing.overlord.autoscaling.AutoScaler;
+import org.apache.druid.indexing.worker.config.WorkerConfig;
 
 import java.util.Map;
 import java.util.Objects;
 
 public class CategoriedWorkerBehaviorConfig implements WorkerBehaviorConfig
 {
-  public static final String DEFAULT_AUTOSCALER_CATEGORY = "_default_autoscaler_category";
+  // Use the same category constant as for worker category to match default workers and autoscalers
+  public static final String DEFAULT_AUTOSCALER_CATEGORY = WorkerConfig.DEFAULT_CATEGORY;
 
   private final WorkerSelectStrategy selectStrategy;
   private final AutoScaler defaultAutoScaler;