Move early stopping code into new early stopping directory

Summary: pretty self explanatory, see some inline comments

Reviewed By: Balandat

Differential Revision: D29146848

fbshipit-source-id: 85ead533b7b8c8f3f99d95a12611770f34bbc980

ax/benchmark/benchmark.py

@@ -36,12 +36,12 @@
 from ax.core.generator_run import GeneratorRun
 from ax.core.observation import ObservationFeatures
 from ax.core.parameter import RangeParameter
+from ax.early_stopping.strategies import BaseEarlyStoppingStrategy
 from ax.modelbridge.base import gen_arms
 from ax.modelbridge.generation_strategy import GenerationStrategy
 from ax.runners.simulated_backend import SimulatedBackendRunner
 from ax.runners.synthetic import SyntheticRunner
 from ax.service.ax_client import AxClient
-from ax.service.early_stopping_strategy import BaseEarlyStoppingStrategy
 from ax.service.scheduler import SchedulerOptions
 from ax.utils.common.logger import get_logger
 from ax.utils.common.typeutils import not_none

ax/service/early_stopping_strategy.py → ax/early_stopping/strategies.py

@@ -123,7 +123,7 @@ def should_stop_trials_early(
         last_progression = df[map_key].max()
         if last_progression < self.min_progression:
             logger.info(
-                "Most recent progression ({last_progression}) is less than the "
+                f"Most recent progression ({last_progression}) is less than the "
                 "specified minimum progression for early stopping "
                 f"({self.min_progression}). "
                 "Not early stopping any trials."
@@ -136,7 +136,7 @@ def should_stop_trials_early(
 
         if len(data_at_last_progression) < self.min_curves:
             logger.info(
-                "The number of trials with data ({len(data_at_last_progression)}) "
+                f"The number of trials with data ({len(data_at_last_progression)}) "
                 "is less than the specified minimum number for early stopping "
                 f"({self.min_curves}). "
                 "Not early stopping any trials."

ax/service/tests/test_early_stopping_strategy.py → ax/early_stopping/tests/test_strategies.py

@@ -4,8 +4,10 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from ax.service.early_stopping_strategy import BaseEarlyStoppingStrategy
-from ax.service.early_stopping_strategy import PercentileEarlyStoppingStrategy
+from ax.early_stopping.strategies import (
+    BaseEarlyStoppingStrategy,
+    PercentileEarlyStoppingStrategy,
+)
 from ax.utils.common.testutils import TestCase
 from ax.utils.testing.core_stubs import (
     get_branin_arms,

ax/service/scheduler.py

@@ -32,6 +32,7 @@
 from ax.core.experiment import Experiment
 from ax.core.metric import Metric
 from ax.core.trial import Trial
+from ax.early_stopping.strategies import BaseEarlyStoppingStrategy
 from ax.exceptions.core import (
     AxError,
     DataRequiredError,
@@ -43,7 +44,6 @@
 from ax.modelbridge.modelbridge_utils import (
     get_pending_observation_features_based_on_trial_status,
 )
-from ax.service.early_stopping_strategy import BaseEarlyStoppingStrategy
 from ax.service.utils.with_db_settings_base import DBSettings, WithDBSettingsBase
 from ax.utils.common.constants import Keys
 from ax.utils.common.executils import retry_on_exception

ax/service/tests/test_scheduler.py

@@ -18,6 +18,7 @@
 from ax.core.metric import Metric
 from ax.core.objective import Objective
 from ax.core.optimization_config import OptimizationConfig
+from ax.early_stopping.strategies import BaseEarlyStoppingStrategy
 from ax.exceptions.core import OptimizationComplete, UnsupportedError
 from ax.metrics.branin import BraninMetric
 from ax.modelbridge.dispatch_utils import choose_generation_strategy
@@ -26,7 +27,6 @@
     get_pending_observation_features_based_on_trial_status,
 )
 from ax.modelbridge.registry import Models
-from ax.service.early_stopping_strategy import BaseEarlyStoppingStrategy
 from ax.service.scheduler import (
     FailureRateExceededError,
     Scheduler,

sphinx/source/early_stopping.rst

@@ -0,0 +1,24 @@
+.. role:: hidden
+    :class: hidden-section
+
+ax.early_stopping
+===================================
+
+.. automodule:: ax.early_stopping
+.. currentmodule:: ax.early_stopping
+
+Strategies
+~~~~~~~~~~~~~~~~~~
+
+.. automodule:: ax.early_stopping.strategies
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Utils
+~~~~~~~~~~~~~~~~~~
+
+.. automodule:: ax.early_stopping.utils
+    :members:
+    :undoc-members:
+    :show-inheritance:

sphinx/source/service.rst

@@ -16,14 +16,6 @@ Ax Client
     :undoc-members:
     :show-inheritance:
 
-Early Stopping Strategy
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. automodule:: ax.service.early_stopping_strategy
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
 Managed Loop
 ~~~~~~~~~~~~~~~~