Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use population bias correction in StandardizeYTransform std computation #589

Closed
wants to merge 1 commit into from
Closed

Use population bias correction in StandardizeYTransform std computation #589

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions ax/modelbridge/tests/test_standardize_y_transform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
# LICENSE file in the root directory of this source tree.

from copy import deepcopy
from math import sqrt

import numpy as np
from ax.core.metric import Metric
Expand Down Expand Up @@ -44,7 +45,7 @@ def setUp(self):

def testInit(self):
self.assertEqual(self.t.Ymean, {"m1": 1.0, "m2": 1.5})
self.assertEqual(self.t.Ystd, {"m1": 1.0, "m2": 0.5})
self.assertEqual(self.t.Ystd, {"m1": 1.0, "m2": sqrt(1 / 3)})
with self.assertRaises(ValueError):
StandardizeY(
search_space=None, observation_features=None, observation_data=[]
Expand All @@ -53,14 +54,20 @@ def testInit(self):
def testTransformObservations(self):
obsd1_t = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([0.0, 1.0, -1.0]),
covariance=np.array([[1.0, 0.4, 0.8], [0.4, 8.0, 3.2], [0.8, 3.2, 12.0]]),
means=np.array([0.0, sqrt(3 / 4), -sqrt(3 / 4)]),
covariance=np.array(
[
[1.0, 0.2 * sqrt(3), 0.4 * sqrt(3)],
[0.2 * sqrt(3), 6.0, 2.4],
[0.4 * sqrt(3), 2.4, 9.0],
],
),
)
obsd2 = [deepcopy(self.obsd1)]
obsd2 = self.t.transform_observation_data(obsd2, [])
self.assertTrue(obsd2[0] == obsd1_t)
self.assertTrue(osd_allclose(obsd2[0], obsd1_t))
obsd2 = self.t.untransform_observation_data(obsd2, [])
self.assertTrue(obsd2[0] == self.obsd1)
self.assertTrue(osd_allclose(obsd2[0], self.obsd1))

def testTransformOptimizationConfig(self):
m1 = Metric(name="m1")
Expand Down Expand Up @@ -89,13 +96,16 @@ def testTransformOptimizationConfig(self):
metric=m1, op=ComparisonOp.GEQ, bound=1.0, relative=False
),
OutcomeConstraint(
metric=m2, op=ComparisonOp.LEQ, bound=4.0, relative=False
metric=m2,
op=ComparisonOp.LEQ,
bound=2.0 * sqrt(3), # (3.5 - 1.5) / sqrt(1/3)
relative=False,
),
ScalarizedOutcomeConstraint(
metrics=[m1, m2],
weights=[0.5, 0.25], # [0.5*1.0, 0.5*0.5]
weights=[0.5 * 1.0, 0.5 * sqrt(1 / 3)],
op=ComparisonOp.LEQ,
bound=2.25, # 3.5 - (0.5* 1.0 + 0.5*1.5)
bound=2.25, # 3.5 - (0.5 * 1.0 + 0.5 * 1.5)
relative=False,
),
]
Expand All @@ -109,3 +119,13 @@ def testTransformOptimizationConfig(self):
oc = OptimizationConfig(objective=objective, outcome_constraints=[con])
with self.assertRaises(ValueError):
oc = self.t.transform_optimization_config(oc, None, None)


def osd_allclose(osd1: ObservationData, osd2: ObservationData) -> bool:
if osd1.metric_names != osd2.metric_names:
return False
if not np.allclose(osd1.means, osd2.means):
return False
if not np.allclose(osd1.covariance, osd2.covariance):
return False
return True
66 changes: 45 additions & 21 deletions ax/modelbridge/tests/test_stratified_standardize_y.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
# LICENSE file in the root directory of this source tree.

from copy import deepcopy
from math import sqrt

import numpy as np
from ax.core.metric import Metric
Expand All @@ -18,6 +19,8 @@
from ax.modelbridge.transforms.stratified_standardize_y import StratifiedStandardizeY
from ax.utils.common.testutils import TestCase

from .test_standardize_y_transform import osd_allclose


class StratifiedStandardizeYTransformTest(TestCase):
def setUp(self):
Expand Down Expand Up @@ -58,14 +61,25 @@ def setUp(self):
)

def testInit(self):
Ymean_expected = {
("m1", "a"): 1.0,
("m1", "b"): 3.0,
("m2", "a"): 5.0,
("m2", "b"): 1.5,
}
Ystd_expected = {
("m1", "a"): 1.0,
("m1", "b"): sqrt(2) * 2.0,
("m2", "a"): sqrt(2) * 3.0,
("m2", "b"): sqrt(2) * 0.5,
}
self.assertEqual(
self.t.Ymean,
{("m1", "a"): 1.0, ("m1", "b"): 3.0, ("m2", "a"): 5.0, ("m2", "b"): 1.5},
)
self.assertEqual(
self.t.Ystd,
{("m1", "a"): 1.0, ("m1", "b"): 2.0, ("m2", "a"): 3.0, ("m2", "b"): 0.5},
Ymean_expected,
)
self.assertEqual(set(self.t.Ystd), set(Ystd_expected))
for k, v in self.t.Ystd.items():
self.assertAlmostEqual(v, Ystd_expected[k])
with self.assertRaises(ValueError):
# No parameter specified
StratifiedStandardizeY(
Expand Down Expand Up @@ -129,48 +143,55 @@ def testInit(self):
)
self.assertEqual(
t2.Ymean,
{("m1", "a"): 1.0, ("m1", "b"): 3.0, ("m2", "a"): 5.0, ("m2", "b"): 1.5},
)
self.assertEqual(
t2.Ystd,
{("m1", "a"): 1.0, ("m1", "b"): 2.0, ("m2", "a"): 3.0, ("m2", "b"): 0.5},
Ymean_expected,
)
self.assertEqual(set(t2.Ystd), set(Ystd_expected))
for k, v in t2.Ystd.items():
self.assertAlmostEqual(v, Ystd_expected[k])

def testTransformObservations(self):
std_m2_a = sqrt(2) * 3
obsd1_ta = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([0.0, -1.0, 1.0]),
means=np.array([0.0, -3.0 / std_m2_a, 3.0 / std_m2_a]),
covariance=np.array(
[
[1.0, 0.2 / 3, 0.4 / 3],
[0.2 / 3, 2.0 / 9, 0.8 / 9],
[0.4 / 3, 0.8 / 9, 3.0 / 9],
[1.0, 0.2 / std_m2_a, 0.4 / std_m2_a],
[0.2 / std_m2_a, 2.0 / 18, 0.8 / 18],
[0.4 / std_m2_a, 0.8 / 18, 3.0 / 18],
]
),
)
std_m1_b, std_m2_b = 2 * sqrt(2), sqrt(1 / 2)
obsd1_tb = ObservationData(
metric_names=["m1", "m2", "m2"],
means=np.array([-1.0, 1.0, 13.0]),
covariance=np.array([[0.25, 0.2, 0.4], [0.2, 8.0, 3.2], [0.4, 3.2, 12.0]]),
means=np.array([-2.0 / std_m1_b, 0.5 / std_m2_b, 6.5 / std_m2_b]),
covariance=np.array(
[
[1.0 / 8, 0.2 / 2, 0.4 / 2],
[0.2 / 2, 2.0 * 2, 0.8 * 2],
[0.4 / 2, 0.8 * 2, 3.0 * 2],
]
),
)
obsd2 = [deepcopy(self.obsd1)]
obsd2 = self.t.transform_observation_data(
obsd2, [ObservationFeatures({"z": "a"})]
)
self.assertEqual(obsd2[0], obsd1_ta)
self.assertTrue(osd_allclose(obsd2[0], obsd1_ta))
obsd2 = self.t.untransform_observation_data(
obsd2, [ObservationFeatures({"z": "a"})]
)
self.assertEqual(obsd2[0], self.obsd1)
self.assertTrue(osd_allclose(obsd2[0], self.obsd1))
obsd2 = [deepcopy(self.obsd1)]
obsd2 = self.t.transform_observation_data(
obsd2, [ObservationFeatures({"z": "b"})]
)
self.assertEqual(obsd2[0], obsd1_tb)
self.assertTrue(osd_allclose(obsd2[0], obsd1_tb))
obsd2 = self.t.untransform_observation_data(
obsd2, [ObservationFeatures({"z": "b"})]
)
self.assertEqual(obsd2[0], self.obsd1)
self.assertTrue(osd_allclose(obsd2[0], self.obsd1))

def testTransformOptimizationConfig(self):
m1 = Metric(name="m1")
Expand All @@ -193,7 +214,10 @@ def testTransformOptimizationConfig(self):
metric=m1, op=ComparisonOp.GEQ, bound=1.0, relative=False
),
OutcomeConstraint(
metric=m2, op=ComparisonOp.LEQ, bound=-0.5, relative=False
metric=m2,
op=ComparisonOp.LEQ,
bound=(3.5 - 5.0) / (sqrt(2) * 3),
relative=False,
),
]
self.assertTrue(oc.outcome_constraints == cons_t)
Expand Down
5 changes: 4 additions & 1 deletion ax/modelbridge/transforms/standardize_y.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,10 @@ def compute_standardization_parameters(
]:
"""Compute mean and std. dev of Ys."""
Ymean = {k: np.mean(y) for k, y in Ys.items()}
Ystd = {k: np.std(y) for k, y in Ys.items()}
# We use the Bessel correction term (divide by N-1) here in order to
# be consistent with the default behavior of torch.std that is used to
# validate input data standardization in BoTorch.
Ystd = {k: np.std(y, ddof=1) if len(y) > 1 else 0.0 for k, y in Ys.items()}
for k, s in Ystd.items():
# Don't standardize if variance is too small.
if s < 1e-8:
Expand Down