Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

support az aware hashring and multiple sts in one hashring #129

Merged
merged 9 commits into from
May 16, 2024
Merged

support az aware hashring and multiple sts in one hashring #129

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
44a6a09
support az aware hashring
christopherzli Jan 24, 2024
4f27cf4
Update receive-controller.json
christopherzli Jan 24, 2024
f6e7a2b
support multiple statefulsets in 1 hashring
christopherzli Jan 24, 2024
8cb9803
add more logs
christopherzli Jan 24, 2024
3c8115c
style
christopherzli Jan 24, 2024
8e22498
fix lint issue
christopherzli Jan 24, 2024
26f12fd
debug
christopherzli Jan 25, 2024
77b5f5f
return when encountering error
christopherzli May 8, 2024
b206582
remove whitespace
christopherzli May 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
support multiple statefulsets in 1 hashring
  • Loading branch information
@christopherzli
christopherzli committed Jan 24, 2024
commit f6e7a2b9492506e30af8f6e10f7c697d1486858a
95 changes: 51 additions & 44 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -557,7 +557,7 @@
return
}

statefulsets := make(map[string]*appsv1.StatefulSet)
statefulsets := make(map[string][]*appsv1.StatefulSet)

for _, obj := range c.ssetInf.GetStore().List() {
sts, ok := obj.(*appsv1.StatefulSet)
Expand Down Expand Up @@ -587,7 +587,12 @@
}

c.replicas[hashring] = *sts.Spec.Replicas
statefulsets[hashring] = sts.DeepCopy()
if _, ok := statefulsets[hashring]; !ok {
// If not, initialize a new slice
statefulsets[hashring] = []*appsv1.StatefulSet{}
}
// Append the new value to the slice associated with the key
statefulsets[hashring] = append(statefulsets[hashring], sts.DeepCopy())
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

support multiple sts placed in a single hashring in our setup


time.Sleep(c.options.scaleTimeout) // Give some time for all replicas before they receive hundreds req/s
}
Expand Down Expand Up @@ -635,63 +640,65 @@
})
}

func (c *controller) populate(ctx context.Context, hashrings []receive.HashringConfig, statefulsets map[string]*appsv1.StatefulSet) {
func (c *controller) populate(ctx context.Context, hashrings []receive.HashringConfig, statefulsets map[string][]*appsv1.StatefulSet) {
for i, h := range hashrings {
sts, exists := statefulsets[h.Hashring]
stsList, exists := statefulsets[h.Hashring]
if !exists {
continue
}

var endpoints []receive.Endpoint

for i := 0; i < int(*sts.Spec.Replicas); i++ {
podName := fmt.Sprintf("%s-%d", sts.Name, i)
pod, err := c.klient.CoreV1().Pods(c.options.namespace).Get(ctx, podName, metav1.GetOptions{})
if c.options.allowDynamicScaling {
if kerrors.IsNotFound(err) {
continue
for _, sts := range stsList {
for i := 0; i < int(*sts.Spec.Replicas); i++ {
podName := fmt.Sprintf("%s-%d", sts.Name, i)
pod, err := c.klient.CoreV1().Pods(c.options.namespace).Get(ctx, podName, metav1.GetOptions{})
if c.options.allowDynamicScaling {
if kerrors.IsNotFound(err) {
continue
}
// Do not add a replica to the hashring if pod is not Ready.
if !podutils.IsPodReady(pod) {
level.Warn(c.logger).Log("msg", "failed adding pod to hashring, pod not ready", "pod", podName, "err", err)
continue
}

if pod.ObjectMeta.DeletionTimestamp != nil && (pod.Status.Phase == corev1.PodRunning || pod.Status.Phase == corev1.PodPending) {
// Pod is terminating, do not add it to the hashring.
continue
}
}
// Do not add a replica to the hashring if pod is not Ready.
if !podutils.IsPodReady(pod) {
level.Warn(c.logger).Log("msg", "failed adding pod to hashring, pod not ready", "pod", podName, "err", err)
continue
// If cluster domain is empty string we don't want dot after svc.
clusterDomain := ""
if c.options.clusterDomain != "" {
clusterDomain = fmt.Sprintf(".%s", c.options.clusterDomain)
}

if pod.ObjectMeta.DeletionTimestamp != nil && (pod.Status.Phase == corev1.PodRunning || pod.Status.Phase == corev1.PodPending) {
// Pod is terminating, do not add it to the hashring.
continue
endpoint := receive.Endpoint{
Address: fmt.Sprintf("%s-%d.%s.%s.svc%s:%d",
sts.Name,
i,
sts.Spec.ServiceName,
c.options.namespace,
clusterDomain,
c.options.port,
),
}
}
// If cluster domain is empty string we don't want dot after svc.
clusterDomain := ""
if c.options.clusterDomain != "" {
clusterDomain = fmt.Sprintf(".%s", c.options.clusterDomain)
}
endpoint := receive.Endpoint{
Address: fmt.Sprintf("%s-%d.%s.%s.svc%s:%d",
sts.Name,
i,
sts.Spec.ServiceName,
c.options.namespace,
clusterDomain,
c.options.port,
),
}

if c.options.useAzAwareHashRing {
//If pod annotation value is not found or key not specified,
//endpoint will the Statefulset name as AZ name
endpoint.AZ = sts.Name
if c.options.podAzAnnotationKey != "" && err == nil {
annotationValue, ok := pod.Annotations[c.options.podAzAnnotationKey]
if ok {
endpoint.AZ = annotationValue
if c.options.useAzAwareHashRing {
//If pod annotation value is not found or key not specified,
//endpoint will the Statefulset name as AZ name
endpoint.AZ = sts.Name
if c.options.podAzAnnotationKey != "" && err == nil {
annotationValue, ok := pod.Annotations[c.options.podAzAnnotationKey]
if ok {
endpoint.AZ = annotationValue
}
}
}
}

endpoints = append(endpoints, endpoint)
endpoints = append(endpoints, endpoint)

}
}

hashrings[i].Endpoints = endpoints
Expand Down Expand Up @@ -818,8 +825,8 @@
}

func (q *queue) add() {
q.Lock()

Check failure on line 828 in main.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

q.Lock undefined (type *queue has no field or method Lock) (typecheck)
defer q.Unlock()

Check failure on line 829 in main.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

q.Unlock undefined (type *queue has no field or method Unlock) (typecheck)

if !q.ok {
return
Expand All @@ -831,8 +838,8 @@
}

func (q *queue) stop() {
q.Lock()

Check failure on line 841 in main.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

q.Lock undefined (type *queue has no field or method Lock) (typecheck)
defer q.Unlock()

Check failure on line 842 in main.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

q.Unlock undefined (type *queue has no field or method Unlock) (typecheck)

if !q.ok {
return
Expand All @@ -844,8 +851,8 @@

func (q *queue) get() bool {
<-q.ch
q.Lock()

Check failure on line 854 in main.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

q.Lock undefined (type *queue has no field or method Lock) (typecheck)
defer q.Unlock()

Check failure on line 855 in main.go

View workflow job for this annotation

GitHub Actions / Lint (ubuntu-latest) 

q.Unlock undefined (type *queue has no field or method Unlock) (typecheck)

return q.ok
}
111 changes: 111 additions & 0 deletions main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,54 @@ func TestController(t *testing.T) {
},
}},
},
{
name: "OneHashringLabelKeyManyStatefulSets",
hashrings: []receive.HashringConfig{{
Hashring: "hashring0",
Tenants: []string{"foo", "bar"},
}},
statefulsets: []*appsv1.StatefulSet{
{
ObjectMeta: metav1.ObjectMeta{
Name: "hashring0",
Labels: map[string]string{
"a": "b",
hashringLabelKey: "hashring0",
},
},
Spec: appsv1.StatefulSetSpec{
Replicas: intPointer(3),
ServiceName: "h0",
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "hashring1",
Labels: map[string]string{
"a": "b",
hashringLabelKey: "hashring0",
},
},
Spec: appsv1.StatefulSetSpec{
Replicas: intPointer(2),
ServiceName: "h0",
},
},
},
clusterDomain: "cluster.local",
expected: []receive.HashringConfig{{
Hashring: "hashring0",
Tenants: []string{"foo", "bar"},
Endpoints: []receive.Endpoint{
{Address: "hashring0-0.h0.namespace.svc.cluster.local:10901"},
{Address: "hashring0-1.h0.namespace.svc.cluster.local:10901"},
{Address: "hashring0-2.h0.namespace.svc.cluster.local:10901"},
{Address: "hashring1-0.h0.namespace.svc.cluster.local:10901"},
{Address: "hashring1-1.h0.namespace.svc.cluster.local:10901"},
},
},
},
},
{
clusterDomain: "",
name: "OneHashringOneStatefulSetNoClusterDomain",
Expand Down Expand Up @@ -603,6 +651,69 @@ func TestControllerWithAzAware(t *testing.T) {
},
}},
},
{
name: "OneHashringLabelKeyManyStatefulSets",
hashrings: []receive.HashringConfig{{
Hashring: "hashring0",
Tenants: []string{"foo", "bar"},
}},
statefulsets: []*appsv1.StatefulSet{
{
ObjectMeta: metav1.ObjectMeta{
Name: "hashring0",
Labels: map[string]string{
"a": "b",
hashringLabelKey: "hashring0",
},
},
Spec: appsv1.StatefulSetSpec{
Replicas: intPointer(3),
ServiceName: "h0",
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "hashring1",
Labels: map[string]string{
"a": "b",
hashringLabelKey: "hashring0",
},
},
Spec: appsv1.StatefulSetSpec{
Replicas: intPointer(2),
ServiceName: "h0",
},
},
},
clusterDomain: "cluster.local",
expected: []receive.HashringConfig{{
Hashring: "hashring0",
Tenants: []string{"foo", "bar"},
Endpoints: []receive.Endpoint{
{
Address: "hashring0-0.h0.namespace.svc.cluster.local:10901",
AZ: "hashring0",
},
{
Address: "hashring0-1.h0.namespace.svc.cluster.local:10901",
AZ: "hashring0",
},
{
Address: "hashring0-2.h0.namespace.svc.cluster.local:10901",
AZ: "hashring0",
},
{
Address: "hashring1-0.h0.namespace.svc.cluster.local:10901",
AZ: "hashring1",
},
{
Address: "hashring1-1.h0.namespace.svc.cluster.local:10901",
AZ: "hashring1",
},
},
},
},
},
{
clusterDomain: "",
name: "OneHashringOneStatefulSetNoClusterDomain",
Expand Down
Loading