Even more release-ready!

.circleci/config.yml

@@ -20,6 +20,7 @@ jobs:
      - run: make update-bindata
      - run:
         name: Build and publish
+        # TODO(p3): build an image that has the script and all dependencies installed
         command: |
           if [[ -z "${CIRCLE_PULL_REQUEST}" ]] && [[ "${CIRCLE_TAG}" ]] && [[ "${CIRCLE_PROJECT_USERNAME}" = "weaveworks" ]] ; then
             curl -sL https://git.io/goreleaser | bash

cmd/eksctl/create.go

@@ -137,7 +137,10 @@ func doCreateCluster(cfg *eks.Config) error {
 			return err
 		}
 
-		// TODO(p1): watch nodes joining
+		// wait for nodes to join
+		if err := cfg.WaitForNodes(clientSet); err != nil {
+			return err
+		}
 
 		// TODO(p2): addons
 

cmd/eksctl/main.go

@@ -19,7 +19,7 @@ func init() {
 
 	addCommands()
 
-	rootCmd.PersistentFlags().IntVarP(&logger.Level, "verbose", "v", 4, "set log level")
+	rootCmd.PersistentFlags().IntVarP(&logger.Level, "verbose", "v", 3, "set log level")
 	rootCmd.PersistentFlags().BoolVarP(&logger.Color, "color", "C", true, "toggle colorized logs")
 }
 

pkg/eks/auth.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/ghodss/yaml"
 	"github.com/pkg/errors"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -15,8 +14,6 @@ import (
 	"github.com/heptio/authenticator/pkg/token"
 	"github.com/kubicorn/kubicorn/pkg/logger"
 
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 
 	"k8s.io/client-go/tools/clientcmd"
@@ -187,44 +184,3 @@ func (c *ClientConfig) NewClientSetWithEmbeddedToken() (*clientset.Clientset, er
 	}
 	return clientSet, nil
 }
-
-func (c *Config) newNodeAuthConfigMap() (*corev1.ConfigMap, error) {
-	mapRoles := make([]map[string]interface{}, 1)
-	mapRoles[0] = make(map[string]interface{})
-
-	mapRoles[0]["rolearn"] = c.nodeInstanceRoleARN
-	mapRoles[0]["username"] = "system:node:{{EC2PrivateDNSName}}"
-	mapRoles[0]["groups"] = []string{
-		"system:bootstrappers",
-		"system:nodes",
-		"system:nodes",
-	}
-
-	mapRolesBytes, err := yaml.Marshal(mapRoles)
-	if err != nil {
-		return nil, err
-	}
-
-	cm := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "aws-auth",
-			Namespace: "default",
-		},
-		BinaryData: map[string][]byte{
-			"mapRoles": mapRolesBytes,
-		},
-	}
-
-	return cm, nil
-}
-
-func (c *Config) CreateDefaultNodeGroupAuthConfigMap(clientSet *clientset.Clientset) error {
-	cm, err := c.newNodeAuthConfigMap()
-	if err != nil {
-		return errors.Wrap(err, "contructing auth ConfigMap for DefaultNodeGroup")
-	}
-	if _, err := clientSet.CoreV1().ConfigMaps("default").Create(cm); err != nil {
-		return errors.Wrap(err, "creating auth ConfigMap for DefaultNodeGroup")
-	}
-	return nil
-}

pkg/eks/cfn.go

@@ -112,7 +112,7 @@ func (c *CloudFormation) CreateStack(name string, templateBody []byte, parameter
 		input.Parameters = append(input.Parameters, p)
 	}
 
-	// TODO(p0): looks like we can block on this forever, if parameters are invalid
+	// TODO(p1): looks like we can block on this forever, if parameters are invalid
 	logger.Debug("input = %#v", input)
 	s, err := c.svc.CreateStack(input)
 	if err != nil {
@@ -238,7 +238,6 @@ func (c *CloudFormation) CreateStacks(tasks map[string]func(chan error) error, t
 	}
 	logger.Debug("waiting for %d tasks to complete", len(tasks))
 	wg.Wait()
-	close(taskErrs)
 }
 
 func (c *CloudFormation) CreateAllStacks(taskErrs chan error) {
@@ -255,6 +254,7 @@ func (c *CloudFormation) CreateAllStacks(taskErrs chan error) {
 	c.CreateStacks(map[string]func(chan error) error{
 		"createStackDefaultNodeGroup": func(errs chan error) error { return c.createStackDefaultNodeGroup(errs) },
 	}, taskErrs)
+	close(taskErrs)
 }
 
 func (c *CloudFormation) stackNameVPC() string {

pkg/eks/nodegroup.go

@@ -0,0 +1,126 @@
+package eks
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/ghodss/yaml"
+	"github.com/kubicorn/kubicorn/pkg/logger"
+	"github.com/pkg/errors"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	clientset "k8s.io/client-go/kubernetes"
+)
+
+func (c *Config) newNodeAuthConfigMap() (*corev1.ConfigMap, error) {
+	mapRoles := make([]map[string]interface{}, 1)
+	mapRoles[0] = make(map[string]interface{})
+
+	mapRoles[0]["rolearn"] = c.nodeInstanceRoleARN
+	mapRoles[0]["username"] = "system:node:{{EC2PrivateDNSName}}"
+	mapRoles[0]["groups"] = []string{
+		"system:bootstrappers",
+		"system:nodes",
+		"system:nodes",
+	}
+
+	mapRolesBytes, err := yaml.Marshal(mapRoles)
+	if err != nil {
+		return nil, err
+	}
+
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "aws-auth",
+			Namespace: "default",
+		},
+		BinaryData: map[string][]byte{
+			"mapRoles": mapRolesBytes,
+		},
+	}
+
+	return cm, nil
+}
+
+func (c *Config) CreateDefaultNodeGroupAuthConfigMap(clientSet *clientset.Clientset) error {
+	cm, err := c.newNodeAuthConfigMap()
+	if err != nil {
+		return errors.Wrap(err, "contructing auth ConfigMap for DefaultNodeGroup")
+	}
+	if _, err := clientSet.CoreV1().ConfigMaps("default").Create(cm); err != nil {
+		return errors.Wrap(err, "creating auth ConfigMap for DefaultNodeGroup")
+	}
+	return nil
+}
+
+func isNodeReady(node *corev1.Node) bool {
+	for _, c := range node.Status.Conditions {
+		if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}
+
+func getNodes(clientSet *clientset.Clientset) (int, error) {
+	nodes, err := clientSet.Core().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return 0, err
+	}
+	logger.Info("the cluster has %d nodes", len(nodes.Items))
+	for _, node := range nodes.Items {
+		// logger.Debug("node[%d]=%#v", n, node)
+		ready := "not ready"
+		if isNodeReady(&node) {
+			ready = "ready"
+		}
+		logger.Info("node %q is %s", node.ObjectMeta.Name, ready)
+	}
+	return len(nodes.Items), nil
+}
+
+func (c *Config) WaitForNodes(clientSet *clientset.Clientset) error {
+	timeoutAfter := 20 * time.Minute
+	timer := time.After(timeoutAfter)
+	timeout := false
+	watcher, err := clientSet.Core().Nodes().Watch(metav1.ListOptions{})
+	if err != nil {
+		return errors.Wrap(err, "creating node watcher")
+	}
+
+	counter, err := getNodes(clientSet)
+	if err != nil {
+		return errors.Wrap(err, "listing nodes")
+	}
+
+	logger.Info("waiting for at least %d nodes to become ready", c.MinNodes)
+	for !timeout && counter <= c.MinNodes {
+		select {
+		case event, _ := <-watcher.ResultChan():
+			logger.Debug("event = %#v", event)
+			if event.Type == watch.Added {
+				node := event.Object.(*corev1.Node)
+				if isNodeReady(node) {
+					counter++
+					logger.Debug("node %q is ready", node.ObjectMeta.Name)
+				} else {
+					logger.Debug("node %q seen, but not ready yet", node.ObjectMeta.Name)
+					logger.Debug("node = %#v", *node)
+				}
+			}
+		case <-timer:
+			timeout = true
+		}
+	}
+	if timeout {
+		return fmt.Errorf("timed out (after %s) waitiing for at least %d nodes to join the cluster and become ready", timeoutAfter, c.MinNodes)
+	}
+
+	if _, err = getNodes(clientSet); err != nil {
+		errors.Wrap(err, "re-listing nodes")
+	}
+
+	return nil
+}