Wait for dapr api to be ready to establish connection from dapr runti…

…me. (dapr#5168)

* Wait for dapr api to be ready to establish connection from dapr runtime.

- add blocking connection call with timeout.
- dapr runtime will crash after defined timeout. In case of k8s, the pod will be restarted post crash by default.

Signed-off-by: Mayank Kumar <makumar@infoblox.com>

* address review comment

- made dial timeout a constant of 30sec.

Signed-off-by: Mayank Kumar <makumar@infoblox.com>

Signed-off-by: Mayank Kumar <makumar@infoblox.com>
Co-authored-by: Yaron Schneider <schneider.yaron@live.com>

pkg/operator/client/client.go

@@ -1,7 +1,9 @@
 package client
 
 import (
+	"context"
 	"crypto/x509"
+	"time"
 
 	grpcMiddleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpcRetry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
@@ -14,6 +16,10 @@ import (
 	operatorv1pb "github.com/dapr/dapr/pkg/proto/operator/v1"
 )
 
+const (
+	dialTimeout = 30 * time.Second
+)
+
 // GetOperatorClient returns a new k8s operator client and the underlying connection.
 // If a cert chain is given, a TLS connection will be established.
 func GetOperatorClient(address, serverName string, certChain *daprCredentials.CertChain) (operatorv1pb.OperatorClient, *grpc.ClientConn, error) {
@@ -42,9 +48,12 @@ func GetOperatorClient(address, serverName string, certChain *daprCredentials.Ce
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to create tls config from cert and key")
 	}
-	opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(config)))
+	// block for connection
+	opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(config)), grpc.WithBlock())
 
-	conn, err := grpc.Dial(address, opts...)
+	ctx, cancelFunc := context.WithTimeout(context.Background(), dialTimeout)
+	defer cancelFunc()
+	conn, err := grpc.DialContext(ctx, address, opts...)
 	if err != nil {
 		return nil, nil, err
 	}