chore: enable gRPC auto retries for transient Unavailable failures up to ~8s

pkg/client/connector/common.go

@@ -0,0 +1,30 @@
+package connector
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// defaultServiceConfig defines the default gRPC service configuration including retry policy for transient failures.
+var defaultServiceConfig = mustMarshalJSON(map[string]any{
+	"methodConfig": []map[string]any{
+		{
+			"name": []map[string]string{{"service": ""}},
+			"retryPolicy": map[string]any{
+				"maxAttempts":          5, // 5 is the maximum allowed by gRPC
+				"initialBackoff":       "0.5s",
+				"maxBackoff":           "5s",
+				"backoffMultiplier":    2,
+				"retryableStatusCodes": []string{"UNAVAILABLE"},
+			},
+		},
+	},
+})
+
+func mustMarshalJSON(v any) string {
+	b, err := json.Marshal(v)
+	if err != nil {
+		panic(fmt.Sprintf("failed to marshal service config: %v", err))
+	}
+	return string(b)
+}

pkg/client/connector/ssh.go

@@ -59,6 +59,7 @@ func (c *SSHConnector) Connect(ctx context.Context) (*grpc.ClientConn, error) {
 	conn, err := grpc.NewClient(
 		"unix://"+sockPath,
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithDefaultServiceConfig(defaultServiceConfig),
 		grpc.WithContextDialer(
 			func(ctx context.Context, addr string) (net.Conn, error) {
 				addr = strings.TrimPrefix(addr, "unix://")

pkg/client/connector/sshcli.go

@@ -90,6 +90,7 @@ func (c *SSHCLIConnector) Connect(ctx context.Context) (*grpc.ClientConn, error)
 	grpcConn, err := grpc.NewClient(
 		"passthrough:///", // Dummy target since we're using a custom dialer.
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithDefaultServiceConfig(defaultServiceConfig),
 		grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) {
 			return c.conn, nil
 		}),

pkg/client/connector/tcp.go

@@ -23,6 +23,7 @@ func (c *TCPConnector) Connect(_ context.Context) (*grpc.ClientConn, error) {
 	conn, err := grpc.NewClient(
 		c.apiAddr.String(),
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithDefaultServiceConfig(defaultServiceConfig),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("create machine API client: %w", err)

pkg/client/connector/unix.go

@@ -25,6 +25,7 @@ func (c *UnixConnector) Connect(_ context.Context) (*grpc.ClientConn, error) {
 	conn, err := grpc.NewClient(
 		target,
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithDefaultServiceConfig(defaultServiceConfig),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("create machine API client: %w", err)

pkg/client/connector/wireguard.go

@@ -66,6 +66,7 @@ func (c *WireGuardConnector) Connect(ctx context.Context) (*grpc.ClientConn, err
 	conn, err := grpc.NewClient(
 		machineAPIAddr,
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithDefaultServiceConfig(defaultServiceConfig),
 		grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
 			return c.tun.DialContext(ctx, "tcp", addr)
 		}),