feat: add :51090/metrics endpoint with Prometheus metrics listening on machine IP (#304)

Signed-off-by: Miek Gieben <miek@miek.nl>
Co-authored-by: Pasha Sviderski <me@psviderski.name>

internal/machine/cluster.go

@@ -20,6 +20,7 @@ import (
 	"github.com/psviderski/uncloud/internal/machine/dns"
 	"github.com/psviderski/uncloud/internal/machine/docker"
 	"github.com/psviderski/uncloud/internal/machine/firewall"
+	"github.com/psviderski/uncloud/internal/machine/metrics"
 	"github.com/psviderski/uncloud/internal/machine/network"
 	"github.com/psviderski/uncloud/internal/machine/store"
 	"github.com/psviderski/unregistry"
@@ -55,6 +56,8 @@ type clusterController struct {
 	// unregistry is the embedded container registry that uses the local Docker (containerd) image store as its backend.
 	unregistry *unregistry.Registry
 
+	metricsServer *metrics.Server
+
 	// stopped is a channel that is closed when the controller is stopped.
 	stopped chan struct{}
 }
@@ -72,6 +75,7 @@ func newClusterController(
 	dnsServer *dns.Server,
 	dnsResolver *dns.ClusterResolver,
 	unregistry *unregistry.Registry,
+	metricsServer *metrics.Server,
 ) (*clusterController, error) {
 	slog.Info("Starting WireGuard network.")
 	wgnet, err := network.NewWireGuardNetwork()
@@ -95,6 +99,7 @@ func newClusterController(
 		dnsServer:       dnsServer,
 		dnsResolver:     dnsResolver,
 		unregistry:      unregistry,
+		metricsServer:   metricsServer,
 		stopped:         make(chan struct{}),
 	}, nil
 }
@@ -186,6 +191,14 @@ func (cc *clusterController) Run(ctx context.Context) error {
 		return err
 	}
 
+	errGroup.Go(func() error {
+		slog.Info("Starting metrics server.")
+		if err := cc.metricsServer.Run(ctx); err != nil {
+			return fmt.Errorf("metrics server failed: %w", err)
+		}
+		return nil
+	})
+
 	errGroup.Go(func() error {
 		slog.Info("Starting embedded DNS resolver.")
 		if err := cc.dnsResolver.Run(ctx); err != nil {

internal/machine/dns/server.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/miekg/dns"
+	"github.com/psviderski/uncloud/internal/metrics"
 )
 
 const (
@@ -196,6 +197,7 @@ func (s *Server) handleRequest(w dns.ResponseWriter, req *dns.Msg) {
 			log.Error("Failed to forward DNS query.", "err", err)
 			resp = new(dns.Msg).SetRcode(req, dns.RcodeServerFailure)
 		}
+		metrics.DNSQuery.WithLabelValues("false", metrics.Status(err)).Inc()
 
 		s.reply(w, req, resp)
 		return
@@ -232,6 +234,7 @@ func (s *Server) handleRequest(w dns.ResponseWriter, req *dns.Msg) {
 		}
 	}
 	resp.Truncate(maxSize)
+	metrics.DNSQuery.WithLabelValues("true", metrics.Ok).Inc() // NameError is not an error
 
 	s.reply(w, req, resp)
 }

internal/machine/machine.go

@@ -33,6 +33,7 @@ import (
 	"github.com/psviderski/uncloud/internal/machine/corroservice"
 	"github.com/psviderski/uncloud/internal/machine/dns"
 	machinedocker "github.com/psviderski/uncloud/internal/machine/docker"
+	"github.com/psviderski/uncloud/internal/machine/metrics"
 	"github.com/psviderski/uncloud/internal/machine/network"
 	"github.com/psviderski/uncloud/internal/machine/store"
 	"github.com/psviderski/uncloud/internal/secret"
@@ -483,6 +484,8 @@ func (m *Machine) Run(ctx context.Context) error {
 				return fmt.Errorf("create embedded DNS server: %w", err)
 			}
 
+			metricsServer := metrics.New(m.IP())
+
 			var unreg *unregistry.Registry
 			if containerdSock := m.ContainerdSock(); containerdSock != "" {
 				isContainerdStore, err := m.dockerService.IsContainerdImageStoreEnabled(ctx)
@@ -524,6 +527,7 @@ func (m *Machine) Run(ctx context.Context) error {
 				dnsServer,
 				dnsResolver,
 				unreg,
+				metricsServer,
 			)
 			m.mu.Unlock()
 			if err != nil {

internal/machine/metrics/server.go

@@ -0,0 +1,58 @@
+package metrics
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/netip"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+const Port = 51090
+
+type Server struct {
+	*http.Server
+	listenAddr netip.Addr
+	log        *slog.Logger
+}
+
+func New(listenAddr netip.Addr) *Server {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	server := &http.Server{Handler: mux, ReadTimeout: 5 * time.Second}
+	return &Server{server, listenAddr, slog.With("component", "metrics")}
+}
+
+func (s *Server) Run(ctx context.Context) error {
+	addr := net.JoinHostPort(s.listenAddr.String(), strconv.Itoa(Port))
+	l, err := net.Listen("tcp", addr)
+	if err != nil {
+		return fmt.Errorf("listen metrics server: %w", err)
+	}
+
+	errCh := make(chan error, 1)
+
+	go func() {
+		if err := s.Serve(l); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			errCh <- fmt.Errorf("metrics server failed: %w", err)
+		}
+	}()
+
+	select {
+	case err := <-errCh:
+		// The server failed on its own so there are no connections to drain.
+		return err
+	case <-ctx.Done():
+		s.log.Info("Stopping metrics server.")
+		// ctx is already cancelled so a graceful drain needs a new one with a reasonable timeout.
+		stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		return s.Shutdown(stopCtx)
+	}
+}