metrics: add prometheus-based libp2p and node-specific metrics

README.md

@@ -53,4 +53,45 @@ Example:
   },
 .... other fields are omitted
 }
-```
+```
+
+## Metrics
+
+The node exposes Prometheus metrics at `/metrics` on its HTTP API address
+(`httpListenAddress`, default `127.0.0.1:9010`):
+
+```bash
+curl http://127.0.0.1:9010/metrics
+```
+
+This includes libp2p's built-in metrics (`libp2p_relaysvc_*`, `libp2p_autonatv2_*`,
+`libp2p_swarm_*`, `libp2p_rcmgr_*`, `libp2p_identify_*`, `libp2p_holepunch_*`, ...),
+Go runtime/process metrics (`go_*`, `process_*`), and a few bootstrap-node-specific
+gauges (`awl_bootstrap_*`): node info/uptime, DHT routing table size, node bandwidth and
+peerstore size. The official libp2p Grafana dashboards can be used as-is.
+
+### Grafana + Prometheus stack
+
+The [`awl`](https://github.com/anywherelan/awl) repository ships a ready-to-use
+Prometheus + Grafana monitoring stack under
+[`monitoring/`](https://github.com/anywherelan/awl/tree/master/monitoring)
+(docker-compose with the official libp2p dashboards). Since the bootstrap node
+exposes the same `libp2p_*` metrics, that stack works here without changes — the
+most relevant dashboards for a bootstrap node are **relaysvc**, **autonatv2**,
+**swarm** and **resource-manager**.
+
+To point it at this node, set the scrape target in `monitoring/prometheus.yml` to
+this node's HTTP address (default port `9010` instead of awl's `8639`):
+
+```yaml
+  - job_name: awl-bootstrap
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - host.docker.internal:9010
+```
+
+## Profiling
+
+pprof endpoints are served under `/api/v0/debug/pprof/` and are enabled by default.
+Set `disablePprof: true` in the config to turn them off.

api/api.go

@@ -8,13 +8,15 @@ import (
 	http_pprof "net/http/pprof"
 	"runtime/pprof"
 
-	"github.com/anywherelan/awl-bootstrap-node/config"
 	"github.com/anywherelan/awl/p2p"
 	"github.com/anywherelan/awl/ringbuffer"
 	"github.com/go-playground/validator/v10"
 	"github.com/ipfs/go-log/v2"
+	"github.com/labstack/echo-contrib/echoprometheus"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
+
+	"github.com/anywherelan/awl-bootstrap-node/config"
 )
 
 type Handler struct {
@@ -34,6 +36,9 @@ func NewHandler(conf *config.Config, p2p *p2p.P2p, logBuffer *ringbuffer.RingBuf
 	}
 }
 
+// global so the /metrics collector is registered only once even if SetupAPI runs multiple times (tests).
+var metricsHandler = echoprometheus.NewHandler()
+
 func (h *Handler) SetupAPI() error {
 	e := echo.New()
 	h.echo = e
@@ -49,11 +54,14 @@ func (h *Handler) SetupAPI() error {
 
 	// Routes
 
+	// Metrics
+	e.GET("/metrics", metricsHandler)
+
 	// Debug
 	e.GET(GetP2pDebugInfoPath, h.GetP2pDebugInfo)
 	e.GET(GetDebugLogPath, h.GetLog)
 
-	if h.conf.DevMode() {
+	if !h.conf.DisablePprof {
 		e.Any(V0Prefix+"debug/pprof/", echo.WrapHandler(http.HandlerFunc(http_pprof.Index)))
 		e.Any(V0Prefix+"debug/pprof/profile", echo.WrapHandler(http.HandlerFunc(http_pprof.Profile)))
 		e.Any(V0Prefix+"debug/pprof/trace", echo.WrapHandler(http.HandlerFunc(http_pprof.Trace)))

application.go

@@ -22,11 +22,13 @@ import (
 	"github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem"
 	rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
 	"github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
+	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
 	"github.com/anywherelan/awl-bootstrap-node/api"
 	"github.com/anywherelan/awl-bootstrap-node/config"
+	"github.com/anywherelan/awl-bootstrap-node/metrics"
 )
 
 const (
@@ -67,6 +69,10 @@ func (a *Application) Init(ctx context.Context) error {
 
 	p2pSrv.Bootstrap()
 
+	// Metrics
+	metrics.SetNodeInfo(config.Version, host.ID().String())
+	go metrics.StartBackgroundUpdater(a.ctx, a.p2pServer)
+
 	handler := api.NewHandler(a.Conf, a.p2pServer, a.LogBuffer)
 	a.Api = handler
 	err = handler.SetupAPI()
@@ -181,7 +187,12 @@ func (a *Application) makeP2pHostConfig() (p2p.HostConfig, error) {
 
 	// TODO: move to config file
 	resourceLimitsConfig := rcmgr.InfiniteLimits
-	mgr, err := rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(resourceLimitsConfig))
+	// Trace reporter enables libp2p_rcmgr_* Prometheus metrics (current streams/connections/memory/fds).
+	rcmgrReporter, err := rcmgr.NewStatsTraceReporter()
+	if err != nil {
+		panic(err)
+	}
+	mgr, err := rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(resourceLimitsConfig), rcmgr.WithTraceReporter(rcmgrReporter))
 	if err != nil {
 		panic(err)
 	}
@@ -219,6 +230,7 @@ func (a *Application) makeP2pHostConfig() (p2p.HostConfig, error) {
 			libp2p.AutoNATServiceRateLimit(0, 2, time.Second),
 			libp2p.ForceReachabilityPublic(),
 			libp2p.ResourceManager(mgr),
+			libp2p.PrometheusRegisterer(prometheus.DefaultRegisterer),
 		},
 		ConnManager: struct {
 			LowWater    int

application_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -63,6 +64,24 @@ func TestApplicationSmoke(t *testing.T) {
 	if resp.StatusCode != http.StatusOK {
 		t.Errorf("log endpoint status = %d, want 200", resp.StatusCode)
 	}
+
+	// Metrics endpoint must expose both our awl_bootstrap__* metrics and libp2p's built-in
+	// families. The libp2p_* ones are the important safety check: they confirm
+	// PrometheusRegisterer was wired through and the relay-service metrics tracer
+	// got registered.
+	metricsURL := fmt.Sprintf("http://127.0.0.1:%d/metrics", httpPort)
+	metricsBody := getWithRetry(t, metricsURL, 5*time.Second)
+	wantFamilies := []string{
+		"awl_bootstrap_node_info",
+		"awl_bootstrap_p2p_dht_routing_table_size",
+		"libp2p_swarm_",    // PrometheusRegisterer reached the swarm
+		"libp2p_relaysvc_", // relay-service metrics tracer was registered
+	}
+	for _, want := range wantFamilies {
+		if !bytes.Contains(metricsBody, []byte(want)) {
+			t.Errorf("/metrics output does not contain %q", want)
+		}
+	}
 }
 
 // writeTestConfig generates a fresh ed25519 identity, builds a minimal Config

config/config.go

@@ -23,6 +23,8 @@ type (
 		P2pNode           P2pNode
 		LoggerLevel       string
 		HttpListenAddress string
+		// DisablePprof turns off the pprof debug endpoints.
+		DisablePprof bool
 	}
 	P2pNode struct {
 		PeerID   string

go.mod

@@ -9,12 +9,14 @@ require (
 	github.com/ipfs/go-datastore v0.9.2
 	github.com/ipfs/go-ds-badger v0.3.4
 	github.com/ipfs/go-log/v2 v2.9.1
+	github.com/labstack/echo-contrib v0.50.1
 	github.com/labstack/echo/v4 v4.15.4
 	github.com/libp2p/go-libp2p v0.48.0
 	github.com/libp2p/go-libp2p-kad-dht v0.39.2
 	github.com/libp2p/go-libp2p-kbucket v0.8.0
 	github.com/mr-tron/base58 v1.3.0
 	github.com/multiformats/go-multiaddr v0.16.1
+	github.com/prometheus/client_golang v1.23.2
 	go.uber.org/zap v1.28.0
 )
 
@@ -105,7 +107,6 @@ require (
 	github.com/pion/webrtc/v4 v4.1.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.20.1 // indirect

go.sum

@@ -279,6 +279,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU=
@@ -293,6 +295,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/labstack/echo-contrib v0.50.1 h1:W9cZZ9viA4TDdFtm8cuA+XGFwOcnfbjJpl7VgfsRLHE=
+github.com/labstack/echo-contrib v0.50.1/go.mod h1:8r/++U/Fw/QniApFnzunLanKaviPfBX7fX7/2QX0qOk=
 github.com/labstack/echo/v4 v4.15.4 h1:DL45vVYa+BWE+XuW+zZNd9H0YEdZ80UAWJGcTVW4EVs=
 github.com/labstack/echo/v4 v4.15.4/go.mod h1:CuMetKIRwsuO/qlAgMq+KTAalwGoB/h4tC+yPdrTj1g=
 github.com/labstack/gommon v0.5.0 h1:6VSQ2NOzsnEJ5W6+84E0RbcaDDmgB6NIAzWCczTEe6c=

metrics/metrics.go

@@ -0,0 +1,70 @@
+// Package metrics provides Prometheus metrics specific to awl-bootstrap-node.
+//
+// Most of the useful telemetry for a bootstrap node (relay service, autonat,
+// swarm, resource manager, identify, holepunch, ...) comes from libp2p's own
+// built-in Prometheus metrics, enabled via libp2p.PrometheusRegisterer. This
+// package only adds the few gauges that libp2p does not expose: DHT routing
+// table size, total node bandwidth and node info/uptime.
+package metrics
+
+import (
+	"context"
+	"time"
+
+	"github.com/libp2p/go-libp2p/core/host"
+	"github.com/libp2p/go-libp2p/core/metrics"
+)
+
+const (
+	namespace = "awl_bootstrap"
+
+	subsystemNode      = "node"
+	subsystemP2P       = "p2p"
+	subsystemPeerstore = "peerstore"
+)
+
+// P2pMetrics is an interface for getting p2p stats used by the background updater.
+// It is satisfied by *github.com/anywherelan/awl/p2p.P2p.
+type P2pMetrics interface {
+	RoutingTableSize() int
+	NetworkStats() metrics.Stats
+	BootstrapPeersStats() (total int, connected int)
+	Host() host.Host
+}
+
+// StartBackgroundUpdater periodically updates gauge-type metrics from their data sources.
+func StartBackgroundUpdater(ctx context.Context, p2pMetrics P2pMetrics) {
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+
+	startTime := time.Now()
+	updateGauges(p2pMetrics, startTime)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			updateGauges(p2pMetrics, startTime)
+		}
+	}
+}
+
+func updateGauges(p2pMetrics P2pMetrics, startTime time.Time) {
+	P2PDHTRoutingTableSize.Set(float64(p2pMetrics.RoutingTableSize()))
+
+	_, bootstrapConnected := p2pMetrics.BootstrapPeersStats()
+	P2PBootstrapPeersConnected.Set(float64(bootstrapConnected))
+
+	stats := p2pMetrics.NetworkStats()
+	P2PBandwidthBytesTotal.WithLabelValues("in").Set(float64(stats.TotalIn))
+	P2PBandwidthBytesTotal.WithLabelValues("out").Set(float64(stats.TotalOut))
+	P2PBandwidthRateBytes.WithLabelValues("in").Set(stats.RateIn)
+	P2PBandwidthRateBytes.WithLabelValues("out").Set(stats.RateOut)
+
+	peerstore := p2pMetrics.Host().Peerstore()
+	PeerstorePeersWithAddrs.Set(float64(len(peerstore.PeersWithAddrs())))
+	PeerstorePeersWithKeys.Set(float64(len(peerstore.PeersWithKeys())))
+
+	NodeUptimeSeconds.Set(time.Since(startTime).Seconds())
+}

metrics/node.go

@@ -0,0 +1,38 @@
+package metrics
+
+import (
+	"runtime"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	NodeInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemNode,
+		Name:      "info",
+		Help:      "Static node information.",
+	}, []string{"version", "peer_id", "go_version", "os", "arch"})
+
+	NodeUptimeSeconds = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemNode,
+		Name:      "uptime_seconds",
+		Help:      "Node uptime in seconds.",
+	})
+
+	NodeStartTimestamp = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemNode,
+		Name:      "start_timestamp",
+		Help:      "Unix timestamp of node start.",
+	})
+)
+
+// SetNodeInfo sets the static node info gauge and the start timestamp.
+func SetNodeInfo(version, peerID string) {
+	NodeInfo.WithLabelValues(version, peerID, runtime.Version(), runtime.GOOS, runtime.GOARCH).Set(1)
+	NodeStartTimestamp.Set(float64(time.Now().Unix()))
+}

metrics/p2p.go

@@ -0,0 +1,43 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	// P2PDHTRoutingTableSize is not provided by libp2p Prometheus metrics
+	// (go-libp2p-kad-dht is instrumented via OpenTelemetry and does not export
+	// routing table size), so we expose it ourselves.
+	P2PDHTRoutingTableSize = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemP2P,
+		Name:      "dht_routing_table_size",
+		Help:      "DHT routing table size.",
+	})
+
+	// P2PBandwidthBytesTotal reports cumulative node-wide traffic. libp2p's
+	// BandwidthCounter is not wired to Prometheus, so we snapshot it here.
+	// Note: relayed traffic is also counted by libp2p_relaysvc_data_transferred_bytes_total.
+	// These are cumulative counters exposed as a Gauge snapshot; use rate()/increase() in queries.
+	P2PBandwidthBytesTotal = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemP2P,
+		Name:      "bandwidth_bytes_total",
+		Help:      "Total node bandwidth in bytes (snapshot of libp2p BandwidthCounter).",
+	}, []string{"direction"})
+
+	P2PBandwidthRateBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemP2P,
+		Name:      "bandwidth_rate_bytes",
+		Help:      "Current node bandwidth rate in bytes per second.",
+	}, []string{"direction"})
+
+	P2PBootstrapPeersConnected = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemP2P,
+		Name:      "bootstrap_peers_connected",
+		Help:      "Number of connected bootstrap peers.",
+	})
+)