diff --git a/README.md b/README.md index bc681d6..cc18c98 100644 --- a/README.md +++ b/README.md @@ -53,4 +53,45 @@ Example: }, .... other fields are omitted } -``` \ No newline at end of file +``` + +## Metrics + +The node exposes Prometheus metrics at `/metrics` on its HTTP API address +(`httpListenAddress`, default `127.0.0.1:9010`): + +```bash +curl http://127.0.0.1:9010/metrics +``` + +This includes libp2p's built-in metrics (`libp2p_relaysvc_*`, `libp2p_autonatv2_*`, +`libp2p_swarm_*`, `libp2p_rcmgr_*`, `libp2p_identify_*`, `libp2p_holepunch_*`, ...), +Go runtime/process metrics (`go_*`, `process_*`), and a few bootstrap-node-specific +gauges (`awl_bootstrap_*`): node info/uptime, DHT routing table size, node bandwidth and +peerstore size. The official libp2p Grafana dashboards can be used as-is. + +### Grafana + Prometheus stack + +The [`awl`](https://github.com/anywherelan/awl) repository ships a ready-to-use +Prometheus + Grafana monitoring stack under +[`monitoring/`](https://github.com/anywherelan/awl/tree/master/monitoring) +(docker-compose with the official libp2p dashboards). Since the bootstrap node +exposes the same `libp2p_*` metrics, that stack works here without changes — the +most relevant dashboards for a bootstrap node are **relaysvc**, **autonatv2**, +**swarm** and **resource-manager**. + +To point it at this node, set the scrape target in `monitoring/prometheus.yml` to +this node's HTTP address (default port `9010` instead of awl's `8639`): + +```yaml + - job_name: awl-bootstrap + metrics_path: /metrics + static_configs: + - targets: + - host.docker.internal:9010 +``` + +## Profiling + +pprof endpoints are served under `/api/v0/debug/pprof/` and are enabled by default. +Set `disablePprof: true` in the config to turn them off. \ No newline at end of file diff --git a/api/api.go b/api/api.go index 9df6f04..91bd234 100644 --- a/api/api.go +++ b/api/api.go @@ -8,13 +8,15 @@ import ( http_pprof "net/http/pprof" "runtime/pprof" - "github.com/anywherelan/awl-bootstrap-node/config" "github.com/anywherelan/awl/p2p" "github.com/anywherelan/awl/ringbuffer" "github.com/go-playground/validator/v10" "github.com/ipfs/go-log/v2" + "github.com/labstack/echo-contrib/echoprometheus" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" + + "github.com/anywherelan/awl-bootstrap-node/config" ) type Handler struct { @@ -34,6 +36,9 @@ func NewHandler(conf *config.Config, p2p *p2p.P2p, logBuffer *ringbuffer.RingBuf } } +// global so the /metrics collector is registered only once even if SetupAPI runs multiple times (tests). +var metricsHandler = echoprometheus.NewHandler() + func (h *Handler) SetupAPI() error { e := echo.New() h.echo = e @@ -49,11 +54,14 @@ func (h *Handler) SetupAPI() error { // Routes + // Metrics + e.GET("/metrics", metricsHandler) + // Debug e.GET(GetP2pDebugInfoPath, h.GetP2pDebugInfo) e.GET(GetDebugLogPath, h.GetLog) - if h.conf.DevMode() { + if !h.conf.DisablePprof { e.Any(V0Prefix+"debug/pprof/", echo.WrapHandler(http.HandlerFunc(http_pprof.Index))) e.Any(V0Prefix+"debug/pprof/profile", echo.WrapHandler(http.HandlerFunc(http_pprof.Profile))) e.Any(V0Prefix+"debug/pprof/trace", echo.WrapHandler(http.HandlerFunc(http_pprof.Trace))) diff --git a/application.go b/application.go index eab5678..33e6b79 100644 --- a/application.go +++ b/application.go @@ -22,11 +22,13 @@ import ( "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" "go.uber.org/zap/zapcore" "github.com/anywherelan/awl-bootstrap-node/api" "github.com/anywherelan/awl-bootstrap-node/config" + "github.com/anywherelan/awl-bootstrap-node/metrics" ) const ( @@ -67,6 +69,10 @@ func (a *Application) Init(ctx context.Context) error { p2pSrv.Bootstrap() + // Metrics + metrics.SetNodeInfo(config.Version, host.ID().String()) + go metrics.StartBackgroundUpdater(a.ctx, a.p2pServer) + handler := api.NewHandler(a.Conf, a.p2pServer, a.LogBuffer) a.Api = handler err = handler.SetupAPI() @@ -181,7 +187,12 @@ func (a *Application) makeP2pHostConfig() (p2p.HostConfig, error) { // TODO: move to config file resourceLimitsConfig := rcmgr.InfiniteLimits - mgr, err := rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(resourceLimitsConfig)) + // Trace reporter enables libp2p_rcmgr_* Prometheus metrics (current streams/connections/memory/fds). + rcmgrReporter, err := rcmgr.NewStatsTraceReporter() + if err != nil { + panic(err) + } + mgr, err := rcmgr.NewResourceManager(rcmgr.NewFixedLimiter(resourceLimitsConfig), rcmgr.WithTraceReporter(rcmgrReporter)) if err != nil { panic(err) } @@ -219,6 +230,7 @@ func (a *Application) makeP2pHostConfig() (p2p.HostConfig, error) { libp2p.AutoNATServiceRateLimit(0, 2, time.Second), libp2p.ForceReachabilityPublic(), libp2p.ResourceManager(mgr), + libp2p.PrometheusRegisterer(prometheus.DefaultRegisterer), }, ConnManager: struct { LowWater int diff --git a/application_test.go b/application_test.go index 9db9862..fc898fd 100644 --- a/application_test.go +++ b/application_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" @@ -63,6 +64,24 @@ func TestApplicationSmoke(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Errorf("log endpoint status = %d, want 200", resp.StatusCode) } + + // Metrics endpoint must expose both our awl_bootstrap__* metrics and libp2p's built-in + // families. The libp2p_* ones are the important safety check: they confirm + // PrometheusRegisterer was wired through and the relay-service metrics tracer + // got registered. + metricsURL := fmt.Sprintf("http://127.0.0.1:%d/metrics", httpPort) + metricsBody := getWithRetry(t, metricsURL, 5*time.Second) + wantFamilies := []string{ + "awl_bootstrap_node_info", + "awl_bootstrap_p2p_dht_routing_table_size", + "libp2p_swarm_", // PrometheusRegisterer reached the swarm + "libp2p_relaysvc_", // relay-service metrics tracer was registered + } + for _, want := range wantFamilies { + if !bytes.Contains(metricsBody, []byte(want)) { + t.Errorf("/metrics output does not contain %q", want) + } + } } // writeTestConfig generates a fresh ed25519 identity, builds a minimal Config diff --git a/config/config.go b/config/config.go index 64a4ffc..009749d 100644 --- a/config/config.go +++ b/config/config.go @@ -23,6 +23,8 @@ type ( P2pNode P2pNode LoggerLevel string HttpListenAddress string + // DisablePprof turns off the pprof debug endpoints. + DisablePprof bool } P2pNode struct { PeerID string diff --git a/go.mod b/go.mod index 6a381a0..128b9be 100644 --- a/go.mod +++ b/go.mod @@ -9,12 +9,14 @@ require ( github.com/ipfs/go-datastore v0.9.2 github.com/ipfs/go-ds-badger v0.3.4 github.com/ipfs/go-log/v2 v2.9.1 + github.com/labstack/echo-contrib v0.50.1 github.com/labstack/echo/v4 v4.15.4 github.com/libp2p/go-libp2p v0.48.0 github.com/libp2p/go-libp2p-kad-dht v0.39.2 github.com/libp2p/go-libp2p-kbucket v0.8.0 github.com/mr-tron/base58 v1.3.0 github.com/multiformats/go-multiaddr v0.16.1 + github.com/prometheus/client_golang v1.23.2 go.uber.org/zap v1.28.0 ) @@ -105,7 +107,6 @@ require ( github.com/pion/webrtc/v4 v4.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect diff --git a/go.sum b/go.sum index 3705dcc..4774141 100644 --- a/go.sum +++ b/go.sum @@ -279,6 +279,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/koron/go-ssdp v0.0.6 h1:Jb0h04599eq/CY7rB5YEqPS83HmRfHP2azkxMN2rFtU= @@ -293,6 +295,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo-contrib v0.50.1 h1:W9cZZ9viA4TDdFtm8cuA+XGFwOcnfbjJpl7VgfsRLHE= +github.com/labstack/echo-contrib v0.50.1/go.mod h1:8r/++U/Fw/QniApFnzunLanKaviPfBX7fX7/2QX0qOk= github.com/labstack/echo/v4 v4.15.4 h1:DL45vVYa+BWE+XuW+zZNd9H0YEdZ80UAWJGcTVW4EVs= github.com/labstack/echo/v4 v4.15.4/go.mod h1:CuMetKIRwsuO/qlAgMq+KTAalwGoB/h4tC+yPdrTj1g= github.com/labstack/gommon v0.5.0 h1:6VSQ2NOzsnEJ5W6+84E0RbcaDDmgB6NIAzWCczTEe6c= diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 0000000..7262349 --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,70 @@ +// Package metrics provides Prometheus metrics specific to awl-bootstrap-node. +// +// Most of the useful telemetry for a bootstrap node (relay service, autonat, +// swarm, resource manager, identify, holepunch, ...) comes from libp2p's own +// built-in Prometheus metrics, enabled via libp2p.PrometheusRegisterer. This +// package only adds the few gauges that libp2p does not expose: DHT routing +// table size, total node bandwidth and node info/uptime. +package metrics + +import ( + "context" + "time" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/metrics" +) + +const ( + namespace = "awl_bootstrap" + + subsystemNode = "node" + subsystemP2P = "p2p" + subsystemPeerstore = "peerstore" +) + +// P2pMetrics is an interface for getting p2p stats used by the background updater. +// It is satisfied by *github.com/anywherelan/awl/p2p.P2p. +type P2pMetrics interface { + RoutingTableSize() int + NetworkStats() metrics.Stats + BootstrapPeersStats() (total int, connected int) + Host() host.Host +} + +// StartBackgroundUpdater periodically updates gauge-type metrics from their data sources. +func StartBackgroundUpdater(ctx context.Context, p2pMetrics P2pMetrics) { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + startTime := time.Now() + updateGauges(p2pMetrics, startTime) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + updateGauges(p2pMetrics, startTime) + } + } +} + +func updateGauges(p2pMetrics P2pMetrics, startTime time.Time) { + P2PDHTRoutingTableSize.Set(float64(p2pMetrics.RoutingTableSize())) + + _, bootstrapConnected := p2pMetrics.BootstrapPeersStats() + P2PBootstrapPeersConnected.Set(float64(bootstrapConnected)) + + stats := p2pMetrics.NetworkStats() + P2PBandwidthBytesTotal.WithLabelValues("in").Set(float64(stats.TotalIn)) + P2PBandwidthBytesTotal.WithLabelValues("out").Set(float64(stats.TotalOut)) + P2PBandwidthRateBytes.WithLabelValues("in").Set(stats.RateIn) + P2PBandwidthRateBytes.WithLabelValues("out").Set(stats.RateOut) + + peerstore := p2pMetrics.Host().Peerstore() + PeerstorePeersWithAddrs.Set(float64(len(peerstore.PeersWithAddrs()))) + PeerstorePeersWithKeys.Set(float64(len(peerstore.PeersWithKeys()))) + + NodeUptimeSeconds.Set(time.Since(startTime).Seconds()) +} diff --git a/metrics/node.go b/metrics/node.go new file mode 100644 index 0000000..2112c5f --- /dev/null +++ b/metrics/node.go @@ -0,0 +1,38 @@ +package metrics + +import ( + "runtime" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + NodeInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemNode, + Name: "info", + Help: "Static node information.", + }, []string{"version", "peer_id", "go_version", "os", "arch"}) + + NodeUptimeSeconds = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemNode, + Name: "uptime_seconds", + Help: "Node uptime in seconds.", + }) + + NodeStartTimestamp = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemNode, + Name: "start_timestamp", + Help: "Unix timestamp of node start.", + }) +) + +// SetNodeInfo sets the static node info gauge and the start timestamp. +func SetNodeInfo(version, peerID string) { + NodeInfo.WithLabelValues(version, peerID, runtime.Version(), runtime.GOOS, runtime.GOARCH).Set(1) + NodeStartTimestamp.Set(float64(time.Now().Unix())) +} diff --git a/metrics/p2p.go b/metrics/p2p.go new file mode 100644 index 0000000..cc1b9f0 --- /dev/null +++ b/metrics/p2p.go @@ -0,0 +1,43 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + // P2PDHTRoutingTableSize is not provided by libp2p Prometheus metrics + // (go-libp2p-kad-dht is instrumented via OpenTelemetry and does not export + // routing table size), so we expose it ourselves. + P2PDHTRoutingTableSize = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemP2P, + Name: "dht_routing_table_size", + Help: "DHT routing table size.", + }) + + // P2PBandwidthBytesTotal reports cumulative node-wide traffic. libp2p's + // BandwidthCounter is not wired to Prometheus, so we snapshot it here. + // Note: relayed traffic is also counted by libp2p_relaysvc_data_transferred_bytes_total. + // These are cumulative counters exposed as a Gauge snapshot; use rate()/increase() in queries. + P2PBandwidthBytesTotal = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemP2P, + Name: "bandwidth_bytes_total", + Help: "Total node bandwidth in bytes (snapshot of libp2p BandwidthCounter).", + }, []string{"direction"}) + + P2PBandwidthRateBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemP2P, + Name: "bandwidth_rate_bytes", + Help: "Current node bandwidth rate in bytes per second.", + }, []string{"direction"}) + + P2PBootstrapPeersConnected = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemP2P, + Name: "bootstrap_peers_connected", + Help: "Number of connected bootstrap peers.", + }) +) diff --git a/metrics/peerstore.go b/metrics/peerstore.go new file mode 100644 index 0000000..7aa84a5 --- /dev/null +++ b/metrics/peerstore.go @@ -0,0 +1,22 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + PeerstorePeersWithAddrs = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemPeerstore, + Name: "peers_with_addrs", + Help: "Number of peers in the peerstore that have known addresses.", + }) + + PeerstorePeersWithKeys = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystemPeerstore, + Name: "peers_with_keys", + Help: "Number of peers in the peerstore that have known public keys.", + }) +)