Skip to content

Commit a4b68ed

Browse files
authored
Add app and runner metrics and operational observability (#20)
Operators need application and runner views of calls, failures, streaming traffic, payments, and redemption progress. This adds per-service Prometheus registries with registered app/runner ownership, live transport observations, chain/discovery metrics, and structured events with redacted dependency errors. Paid orchestrator readiness now observes usable payment storage, fresh chain state, recipient activity, pricing, and redemption-worker completion. The operational guide documents scrape endpoints, the metric/label contract, payment and traffic queries, and starting alert conditions.
1 parent dfcf033 commit a4b68ed

65 files changed

Lines changed: 3258 additions & 323 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
!destination/
1212
!discovery/
1313
!eth/
14+
!internal/
1415
!nodeconfig/
1516
!orchestrator/
1617
!pm/

‎destination/policy.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ import (
1414
"time"
1515
)
1616

17+
// ErrDeniedAddress identifies a destination blocked by the grants policy.
18+
var ErrDeniedAddress = errors.New("destination contains a denied address; configure grants for this purpose")
19+
1720
var deniedV4 = mustPrefixes(
1821
"0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8",
1922
"169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.0.2.0/24",
@@ -149,7 +152,7 @@ func (p Policy) DialContext(ctx context.Context, network, address string) (net.C
149152
}
150153
for _, ip := range ips {
151154
if !Public(ip) && !p.granted(net.JoinHostPort(ip.String(), port)) {
152-
return nil, fmt.Errorf("%s destination contains a denied address; configure grants for this purpose", p.Purpose)
155+
return nil, fmt.Errorf("%s %w", p.Purpose, ErrDeniedAddress)
153156
}
154157
}
155158
dialer := net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}

‎discovery/app.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,11 +86,9 @@ func Serve(parent context.Context, p Params, logOut io.Writer) (result error) {
8686
}
8787
defer metricsListener.Close()
8888
metrics := http.NewServeMux()
89-
for path, body := range map[string]string{"/healthz": "ok\n", "/readyz": "ready\n", "/metrics": "livepeer_discovery_up 1\n"} {
89+
metrics.Handle("GET /metrics", app.metrics.Handler())
90+
for path, body := range map[string]string{"/healthz": "ok\n", "/readyz": "ready\n"} {
9091
metrics.HandleFunc("GET "+path, func(w http.ResponseWriter, _ *http.Request) {
91-
if path == "/metrics" {
92-
w.Header().Set("Content-Type", "text/plain; version=0.0.4")
93-
}
9492
_, _ = io.WriteString(w, body)
9593
})
9694
}

‎discovery/app_test.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"net/url"
1111
"os"
1212
"path/filepath"
13+
"strings"
1314
"testing"
1415
"time"
1516

@@ -70,7 +71,11 @@ func TestServeWithoutRunnerOrPaymentDependencies(t *testing.T) {
7071
require.NoError(t, err)
7172
require.NoError(t, response.Body.Close())
7273
require.Equal(t, 200, response.StatusCode)
73-
require.Equal(t, want, string(body))
74+
if strings.HasSuffix(path, "/metrics") {
75+
require.Contains(t, string(body), want)
76+
} else {
77+
require.Equal(t, want, string(body))
78+
}
7479
}
7580
})
7681
}

‎discovery/server.go‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,15 @@ import (
1313

1414
"github.com/livepeer/node/destination"
1515
discovery "github.com/livepeer/node/discovery/shared"
16+
"github.com/livepeer/node/internal/observability"
1617
)
1718

1819
type discoveryNode struct {
1920
URL string `json:"url"`
2021
QueriedAt time.Time `json:"queried_at"`
2122
}
2223
type Server struct {
24+
metrics *observability.Metrics
2325
logger *slog.Logger
2426
mux *http.ServeMux
2527
urls []*url.URL
@@ -71,10 +73,11 @@ func NewServer(p Params, logger *slog.Logger) (*Server, error) {
7173
client := policy.Client()
7274
client.Timeout = 3 * time.Second
7375
client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
74-
s := &Server{logger: logger, mux: http.NewServeMux(),
76+
metrics := observability.New("discovery")
77+
s := &Server{metrics: metrics, logger: logger.With("component", "discovery"), mux: http.NewServeMux(),
7578
urls: urls, interval: p.RefreshInterval,
7679
advertiseNodes: p.AdvertiseNodes, advertiseRunners: p.AdvertiseRunners,
77-
collector: &discovery.Collector[discoveryRunner]{Client: client, MaxFailures: max(1, p.MaxFailedRefreshes), Logger: logger}}
80+
collector: &discovery.Collector[discoveryRunner]{Client: client, MaxFailures: max(1, p.MaxFailedRefreshes), Logger: logger.With("component", "discovery"), Observe: metrics.Discovery}}
7881
endpoints := make([]*url.URL, 0, len(urls))
7982
for _, base := range urls {
8083
endpoint := base.Clone()
@@ -86,10 +89,18 @@ func NewServer(p Params, logger *slog.Logger) (*Server, error) {
8689
}
8790
s.collector.SetSources(endpoints)
8891
s.mux.HandleFunc("GET /discovery", s.discovery)
92+
metrics.Snapshot(s.collector.CollectMetrics)
93+
metrics.Legacy(func(emit observability.Emit) { emit("discovery_up", 1, nil) })
8994
return s, nil
9095
}
9196

92-
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.mux.ServeHTTP(w, r) }
97+
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
98+
s.metrics.HTTP(w, r, func(w http.ResponseWriter, r *http.Request) {
99+
_, pattern := s.mux.Handler(r)
100+
observability.Operation(r, pattern)
101+
s.mux.ServeHTTP(w, r)
102+
})
103+
}
93104

94105
func (s *Server) Close() { s.collector.Close() }
95106

‎discovery/shared/collector.go‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,25 @@ import (
55
"errors"
66
"fmt"
77
"log/slog"
8+
"net"
89
"net/http"
910
"net/url"
1011
"slices"
1112
"strings"
1213
"sync"
1314
"time"
15+
16+
"github.com/livepeer/node/destination"
17+
"github.com/livepeer/node/internal/observability"
1418
)
1519

1620
// Snapshot is an immutable successful inventory observation.
1721
type Snapshot[R Runner] struct {
18-
URL string
19-
QueriedAt time.Time
20-
Entries []Entry[R]
21-
Failures int
22+
URL string
23+
QueriedAt time.Time
24+
Entries []Entry[R]
25+
Failures int
26+
LastSuccess time.Time
2227
}
2328

2429
// Collector polls complete discovery endpoints and retains per-source inventories.
@@ -27,6 +32,7 @@ type Collector[R Runner] struct {
2732
Client *http.Client
2833
Logger *slog.Logger
2934
MaxFailures int
35+
Observe func(int, time.Duration, error)
3036
mu sync.Mutex
3137
sources []*Snapshot[R]
3238
pending chan struct{}
@@ -141,7 +147,11 @@ feedLoop:
141147
}
142148
workers.Go(func() {
143149
defer func() { <-slots }()
150+
started := time.Now()
144151
entries, err := c.fetch(ctx, source.URL)
152+
if c.Observe != nil {
153+
c.Observe(index, time.Since(started), err)
154+
}
145155
if ctx.Err() != nil {
146156
return
147157
}
@@ -150,6 +160,7 @@ feedLoop:
150160
if err == nil {
151161
source.Entries = entries
152162
source.QueriedAt = time.Now().UTC()
163+
source.LastSuccess = source.QueriedAt
153164
source.Failures = 0
154165
c.initialized = true
155166
} else {
@@ -166,13 +177,24 @@ feedLoop:
166177
if logger == nil {
167178
logger = slog.Default()
168179
}
169-
logger.WarnContext(ctx, "discovery refresh failed", "source_index", index, "error", err)
180+
kind := observability.ErrorKind(err)
181+
var dns *net.DNSError
182+
if errors.Is(err, destination.ErrDeniedAddress) {
183+
kind = "denied_destination"
184+
} else if errors.As(err, &dns) {
185+
kind = "dns"
186+
} else if errors.Is(err, errInvalidRedirect) {
187+
kind = "invalid_redirect"
188+
}
189+
logger.WarnContext(ctx, "discovery refresh failed", "event", "discovery_refresh", "operation", "refresh", "outcome", "error", "source_index", index, "error_kind", kind)
170190
}
171191
})
172192
}
173193
workers.Wait()
174194
}
175195

196+
var errInvalidRedirect = errors.New("invalid discovery redirect URL")
197+
176198
func (c *Collector[R]) fetch(ctx context.Context, endpoint string) ([]Entry[R], error) {
177199
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
178200
if err != nil {
@@ -186,7 +208,7 @@ func (c *Collector[R]) fetch(ctx context.Context, endpoint string) ([]Entry[R],
186208
}
187209
// Go includes the raw Location URL in this error, even after unwrapping.
188210
if strings.HasPrefix(err.Error(), "failed to parse Location header ") {
189-
err = errors.New("invalid discovery redirect URL")
211+
err = errInvalidRedirect
190212
}
191213
return nil, err
192214
}

‎discovery/shared/collector_test.go‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import (
1414
"time"
1515

1616
"github.com/livepeer/node/destination"
17+
"github.com/livepeer/node/internal/observability"
18+
"github.com/livepeer/node/internal/test"
1719
"github.com/stretchr/testify/require"
1820
)
1921

@@ -28,8 +30,30 @@ func TestCollectorCacheLifecycle(t *testing.T) {
2830
status, body := 200, envelope(candidate("1999-01-01T00:00:00Z"))
2931
c := &Collector[record]{MaxFailures: 6, Client: &http.Client{Transport: collectorTransport(func(*http.Request) (*http.Response, error) { return collectorResponse(status, body), nil })}}
3032
c.SetSources([]*url.URL{{Scheme: "https", Host: "source.example"}})
33+
m := observability.New("discovery")
34+
m.Snapshot(c.CollectMetrics)
35+
assertMetrics := func(state string, count, failures int, last time.Time) {
36+
t.Helper()
37+
require.Len(t, test.MetricSamples(t, m.Registry, "discovery_source_state", nil), 4)
38+
for _, candidate := range []string{"uninitialized", "healthy", "retained", "evicted"} {
39+
require.Equal(t, observability.Bool(candidate == state), test.MetricSum(t, m.Registry, "discovery_source_state", map[string]string{"state": candidate}))
40+
}
41+
wantLast := 0.0
42+
if !last.IsZero() {
43+
wantLast = float64(last.Unix())
44+
}
45+
require.Equal(t, wantLast, test.MetricSum(t, m.Registry, "discovery_last_success_timestamp_seconds", nil))
46+
require.EqualValues(t, count, test.MetricSum(t, m.Registry, "discovery_cached_runners", nil))
47+
require.EqualValues(t, failures, test.MetricSum(t, m.Registry, "discovery_source_failures", nil))
48+
}
49+
assertMetrics("uninitialized", 0, 0, time.Time{})
50+
status = 500
51+
c.Refresh(t.Context(), false)
52+
assertMetrics("uninitialized", 0, 1, time.Time{})
53+
status = 200
3154
c.Refresh(t.Context(), false)
3255
original := c.Snapshots()[0]
56+
assertMetrics("healthy", 1, 0, original.LastSuccess)
3357
for i, failure := range []struct {
3458
status int
3559
body string
@@ -40,22 +64,27 @@ func TestCollectorCacheLifecycle(t *testing.T) {
4064
c.Refresh(t.Context(), false)
4165
got := c.Snapshots()[0]
4266
require.Equal(t, original.Entries, got.Entries)
67+
require.Equal(t, original.LastSuccess, got.LastSuccess)
68+
assertMetrics("retained", 1, i+1, original.LastSuccess)
4369
require.Equal(t, original.QueriedAt, got.QueriedAt)
4470
require.Equal(t, i+1, got.Failures)
4571
}
4672
c.Refresh(t.Context(), false)
4773
require.Empty(t, c.Snapshots())
74+
assertMetrics("evicted", 0, 6, original.LastSuccess)
4875
require.True(t, c.Initialized(), "eviction does not reset startup")
4976
status, body = 200, envelope(candidate("1999-02-01T00:00:00Z"))
5077
c.Refresh(t.Context(), false)
5178
require.Equal(t, "1999-02-01T00:00:00Z", c.Snapshots()[0].Entries[0].Runners[0].Observed)
5279
require.Zero(t, c.Snapshots()[0].Failures)
80+
assertMetrics("healthy", 1, 0, c.Snapshots()[0].LastSuccess)
5381
status = 500
5482
c.Refresh(t.Context(), false)
5583
status, body = 200, "[]"
5684
c.Refresh(t.Context(), false)
5785
require.Len(t, c.Snapshots(), 1, "empty success is still a usable source")
5886
require.Empty(t, c.Snapshots()[0].Entries)
87+
assertMetrics("healthy", 0, 0, c.Snapshots()[0].LastSuccess)
5988
require.Zero(t, c.Snapshots()[0].Failures)
6089
}
6190

@@ -136,10 +165,10 @@ func TestCollectorFailureDiagnostics(t *testing.T) {
136165
name, endpoint, want string
137166
transport http.RoundTripper
138167
}{
139-
{"grant", "http://127.0.0.1/private-token", "denied address", policy.Client().Transport},
140-
{"DNS", "https://missing.example/private-token", "no such host", policy.Client().Transport},
141-
{"timeout", "https://source.example/private-token", "context deadline exceeded", collectorTransport(func(*http.Request) (*http.Response, error) { return nil, context.DeadlineExceeded })},
142-
{"redirect", "https://source.example/private-token", "invalid discovery redirect URL", collectorTransport(func(*http.Request) (*http.Response, error) {
168+
{"grant", "http://127.0.0.1/private-token", "denied_destination", policy.Client().Transport},
169+
{"DNS", "https://missing.example/private-token", "dns", policy.Client().Transport},
170+
{"timeout", "https://source.example/private-token", "timeout", collectorTransport(func(*http.Request) (*http.Response, error) { return nil, context.DeadlineExceeded })},
171+
{"redirect", "https://source.example/private-token", "invalid_redirect", collectorTransport(func(*http.Request) (*http.Response, error) {
143172
response := collectorResponse(302, "")
144173
response.Header = http.Header{"Location": {"https://redirect.example/private-token/%zz"}}
145174
return response, nil

‎discovery/shared/metrics.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package shared
2+
3+
import (
4+
"strconv"
5+
6+
"github.com/livepeer/node/internal/observability"
7+
)
8+
9+
// CollectMetrics includes configured sources that have never succeeded and
10+
// sources whose inventory has been evicted. Labels never contain source URLs.
11+
func (c *Collector[R]) CollectMetrics(emit observability.Emit) {
12+
c.mu.Lock()
13+
defer c.mu.Unlock()
14+
for index, source := range c.sources {
15+
labels := observability.Labels{"source_index": strconv.Itoa(index)}
16+
state := "healthy"
17+
if source.QueriedAt.IsZero() {
18+
state = "uninitialized"
19+
if !source.LastSuccess.IsZero() {
20+
state = "evicted"
21+
}
22+
} else if source.Failures > 0 {
23+
state = "retained"
24+
}
25+
for _, candidate := range []string{"healthy", "uninitialized", "evicted", "retained"} {
26+
emit("discovery_source_state", observability.Bool(state == candidate), labels.With("state", candidate))
27+
}
28+
emit("discovery_source_failures", float64(source.Failures), labels)
29+
last := 0.0
30+
if !source.LastSuccess.IsZero() {
31+
last = float64(source.LastSuccess.Unix())
32+
}
33+
emit("discovery_last_success_timestamp_seconds", last, labels)
34+
count := 0
35+
for _, entry := range source.Entries {
36+
count += len(entry.Runners)
37+
}
38+
emit("discovery_cached_runners", float64(count), labels)
39+
}
40+
}

‎docs/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ Start with [installation](../README.md#install) and the
1313
| [Signer](signer.md) | Configure price limits, authorization, discovery, and Kafka accounting. |
1414
| [Chain commands](chain.md) | Inspect accounts, manage stake and funds, and submit transactions. |
1515
| [Payment recovery](payment-recovery.md) | Back up state, upgrade or roll back, and recover accounting/redemption work. |
16+
| [Metrics and events](metrics.md) | Scrape app/runner metrics, investigate payments, and configure operational alerts. |
1617

1718
## Developing and integrating
1819

‎docs/configuration.md‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,5 @@ curl -fsS http://127.0.0.1:8940/readyz
157157
```
158158

159159
All metrics listeners serve `/healthz`, `/readyz`, and `/metrics`.
160-
Orchestrator and discovery readiness report that their listeners are running;
161-
they do not check runner health, upstream availability, or payment dependencies.
162-
Signer readiness checks its payment dependencies and, when enabled, local
163-
accounting storage. See the component guides for those checks.
160+
See [metrics and operational events](metrics.md) for readiness requirements,
161+
scrape configuration, measurements, and query/alert examples.

0 commit comments

Comments
 (0)