@@ -14,6 +14,8 @@ import (
1414 "time"
1515
1616 "github.com/livepeer/node/destination"
17+ "github.com/livepeer/node/internal/observability"
18+ "github.com/livepeer/node/internal/test"
1719 "github.com/stretchr/testify/require"
1820)
1921
@@ -28,8 +30,30 @@ func TestCollectorCacheLifecycle(t *testing.T) {
2830 status , body := 200 , envelope (candidate ("1999-01-01T00:00:00Z" ))
2931 c := & Collector [record ]{MaxFailures : 6 , Client : & http.Client {Transport : collectorTransport (func (* http.Request ) (* http.Response , error ) { return collectorResponse (status , body ), nil })}}
3032 c .SetSources ([]* url.URL {{Scheme : "https" , Host : "source.example" }})
33+ m := observability .New ("discovery" )
34+ m .Snapshot (c .CollectMetrics )
35+ assertMetrics := func (state string , count , failures int , last time.Time ) {
36+ t .Helper ()
37+ require .Len (t , test .MetricSamples (t , m .Registry , "discovery_source_state" , nil ), 4 )
38+ for _ , candidate := range []string {"uninitialized" , "healthy" , "retained" , "evicted" } {
39+ require .Equal (t , observability .Bool (candidate == state ), test .MetricSum (t , m .Registry , "discovery_source_state" , map [string ]string {"state" : candidate }))
40+ }
41+ wantLast := 0.0
42+ if ! last .IsZero () {
43+ wantLast = float64 (last .Unix ())
44+ }
45+ require .Equal (t , wantLast , test .MetricSum (t , m .Registry , "discovery_last_success_timestamp_seconds" , nil ))
46+ require .EqualValues (t , count , test .MetricSum (t , m .Registry , "discovery_cached_runners" , nil ))
47+ require .EqualValues (t , failures , test .MetricSum (t , m .Registry , "discovery_source_failures" , nil ))
48+ }
49+ assertMetrics ("uninitialized" , 0 , 0 , time.Time {})
50+ status = 500
51+ c .Refresh (t .Context (), false )
52+ assertMetrics ("uninitialized" , 0 , 1 , time.Time {})
53+ status = 200
3154 c .Refresh (t .Context (), false )
3255 original := c .Snapshots ()[0 ]
56+ assertMetrics ("healthy" , 1 , 0 , original .LastSuccess )
3357 for i , failure := range []struct {
3458 status int
3559 body string
@@ -40,22 +64,27 @@ func TestCollectorCacheLifecycle(t *testing.T) {
4064 c .Refresh (t .Context (), false )
4165 got := c .Snapshots ()[0 ]
4266 require .Equal (t , original .Entries , got .Entries )
67+ require .Equal (t , original .LastSuccess , got .LastSuccess )
68+ assertMetrics ("retained" , 1 , i + 1 , original .LastSuccess )
4369 require .Equal (t , original .QueriedAt , got .QueriedAt )
4470 require .Equal (t , i + 1 , got .Failures )
4571 }
4672 c .Refresh (t .Context (), false )
4773 require .Empty (t , c .Snapshots ())
74+ assertMetrics ("evicted" , 0 , 6 , original .LastSuccess )
4875 require .True (t , c .Initialized (), "eviction does not reset startup" )
4976 status , body = 200 , envelope (candidate ("1999-02-01T00:00:00Z" ))
5077 c .Refresh (t .Context (), false )
5178 require .Equal (t , "1999-02-01T00:00:00Z" , c .Snapshots ()[0 ].Entries [0 ].Runners [0 ].Observed )
5279 require .Zero (t , c .Snapshots ()[0 ].Failures )
80+ assertMetrics ("healthy" , 1 , 0 , c .Snapshots ()[0 ].LastSuccess )
5381 status = 500
5482 c .Refresh (t .Context (), false )
5583 status , body = 200 , "[]"
5684 c .Refresh (t .Context (), false )
5785 require .Len (t , c .Snapshots (), 1 , "empty success is still a usable source" )
5886 require .Empty (t , c .Snapshots ()[0 ].Entries )
87+ assertMetrics ("healthy" , 0 , 0 , c .Snapshots ()[0 ].LastSuccess )
5988 require .Zero (t , c .Snapshots ()[0 ].Failures )
6089}
6190
@@ -136,10 +165,10 @@ func TestCollectorFailureDiagnostics(t *testing.T) {
136165 name , endpoint , want string
137166 transport http.RoundTripper
138167 }{
139- {"grant" , "http://127.0.0.1/private-token" , "denied address " , policy .Client ().Transport },
140- {"DNS" , "https://missing.example/private-token" , "no such host " , policy .Client ().Transport },
141- {"timeout" , "https://source.example/private-token" , "context deadline exceeded " , collectorTransport (func (* http.Request ) (* http.Response , error ) { return nil , context .DeadlineExceeded })},
142- {"redirect" , "https://source.example/private-token" , "invalid discovery redirect URL " , collectorTransport (func (* http.Request ) (* http.Response , error ) {
168+ {"grant" , "http://127.0.0.1/private-token" , "denied_destination " , policy .Client ().Transport },
169+ {"DNS" , "https://missing.example/private-token" , "dns " , policy .Client ().Transport },
170+ {"timeout" , "https://source.example/private-token" , "timeout " , collectorTransport (func (* http.Request ) (* http.Response , error ) { return nil , context .DeadlineExceeded })},
171+ {"redirect" , "https://source.example/private-token" , "invalid_redirect " , collectorTransport (func (* http.Request ) (* http.Response , error ) {
143172 response := collectorResponse (302 , "" )
144173 response .Header = http.Header {"Location" : {"https://redirect.example/private-token/%zz" }}
145174 return response , nil
0 commit comments