Skip to content

Commit c6d7411

Browse files
authored
Share canonical payment block and round observations (#39)
Depends on #38. Merge after that PR. Paid signer and orchestrator processes share an immutable, canonical block/round observation across requests and readiness. Warm metadata reads use memory; independent redemption and finality reads continue through the direct chain adapter. Separates the protocol L1 clock, chain block identity, current round, and initialized round. Freshness checks distinguish contact from head/L1 progress, preserve valid state across verified polling gaps, and invalidate proven reorgs before further reads. Admission and retained-snapshot consumers revalidate locally at their commit boundaries. Corrects the existing readiness description because its previous expiry wording becomes inaccurate. Source attribution is retained. Validation: full race suite, vet, Staticcheck, formatting, and diff checks passed; regressions cover stale providers, reorgs, L1 rollback, delayed signing/commit, and readiness recovery. Closes #31.
2 parents 4662540 + a813d7f commit c6d7411

23 files changed

Lines changed: 926 additions & 64 deletions

‎docs/metrics.md‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,12 @@ Other useful starting conditions:
169169

170170
`/healthz` checks liveness. Paid orchestrator `/readyz` requires writable payment
171171
storage, chain access, an active recipient, valid pricing, and worker completion.
172-
Checks run at heartbeat cadence even with empty queues. Dependency observations
173-
and worker completion expire after `max(60 seconds, 3 × HeartbeatInterval)`;
172+
Checks run at heartbeat cadence even with empty queues. Payment chain metadata
173+
refreshes every five seconds. An observation is refused at thirty seconds old,
174+
if the head timestamp is stale, or after thirty seconds without protocol L1
175+
progress. A successful RPC response with an unchanged head does not renew its
176+
age. One provider cannot prove absolute L1 lag. Other dependency checks and
177+
worker completion expire after `max(60 seconds, 3 × HeartbeatInterval)`;
174178
queue stalls and individual ticket errors remain separate signals.
175179
Off-chain orchestrator/discovery readiness reports listener health.
176180
Signer readiness requires chain access, positive deposit/total reserve, no

‎eth/payment.go‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"errors"
66
"math/big"
7+
"time"
78

89
"github.com/ethereum/go-ethereum"
910
"github.com/ethereum/go-ethereum/accounts/abi/bind"
@@ -15,7 +16,10 @@ import (
1516

1617
// PaymentChain adapts the retained Livepeer contracts to pm's narrow chain
1718
// interface. Ethereum contract reads and redemption stay in eth.
18-
type PaymentChain struct{ Contracts *Contracts }
19+
type PaymentChain struct {
20+
Contracts *Contracts
21+
Observer *PaymentObserver // Nil for direct transaction/redemption reads.
22+
}
1923

2024
func (c PaymentChain) IsActive(ctx context.Context, recipient ethcommon.Address) (bool, error) {
2125
snapshot, err := c.Snapshot(ctx)
@@ -26,6 +30,9 @@ func (c PaymentChain) IsActive(ctx context.Context, recipient ethcommon.Address)
2630
}
2731

2832
func (c PaymentChain) IsActiveAt(ctx context.Context, recipient ethcommon.Address, snapshot ChainSnapshot) (bool, error) {
33+
if err := snapshot.Validate(); err != nil {
34+
return false, err
35+
}
2936
if snapshot.blockReference == nil {
3037
return false, errors.New("canonical snapshot required")
3138
}
@@ -37,7 +44,11 @@ func (c PaymentChain) IsActiveAt(ctx context.Context, recipient ethcommon.Addres
3744
if err != nil {
3845
return false, err
3946
}
40-
return bonding.IsActiveTranscoder(&bind.CallOpts{Context: ctx}, recipient)
47+
active, err := bonding.IsActiveTranscoder(&bind.CallOpts{Context: ctx}, recipient)
48+
if err == nil {
49+
err = snapshot.Validate()
50+
}
51+
return active, err
4152
}
4253

4354
// Receipt reports a submitted redemption without retrying or resubmitting it.
@@ -73,14 +84,29 @@ type ChainSnapshot struct {
7384
Block *big.Int // Protocol L1 block, including on Arbitrum.
7485
Round *big.Int // Last initialized round.
7586
RoundHash ethcommon.Hash
87+
CurrentRound *big.Int // Protocol round; may exceed the initialized Round.
7688
blockReference map[string]any
89+
observer *PaymentObserver
90+
generation uint64
91+
observedAt time.Time
92+
blockTime time.Time
7793
}
7894

7995
func (c PaymentChain) Snapshot(ctx context.Context) (ChainSnapshot, error) {
96+
if c.Observer != nil {
97+
return c.Observer.Snapshot(ctx)
98+
}
99+
at := time.Now()
80100
header, err := c.Contracts.RPC.header(ctx, "latest")
81101
if err != nil {
82102
return ChainSnapshot{}, err
83103
}
104+
snapshot, err := c.snapshotAt(ctx, header, false)
105+
snapshot.observedAt = at
106+
return snapshot, err
107+
}
108+
109+
func (c PaymentChain) snapshotAt(ctx context.Context, header *miniHeader, currentRound bool) (ChainSnapshot, error) {
84110
if header.Hash == (ethcommon.Hash{}) || header.Number == nil {
85111
return ChainSnapshot{}, errors.New("invalid chain header")
86112
}
@@ -114,7 +140,17 @@ func (c PaymentChain) Snapshot(ctx context.Context) (ChainSnapshot, error) {
114140
if hash == ([32]byte{}) {
115141
return ChainSnapshot{}, errors.New("invalid round block hash")
116142
}
117-
return ChainSnapshot{Block: block, Round: round, RoundHash: ethcommon.Hash(hash), blockReference: ref}, nil
143+
snapshot := ChainSnapshot{Block: block, Round: round, RoundHash: ethcommon.Hash(hash), blockReference: ref}
144+
if currentRound {
145+
snapshot.CurrentRound, err = rounds.CurrentRound(opts)
146+
if err != nil {
147+
return ChainSnapshot{}, err
148+
}
149+
if snapshot.CurrentRound.Cmp(round) < 0 {
150+
return ChainSnapshot{}, errors.New("current round precedes initialized round")
151+
}
152+
}
153+
return snapshot, nil
118154
}
119155

120156
// SenderInfo is a coherent view of a sender's collateral and the amount of
@@ -153,7 +189,7 @@ func (c PaymentChain) SenderInfo(ctx context.Context, sender, recipient ethcommo
153189
return SenderInfo{}, err
154190
}
155191
}
156-
return SenderInfo{Snapshot: snapshot, Deposit: info.Sender.Deposit, Reserve: reserve, WithdrawRound: info.Sender.WithdrawRound}, nil
192+
return SenderInfo{Snapshot: snapshot, Deposit: info.Sender.Deposit, Reserve: reserve, WithdrawRound: info.Sender.WithdrawRound}, snapshot.Validate()
157193
}
158194

159195
// The redemption tuple follows Yondon Fu's go-livepeer/eth/client_ticketbroker.go

‎eth/payment_observer.go‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
// Adapted from livepeer/go-livepeer eth/watchers/timewatcher.go and
2+
// eth/blockwatch/block_watcher.go at 597f882cb9f4d42ad8430b0f592afd5929af4a6b:
3+
// Nico Vergauwen (b4605f705, 3aec3aa16), Yondon Fu (0acd88bbd), and
4+
// Rafał Leszko (4b6ede31f, 8c7ec0155). Immutable pinned observations and age
5+
// refusal also adapt Mike Zupper's BlueClawNetwork/livepeer-node eth/chainview.go
6+
// (2d95d40b8, 74f41b11f). Copyright (c) 2017 Livepeer; MIT notice in LICENSE.
7+
package eth
8+
9+
import (
10+
"context"
11+
"errors"
12+
"math/big"
13+
"sync"
14+
"time"
15+
16+
"github.com/ethereum/go-ethereum/common"
17+
"github.com/ethereum/go-ethereum/common/hexutil"
18+
)
19+
20+
var ErrObservationUnavailable = errors.New("chain observation unavailable")
21+
22+
// PaymentObserver publishes complete canonical observations. Gaps and reorgs
23+
// rebuild state at the new hash instead of replaying uncertain event history.
24+
type PaymentObserver struct {
25+
chain PaymentChain
26+
interval, maxAge time.Duration
27+
now func() time.Time
28+
mu sync.RWMutex
29+
current ChainSnapshot
30+
header *miniHeader
31+
generation uint64
32+
requireL1 bool
33+
invalid, stopped bool
34+
contact, progress time.Time
35+
l1Progress time.Time
36+
l1HighWater *big.Int
37+
}
38+
39+
func NewPaymentObserver(chain PaymentChain, requireL1 bool) *PaymentObserver {
40+
chain.Observer = nil
41+
return &PaymentObserver{chain: chain, interval: 5 * time.Second, maxAge: 30 * time.Second, now: time.Now, requireL1: requireL1 || chain.Contracts.RPC.requireL1}
42+
}
43+
44+
// Run has one service-owned caller. Every RPC inherits the service lifetime.
45+
func (o *PaymentObserver) Run(ctx context.Context) {
46+
defer func() { o.mu.Lock(); o.stopped = true; o.mu.Unlock() }()
47+
ticker := time.NewTicker(o.interval)
48+
defer ticker.Stop()
49+
for {
50+
read, cancel := context.WithTimeout(ctx, o.maxAge)
51+
_ = o.refresh(read)
52+
cancel()
53+
select {
54+
case <-ctx.Done():
55+
return
56+
case <-ticker.C:
57+
}
58+
}
59+
}
60+
61+
// ObserverStatus separates successful contact from accepted state and progress.
62+
// A single provider cannot establish an absolute bound on protocol L1 lag.
63+
type ObserverStatus struct {
64+
ObservedAt, ContactAt, HeadProgressAt, L1ProgressAt time.Time
65+
Healthy bool
66+
}
67+
68+
func (o *PaymentObserver) Status() ObserverStatus {
69+
o.mu.RLock()
70+
defer o.mu.RUnlock()
71+
return ObserverStatus{o.current.observedAt, o.contact, o.progress, o.l1Progress, o.validate(o.current) == nil}
72+
}
73+
74+
func (o *PaymentObserver) Snapshot(ctx context.Context) (ChainSnapshot, error) {
75+
if err := ctx.Err(); err != nil {
76+
return ChainSnapshot{}, err
77+
}
78+
o.mu.RLock()
79+
defer o.mu.RUnlock()
80+
if err := o.validate(o.current); err != nil {
81+
return ChainSnapshot{}, err
82+
}
83+
s := o.current
84+
s.Block, s.Round, s.CurrentRound = new(big.Int).Set(s.Block), new(big.Int).Set(s.Round), new(big.Int).Set(s.CurrentRound)
85+
return s, nil
86+
}
87+
88+
func (s ChainSnapshot) ObservedAt() time.Time {
89+
if s.observedAt.IsZero() { // Injected direct chains have no retained observation.
90+
return time.Now()
91+
}
92+
return s.observedAt
93+
}
94+
95+
// Validate also rejects snapshots retained across an invalidation or shutdown.
96+
// Direct reads and injected test chains retain their existing behavior.
97+
func (s ChainSnapshot) Validate() error {
98+
if s.observer == nil {
99+
return nil
100+
}
101+
s.observer.mu.RLock()
102+
defer s.observer.mu.RUnlock()
103+
return s.observer.validate(s)
104+
}
105+
106+
func (o *PaymentObserver) validate(s ChainSnapshot) error {
107+
if o.stopped || o.invalid || s.generation != o.generation || o.expired(s, o.l1Progress) {
108+
return ErrObservationUnavailable
109+
}
110+
return nil
111+
}
112+
113+
func (o *PaymentObserver) expired(s ChainSnapshot, l1Progress time.Time) bool {
114+
now := o.now()
115+
return s.observedAt.IsZero() || now.Sub(s.observedAt) >= o.maxAge || now.Sub(s.blockTime) >= o.maxAge || now.Sub(l1Progress) >= o.maxAge
116+
}
117+
118+
// Only Run calls refresh. Previous/current identity is the extension point for
119+
// complete hash-pinned logs or reset; no partial delta authorizes publication.
120+
func (o *PaymentObserver) refresh(ctx context.Context) error {
121+
at := o.now()
122+
head, err := o.chain.Contracts.RPC.header(ctx, "latest")
123+
if err != nil {
124+
return err
125+
}
126+
o.mu.Lock()
127+
o.contact = o.now()
128+
prior := o.header
129+
gap := false
130+
if head.Hash == (common.Hash{}) || head.Number == nil {
131+
o.mu.Unlock()
132+
return ErrObservationUnavailable
133+
}
134+
if prior != nil {
135+
distance := new(big.Int).Sub((*big.Int)(head.Number), (*big.Int)(prior.Number))
136+
if distance.Sign() < 0 {
137+
o.invalid = true
138+
o.generation++
139+
o.mu.Unlock()
140+
return ErrObservationUnavailable
141+
}
142+
gap = head.Hash != prior.Hash && distance.Cmp(big.NewInt(1)) > 0
143+
if head.Hash != prior.Hash && !gap && (distance.Sign() == 0 || head.ParentHash != prior.Hash) {
144+
o.invalid = true
145+
o.generation++
146+
}
147+
}
148+
// Usable identity conflicts invalidate even when other header metadata is
149+
// missing or stale. Such metadata must not restore a rejected observation.
150+
if head.Timestamp == nil || (o.requireL1 && head.L1BlockNumber == nil) {
151+
o.mu.Unlock()
152+
return ErrObservationUnavailable
153+
}
154+
blockTime := time.Unix(int64(*head.Timestamp), 0)
155+
if at.Sub(blockTime) >= o.maxAge || blockTime.After(at.Add(o.maxAge)) {
156+
o.mu.Unlock()
157+
return ErrObservationUnavailable
158+
}
159+
if prior != nil && head.Hash == prior.Hash {
160+
// A lagging backend may return N, N-1, then N again. Restore the
161+
// immutable observation without renewing any freshness deadline.
162+
o.current.generation, o.invalid = o.generation, false
163+
err := o.validate(o.current)
164+
o.mu.Unlock()
165+
return err
166+
}
167+
o.mu.Unlock()
168+
if gap {
169+
// A polling gap does not itself orphan the retained observation. Prove
170+
// its anchor before keeping it available during the pinned backfill.
171+
anchor, err := o.chain.Contracts.RPC.header(ctx, hexutil.EncodeBig((*big.Int)(prior.Number)))
172+
if err != nil || anchor.Hash != prior.Hash || anchor.Number == nil || (*big.Int)(anchor.Number).Cmp((*big.Int)(prior.Number)) != 0 {
173+
o.mu.Lock()
174+
o.invalid = true
175+
o.generation++
176+
o.mu.Unlock()
177+
}
178+
}
179+
s, err := o.chain.snapshotAt(ctx, head, true)
180+
if err != nil {
181+
return err
182+
}
183+
if err := ctx.Err(); err != nil {
184+
return err
185+
}
186+
o.mu.Lock()
187+
defer o.mu.Unlock()
188+
progress, l1Progress, highWater := o.progress, o.l1Progress, o.l1HighWater
189+
if prior == nil || (*big.Int)(head.Number).Cmp((*big.Int)(prior.Number)) > 0 {
190+
progress = at
191+
}
192+
if highWater == nil || s.Block.Cmp(highWater) > 0 {
193+
l1Progress, highWater = at, s.Block
194+
}
195+
s.observer, s.generation, s.observedAt, s.blockTime = o, o.generation, at, blockTime
196+
if o.stopped || o.expired(s, l1Progress) {
197+
return ErrObservationUnavailable
198+
}
199+
o.current, o.header, o.invalid = s, head, false
200+
o.progress, o.l1Progress, o.l1HighWater = progress, l1Progress, highWater
201+
o.requireL1 = o.requireL1 || head.L1BlockNumber != nil
202+
return nil
203+
}

0 commit comments

Comments
 (0)