Skip to content

Commit 17713df

Browse files
authored
Add heartbeat seq tracking (#68)
Stale heartbeats can arrive after an app is removed, making the app available for use again. Heartbeats now carry a strictly increasing seq number; invalid seq numbers are rejected and the heartbeat is not applied.
1 parent dad86b0 commit 17713df

4 files changed

Lines changed: 220 additions & 5 deletions

File tree

‎docs/http-api.md‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ See [orchestrator setup](orchestrator.md), [signer setup](signer.md), and
1010

1111
| Method and route | Purpose | Auth / envelope |
1212
| --- | --- | --- |
13-
| `POST /runners/heartbeat` | Register or reconcile dynamic runner | `Authorization` bootstrap credential on first call, runner-scoped heartbeat credential thereafter; JSON `runner_id`, `version`, and `apps` snapshot; response includes interval, TTL, optional new credential, app reconciliation and `o2r`. |
13+
| `POST /runners/heartbeat` | Register or reconcile dynamic runner | `Authorization` bootstrap credential on first call, runner-scoped heartbeat credential thereafter; JSON `runner_id`, `version`, `seq`, and `apps` snapshot; response includes interval, TTL, optional new credential, app reconciliation and `o2r`. |
1414
| `POST /runners/{runner_id}/unregister` | Remove dynamic runner | Scoped heartbeat credential. |
1515
| `GET /discovery` | Find available runners | JSON discovery entries with app, URL, mode, capacity and price. |
1616
| `POST /apps/{app_id}/session` | Reserve persistent session | JSON `session_id`, `app_url`, `control_url`; paid mode may first return a payment challenge. |
@@ -34,7 +34,11 @@ has `id`, `app`, `version`, and `runner_url`, plus `mode`, `proxy`, `capacity`,
3434
`version` is the runner SDK version; `apps[].version` is the application version.
3535
Capacity defaults to one; price units and currencies are trimmed and lowercased.
3636

37-
Dynamic runners are marked as ready in the heartbeat after registration.
37+
Runners send a strictly increasing uint64 `seq` after registration with
38+
each heartbeat. Successful responses echo the `seq`. Rejected `seq` values do
39+
not update runner state on the orchestrator. A fresh registration or loss of
40+
orchestrator state starts a new sequence. A reconnecting runner can resume from
41+
the `seq` returned in a 409 response.
3842

3943
App IDs are unique per node; names may repeat. Omit `id` to generate one, then
4044
retain the returned ID in later snapshots. Omitted apps finish existing work
@@ -43,6 +47,8 @@ See [public URLs](orchestrator.md#public-urls-and-tls) for `RunnerServiceURL`, a
4347
[sessions and proxies](orchestrator.md#sessions-and-proxies) for pricing, release
4448
behavior, and proxy routes.
4549

50+
Dynamic runners are marked as ready in the heartbeat after registration.
51+
4652
## Discovery HTTP
4753

4854
The response body follows the [discovery JSON Schema](discovery.schema.json).

‎orchestrator/http.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,8 +204,13 @@ func writeJSON(w http.ResponseWriter, status int, value any) (int, error) {
204204
return w.Write(append(body, '\n'))
205205
}
206206

207-
func fail(w http.ResponseWriter, status int, message string) {
208-
writeJSON(w, status, map[string]string{"error": message})
207+
func fail(w http.ResponseWriter, status int, message any) {
208+
body := map[string]any{"error": fmt.Sprint(message)}
209+
var conflict seqConflict
210+
if err, ok := message.(error); ok && errors.As(err, &conflict) {
211+
body["seq"] = uint64(conflict)
212+
}
213+
writeJSON(w, status, body)
209214
}
210215

211216
func failReservation(w http.ResponseWriter, r *http.Request, status int, err error) {
@@ -240,7 +245,7 @@ func (s *Server) heartbeat(w http.ResponseWriter, r *http.Request) {
240245
}
241246
resp, status, err := s.registry.Heartbeat(req, r.Header.Get("Authorization"))
242247
if err != nil {
243-
fail(w, status, err.Error())
248+
fail(w, status, err)
244249
return
245250
}
246251
if resp.HeartbeatSecret != "" || req.Apps != nil {

‎orchestrator/registry.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ type appResponse struct {
8181
}
8282

8383
type heartbeatRequest struct {
84+
Seq *uint64 `json:"seq,omitempty"`
8485
Apps []appRequest `json:"apps,omitzero"`
8586
RunnerID string `json:"runner_id"`
8687
RunnerURL string `json:"runner_url"`
@@ -97,6 +98,7 @@ type heartbeatRequest struct {
9798
}
9899

99100
type heartbeatResponse struct {
101+
Seq *uint64 `json:"seq,omitempty"`
100102
RunnerID string `json:"runner_id"`
101103
Orchestrator string `json:"orchestrator,omitempty"`
102104
HeartbeatInterval string `json:"heartbeat_interval"`
@@ -153,9 +155,14 @@ type runner struct {
153155
ID, Version, Credential string
154156
Last time.Time
155157
Static, Confirmed bool
158+
Seq *uint64
156159
Apps map[string]*app
157160
}
158161

162+
type seqConflict uint64
163+
164+
func (seqConflict) Error() string { return "seq must increase after owner opt-in" }
165+
159166
// appLifetime distinguishes registration instances without retaining their state.
160167
type appLifetime struct{ _ byte }
161168

@@ -445,6 +452,9 @@ func (r *Registry) Heartbeat(req heartbeatRequest, auth string) (heartbeatRespon
445452
if !equalSecret(auth, current.Credential) {
446453
return heartbeatResponse{}, http.StatusUnauthorized, errors.New("invalid runner credential")
447454
}
455+
if current.Seq != nil && (req.Seq == nil || *req.Seq <= *current.Seq) {
456+
return heartbeatResponse{}, http.StatusConflict, seqConflict(*current.Seq)
457+
}
448458
}
449459
if req.Apps == nil {
450460
requests[0].ID = current.ID
@@ -486,6 +496,11 @@ func (r *Registry) Heartbeat(req heartbeatRequest, auth string) (heartbeatRespon
486496
// Nothing above this point mutates the registry, including existing prices.
487497
current.Version, current.Last = req.Version, time.Now()
488498
current.Confirmed = confirm
499+
if confirm && req.Seq != nil {
500+
accepted := *req.Seq
501+
current.Seq = &accepted
502+
}
503+
resp.Seq = current.Seq
489504
r.runners[current.ID] = current
490505
resp.RunnerID, resp.Apps = current.ID, make([]appResponse, 0, retained)
491506
for i, entry := range requests {

‎orchestrator/seq_test.go‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
package orchestrator
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"net/http"
7+
"net/http/httptest"
8+
"strings"
9+
"testing"
10+
"time"
11+
12+
"github.com/stretchr/testify/require"
13+
)
14+
15+
func ownerSeq(n uint64) *uint64 { return &n }
16+
17+
func ownerHeartbeatHTTP(t *testing.T, server *Server, req heartbeatRequest, credential string) *httptest.ResponseRecorder {
18+
t.Helper()
19+
body := map[string]any{"runner_id": req.RunnerID, "version": req.Version, "apps": req.Apps}
20+
if req.Seq != nil {
21+
body["seq"] = *req.Seq
22+
}
23+
return heartbeatHTTP(t, server, body, credential)
24+
}
25+
26+
func TestOwnerSeqOrdering(t *testing.T) {
27+
registry, server := reviewServer(t, "http://127.0.0.1:1")
28+
defer server.Close()
29+
a := registeredApp("a")
30+
request := heartbeatRequest{RunnerID: "owner", Seq: ownerSeq(100), Apps: []appRequest{a}}
31+
initial := ownerHeartbeatHTTP(t, server, request, "bootstrap")
32+
require.Equal(t, 200, initial.Code, initial.Body.String())
33+
var owner heartbeatResponse
34+
require.NoError(t, json.Unmarshal(initial.Body.Bytes(), &owner))
35+
require.Nil(t, owner.Seq, "bootstrap cannot opt an issued credential into ordering")
36+
request.Seq = nil
37+
for range 2 {
38+
w := ownerHeartbeatHTTP(t, server, request, owner.HeartbeatSecret)
39+
require.Equal(t, 200, w.Code, w.Body.String())
40+
require.NotContains(t, w.Body.String(), `"seq"`)
41+
}
42+
lastLegacy := registry.runners["owner"].Last
43+
for _, seq := range []string{`-1`, `1.5`, `"1"`, `18446744073709551616`, `true`, `[]`} {
44+
req := httptest.NewRequest(http.MethodPost, "/runners/heartbeat", strings.NewReader(`{"runner_id":"owner","seq":`+seq+`,"apps":[]}`))
45+
req.Header.Set("Authorization", owner.HeartbeatSecret)
46+
w := httptest.NewRecorder()
47+
server.ServeHTTP(w, req)
48+
require.Equal(t, 400, w.Code, w.Body.String())
49+
require.Nil(t, registry.runners["owner"].Seq)
50+
require.Equal(t, lastLegacy, registry.runners["owner"].Last)
51+
require.Contains(t, registry.apps, "a")
52+
}
53+
request.Seq = ownerSeq(0)
54+
w := ownerHeartbeatHTTP(t, server, request, owner.HeartbeatSecret)
55+
require.Equal(t, 200, w.Code, w.Body.String())
56+
require.Contains(t, w.Body.String(), `"seq":0`, "zero enables ordering")
57+
request.Seq = nil
58+
w = ownerHeartbeatHTTP(t, server, request, owner.HeartbeatSecret)
59+
require.Equal(t, 409, w.Code, w.Body.String())
60+
request.Seq = ownerSeq(5)
61+
w = ownerHeartbeatHTTP(t, server, request, owner.HeartbeatSecret)
62+
require.Equal(t, 200, w.Code, w.Body.String())
63+
require.Contains(t, w.Body.String(), `"seq":5`)
64+
last := registry.runners["owner"].Last
65+
for _, seq := range []*uint64{nil, ownerSeq(0), ownerSeq(4), ownerSeq(5)} {
66+
request.Seq, request.Version = seq, "rejected"
67+
request.Apps[0].Status = "draining"
68+
w := ownerHeartbeatHTTP(t, server, request, owner.HeartbeatSecret)
69+
require.Equal(t, 409, w.Code, w.Body.String())
70+
require.JSONEq(t, `{"error":"seq must increase after owner opt-in","seq":5}`, w.Body.String())
71+
require.Equal(t, last, registry.runners["owner"].Last)
72+
require.Empty(t, registry.runners["owner"].Version)
73+
require.Equal(t, "ready", registry.apps["a"].Status)
74+
}
75+
other := appHeartbeat(t, registry, "other", "bootstrap", registeredApp("foreign"))
76+
request.Seq = ownerSeq(50)
77+
for _, credential := range []string{"", "invalid", "bootstrap", other.HeartbeatSecret} {
78+
w = ownerHeartbeatHTTP(t, server, request, credential)
79+
require.Equal(t, 401, w.Code, w.Body.String())
80+
require.EqualValues(t, 5, *registry.runners["owner"].Seq)
81+
require.Equal(t, last, registry.runners["owner"].Last)
82+
}
83+
request.Apps[0] = registeredApp("foreign")
84+
w = ownerHeartbeatHTTP(t, server, request, owner.HeartbeatSecret)
85+
require.Equal(t, 409, w.Code)
86+
require.JSONEq(t, `{"error":"app id belongs to another runner"}`, w.Body.String())
87+
require.EqualValues(t, 5, *registry.runners["owner"].Seq)
88+
require.Equal(t, last, registry.runners["owner"].Last)
89+
require.Equal(t, "ready", registry.apps["a"].Status)
90+
require.Same(t, registry.runners["owner"], registry.apps["a"].owner)
91+
require.Same(t, registry.runners["other"], registry.apps["foreign"].owner)
92+
request.Apps[0], request.Seq = a, ownerSeq(9)
93+
w = ownerHeartbeatHTTP(t, server, request, owner.HeartbeatSecret)
94+
require.Equal(t, 200, w.Code, w.Body.String())
95+
require.EqualValues(t, 9, *registry.runners["owner"].Seq, "sequence gaps are allowed")
96+
}
97+
98+
func TestHeartbeatSeqWithdrawalAfterRegistrarRestart(t *testing.T) {
99+
registry, server := reviewServer(t, "http://127.0.0.1:1")
100+
defer server.Close()
101+
a, b := registeredApp("a"), registeredApp("b")
102+
w := ownerHeartbeatHTTP(t, server, heartbeatRequest{RunnerID: "owner", Apps: []appRequest{a, b}}, "bootstrap")
103+
require.Equal(t, 200, w.Code, w.Body.String())
104+
var saved heartbeatResponse
105+
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &saved))
106+
ready := heartbeatRequest{RunnerID: saved.RunnerID, Apps: []appRequest{a, b}, Seq: ownerSeq(1)}
107+
w = ownerHeartbeatHTTP(t, server, ready, saved.HeartbeatSecret)
108+
require.Equal(t, 200, w.Code, w.Body.String())
109+
sidA, _, _, _, err := registry.Reserve("a")
110+
require.NoError(t, err)
111+
sidB, _, _, _, err := registry.Reserve("b")
112+
require.NoError(t, err)
113+
ctxA, _ := registry.sessionContext("a", sidA)
114+
ctxB, _ := registry.sessionContext("b", sidB)
115+
// A restarted registrar withdraws using its saved identity and a newer sequence.
116+
ready.Seq = ownerSeq(2)
117+
withdrawn := heartbeatRequest{RunnerID: saved.RunnerID, Apps: []appRequest{b}, Seq: ownerSeq(3)}
118+
w = ownerHeartbeatHTTP(t, server, withdrawn, saved.HeartbeatSecret)
119+
require.Equal(t, 200, w.Code, w.Body.String())
120+
var observed heartbeatResponse
121+
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &observed))
122+
require.Equal(t, []appResponse{{ID: "b", Status: "ready", SessionIDs: []string{sidB}}, {ID: "a", Status: "draining", SessionIDs: []string{sidA}}}, observed.Apps)
123+
last := registry.runners[saved.RunnerID].Last
124+
w = ownerHeartbeatHTTP(t, server, ready, saved.HeartbeatSecret)
125+
require.Equal(t, 409, w.Code, w.Body.String())
126+
require.True(t, registry.apps["a"].Removing)
127+
require.NoError(t, ctxA.Err(), "withdrawal preserves admitted work")
128+
require.NoError(t, ctxB.Err())
129+
_, _, _, status, err := registry.Reserve("a")
130+
require.Error(t, err)
131+
require.Equal(t, 404, status)
132+
// Normal completion removes the withdrawn registration and its session resources.
133+
registry.ReleaseBySession(sidA, "completed")
134+
require.ErrorIs(t, ctxA.Err(), context.Canceled)
135+
require.NotContains(t, registry.apps, "a")
136+
w = ownerHeartbeatHTTP(t, server, ready, saved.HeartbeatSecret)
137+
require.Equal(t, 409, w.Code)
138+
require.NotContains(t, registry.apps, "a", "a stale ready request cannot revive a completed registration")
139+
require.Equal(t, last, registry.runners[saved.RunnerID].Last)
140+
withdrawn.Seq = ownerSeq(5)
141+
w = ownerHeartbeatHTTP(t, server, withdrawn, saved.HeartbeatSecret)
142+
require.Equal(t, 200, w.Code, w.Body.String())
143+
require.NoError(t, json.Unmarshal(w.Body.Bytes(), &observed))
144+
require.Equal(t, []appResponse{{ID: "b", Status: "ready", SessionIDs: []string{sidB}}}, observed.Apps)
145+
require.NoError(t, ctxB.Err())
146+
}
147+
148+
func TestOwnerSeqScopeReset(t *testing.T) {
149+
for _, reset := range []string{"expired", "unregistered", "restart"} {
150+
t.Run(reset, func(t *testing.T) {
151+
registry, server := reviewServer(t, "http://127.0.0.1:1")
152+
defer func() { server.Close() }()
153+
request := heartbeatRequest{RunnerID: "owner", Apps: []appRequest{registeredApp("a")}}
154+
owner, _, err := registry.Heartbeat(request, "bootstrap")
155+
require.NoError(t, err)
156+
request.Seq = ownerSeq(8)
157+
_, _, err = registry.Heartbeat(request, owner.HeartbeatSecret)
158+
require.NoError(t, err)
159+
sid, _, _, _, err := registry.Reserve("a")
160+
require.NoError(t, err)
161+
ctx, _ := registry.sessionContext("a", sid)
162+
switch reset {
163+
case "expired":
164+
registry.runners["owner"].Last = time.Now().Add(-2 * time.Minute)
165+
registry.Expire()
166+
case "unregistered":
167+
_, err = registry.Unregister("owner", owner.HeartbeatSecret)
168+
require.NoError(t, err)
169+
case "restart":
170+
server.Close()
171+
registry, server = reviewServer(t, "http://127.0.0.1:1")
172+
}
173+
require.Error(t, ctx.Err())
174+
w := ownerHeartbeatHTTP(t, server, request, owner.HeartbeatSecret)
175+
require.Equal(t, 401, w.Code)
176+
replacement, _, err := registry.Heartbeat(request, "bootstrap")
177+
require.NoError(t, err)
178+
require.NotEqual(t, owner.HeartbeatSecret, replacement.HeartbeatSecret)
179+
require.Nil(t, replacement.Seq)
180+
w = ownerHeartbeatHTTP(t, server, request, owner.HeartbeatSecret)
181+
require.Equal(t, 401, w.Code)
182+
request.Seq = ownerSeq(1)
183+
w = ownerHeartbeatHTTP(t, server, request, replacement.HeartbeatSecret)
184+
require.Equal(t, 200, w.Code, w.Body.String())
185+
require.Empty(t, registry.sessions)
186+
require.EqualValues(t, 1, *registry.runners["owner"].Seq)
187+
})
188+
}
189+
}

0 commit comments

Comments
 (0)