Skip to content

Commit 2de42ef

Browse files
committed
Add Kafka
1 parent 0cc9d72 commit 2de42ef

24 files changed

Lines changed: 1869 additions & 323 deletions

‎README.md‎

Lines changed: 64 additions & 275 deletions
Large diffs are not rendered by default.

‎cmd/livepeer/testdata/cli/signer-help.golden‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,13 @@ Usage:
44
livepeer-signer [flags]
55

66
Flags:
7+
-k, --kafka-broker string One Kafka bootstrap address (TLS by default; kafka:// selects plaintext) (env: LIVEPEER_SIGNER_KAFKA_BROKER, required)
8+
--kafka-topic string Kafka topic for signing events (env: LIVEPEER_SIGNER_KAFKA_TOPIC, required)
9+
--kafka-auth-method string SASL authentication method when credentials are supplied (env: LIVEPEER_SIGNER_KAFKA_AUTH_METHOD) (default "scram-sha-512")
10+
--kafka-username-file string (env: LIVEPEER_SIGNER_KAFKA_USERNAME_FILE)
11+
--kafka-password-file string (env: LIVEPEER_SIGNER_KAFKA_PASSWORD_FILE)
12+
--kafka-outbox-db string Durable signer event SQLite path (env: LIVEPEER_SIGNER_KAFKA_OUTBOX_DB) (default "signer-events.sqlite")
13+
--kafka-outbox-max-bytes int Maximum pending event payload bytes (env: LIVEPEER_SIGNER_KAFKA_OUTBOX_MAX_BYTES) (default 268435456)
714
-a, --auth-webhook-headers-file string (env: LIVEPEER_SIGNER_AUTH_WEBHOOK_HEADERS_FILE)
815
--auth-webhook-file string (env: LIVEPEER_SIGNER_AUTH_WEBHOOK_FILE)
916
-m, --max-ticket-ev string Maximum expected ticket value in wei (env: LIVEPEER_SIGNER_MAX_TICKET_EV) (default "3000000000000")
@@ -26,6 +33,8 @@ Flags:
2633
-v, --version version for livepeer-signer
2734

2835
Environment Variables:
36+
LIVEPEER_SIGNER_KAFKA_USERNAME
37+
LIVEPEER_SIGNER_KAFKA_PASSWORD
2938
LIVEPEER_SIGNER_AUTH_WEBHOOK
3039
LIVEPEER_SIGNER_AUTH_WEBHOOK_HEADERS
3140
LIVEPEER_SIGNER_RPC_URL

‎configs/chain/config.example.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ RPCURLFile = "/run/secrets/livepeer-chain-rpc-url"
44
# Controller = "0xD8E8328501E9645d16Cf49539efC04f734606ee4" # Arbitrum mainnet default
55
# MaxFeePerGas = "1000000000" # Optional ceiling, wei per gas
66
# KeyFile = "/run/secrets/livepeer-chain-key"
7+
# KeyFile contains a raw Ethereum key: 64 hex digits, optional 0x prefix,
8+
# owner-only permissions. See docs/configuration.md and docs/chain.md.
79

810
# Action inputs and output controls must be provided on the command line.
911
# Example dry run:

‎configs/orchestrator/config.example.toml‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ service_url = "http://127.0.0.1:8935"
44
# runner_service_url = "http://127.0.0.1:8935" # Runner-facing callbacks and trickle; defaults to service_url.
55
bootstrap_secret_file = "/run/secrets/livepeer-runner-bootstrap"
66
runner_config = "/etc/livepeer/runners.toml"
7+
# Use either registration method or both. Omit runner_config for dynamic-only
8+
# operation; omit bootstrap_secret_file for static-only operation.
79
heartbeat_interval = "5s"
810
heartbeat_ttl = "30s"
911
runner_grants = []
@@ -18,10 +20,11 @@ behind_tls = false
1820
# tls_cert_file = "/etc/livepeer/tls/fullchain.pem"
1921
# tls_key_file = "/run/secrets/livepeer-tls-key.pem"
2022

21-
# To enable paid runners, configure every field in this block. Prices sent by
22-
# runners are USD per hour or per fixed request; wei_per_usd is an operator
23-
# supplied fixed exchange rate. Alternatively configure eth_usd_feed alone;
24-
# choose an ETH/USD feed on the same payment chain and verify its address.
23+
# Paid operation requires the database, key, RPC, chain ID, Controller, face
24+
# value, winning probability, and exactly one conversion source below.
25+
# Runner prices are USD per hour or per fixed request. Choose wei_per_usd
26+
# for a fixed conversion OR eth_usd_feed for an ETH/USD feed on the payment
27+
# chain. The fee ceiling and price_max_age are optional. See docs/orchestrator.md.
2528
# payment_db = "/var/lib/livepeer/orchestrator-payment.sqlite"
2629
# payment_key_file = "/run/secrets/livepeer-orchestrator-key"
2730
# payment_rpc_url_file = "/run/secrets/livepeer-payment-rpc-url"
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Run from the repository root. See README.md for the local HTTP runner.
2+
listen = "127.0.0.1:8935"
3+
metrics_listen = "127.0.0.1:8936"
4+
service_url = "http://127.0.0.1:8935"
5+
runner_config = "configs/orchestrator/runners.local.example.toml"
6+
runner_grants = ["127.0.0.1:9000"]
7+
health_grants = ["127.0.0.1:9000"]
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[[runners]]
2+
id = "local-runner"
3+
runner_url = "http://127.0.0.1:9000"
4+
app = "example"
5+
mode = "single-shot"
6+
status = "ready"
7+
capacity = 2
8+
health_url = "http://127.0.0.1:9000/healthz"
9+
healthy_status_code = 200

‎configs/signer/config.example.toml‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
# Signer TOML keys use the implicit Go field names; CLI flags use kebab-case.
1+
# Signer TOML uses the names shown below; flags use kebab-case.
2+
# See docs/signer.md and docs/configuration.md.
23
Listen = "127.0.0.1:8937"
34
MetricsListen = "127.0.0.1:8938"
45
KeyFile = "/run/secrets/livepeer-signer-key"
6+
# Raw Ethereum key: 64 hex digits, optional 0x prefix, owner-only permissions.
57
Orchestrators = []
8+
# Add orchestrator discovery URLs to make runners discoverable through the signer.
69

710
# On-chain configuration is required, including for identity and discovery.
811
# The RPC URL is an exact-byte secret file (omit a trailing newline).
@@ -26,19 +29,40 @@ ETHUSDMaxAge = "2h"
2629
# The signer retries every 30 seconds and resumes when it observes a fresh rate.
2730
# Identity signing and discovery remain available while the feed is stale.
2831

29-
# Exposure limits are independent of the requested work's maxPrice.
32+
# Expected value (EV) limits are in wei and independent of work's maxPrice.
3033
MaxTicketEV = "3000000000000"
3134
MaxBatchEV = "20000000000000"
3235
DepositMultiplier = 1
3336

3437
# Run alongside a clearinghouse for authorization and cumulative spending limits;
35-
# see README.md for the clearinghouse-batteries accounting integration requirements.
38+
# see docs/signer.md for webhook and Kafka accounting integration.
3639
# Signed state is a bearer credential. Authorization caching is generally discouraged;
3740
# the webhook should omit expiry or return zero to authorize every payment.
38-
# Tickets are generated and signed before authorization to identify the actual payment.
41+
# The webhook receives proposed payment state; data is returned only after approval.
3942
# Both webhook values are secret files.
4043
# Header syntax matches go-livepeer: Authorization: Bearer credential,X-Key: key
4144
# Quote an entire entry using CSV syntax if its value contains commas:
4245
# "X-List: a,b",Authorization: Bearer credential
4346
# AuthWebhookFile = "/run/secrets/livepeer-signer-auth-url"
4447
# AuthWebhookHeadersFile = "/run/secrets/livepeer-signer-auth-headers"
48+
49+
# Optional durable accounting events. Omit all Kafka settings to disable.
50+
# Supplying any Kafka setting requires Broker and Topic.
51+
# Provide one bootstrap host[:port]; Kafka discovers other brokers.
52+
# OutboxDB defaults to signer-events.sqlite in the working directory.
53+
# For an override, create the parent directory first; keep one outbox per replica.
54+
# TLS with system trust roots is the default when the scheme is omitted.
55+
# kafka:// explicitly uses plaintext transport; kafkas:// explicitly uses TLS.
56+
# Credentials must be paired and enable SASL. AuthMethod is a strict enum:
57+
# plain, scram-sha-256, scram-sha-512 (default).
58+
# Omitted ports use signer conventions: plaintext 9092, TLS-only 9093,
59+
# SASL/SCRAM 9096, SASL/PLAIN 9094. An explicit port always overrides these.
60+
# Provider listeners can differ: e.g. MSK private TLS-only uses 9094.
61+
# [Kafka]
62+
# Broker = "broker.example.com"
63+
# Topic = "livepeer-signing"
64+
# AuthMethod = "scram-sha-512"
65+
# UsernameFile = "/run/secrets/kafka-username"
66+
# PasswordFile = "/run/secrets/kafka-password"
67+
# OutboxDB = "/var/lib/livepeer/signer-events.sqlite"
68+
# OutboxMaxBytes = 268435456 # Pending JSON payloads; default 256 MiB.

‎eth/contracts/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,5 @@ git -C /path/to/go-livepeer show REVISION:eth/contracts/bondingManager.go > eth/
3232

3333
Application policy lives in the parent `eth` package: canonical snapshots,
3434
transaction planning, hints, fee limits, and saved transaction identities.
35-
Whole generated files retain unused methods; their presence does not add CLI
36-
commands or authorize broader protocol behavior.
35+
Generated files include unused upstream methods. The supported CLI operations
36+
are listed in [chain commands](../../docs/chain.md).

‎go.mod‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ require (
88
github.com/gorilla/websocket v1.5.3
99
github.com/holiman/uint256 v1.3.2
1010
github.com/j0sh/boa v0.0.3-0.20260930225304-334f1793b0a2
11+
github.com/j0sh/minikafka v0.0.0-20261001174935-0bc8dcedd322
12+
github.com/segmentio/kafka-go v0.4.51
1113
github.com/spf13/cobra v1.10.2
1214
github.com/stretchr/testify v1.11.1
1315
google.golang.org/protobuf v1.36.12
@@ -34,23 +36,30 @@ require (
3436
github.com/go-ole/go-ole v1.3.0 // indirect
3537
github.com/google/uuid v1.6.0 // indirect
3638
github.com/inconshreveable/mousetrap v1.1.0 // indirect
39+
github.com/klauspost/compress v1.18.7 // indirect
3740
github.com/mattn/go-isatty v0.0.20 // indirect
3841
github.com/ncruces/go-strftime v0.1.9 // indirect
42+
github.com/pierrec/lz4/v4 v4.1.26 // indirect
3943
github.com/pmezard/go-difflib v1.0.0 // indirect
4044
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
4145
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
4246
github.com/spf13/pflag v1.0.10 // indirect
4347
github.com/supranational/blst v0.3.16 // indirect
4448
github.com/tklauser/go-sysconf v0.3.12 // indirect
4549
github.com/tklauser/numcpus v0.6.1 // indirect
50+
github.com/twmb/franz-go/pkg/kmsg v1.13.1 // indirect
51+
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
52+
github.com/xdg-go/scram v1.2.0 // indirect
53+
github.com/xdg-go/stringprep v1.0.4 // indirect
4654
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
4755
go.opentelemetry.io/otel v1.41.0 // indirect
4856
go.opentelemetry.io/otel/metric v1.41.0 // indirect
4957
go.opentelemetry.io/otel/trace v1.41.0 // indirect
5058
golang.org/x/crypto v0.48.0 // indirect
5159
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
52-
golang.org/x/sync v0.19.0 // indirect
60+
golang.org/x/sync v0.21.0 // indirect
5361
golang.org/x/sys v0.44.0 // indirect
62+
golang.org/x/text v0.39.0 // indirect
5463
gopkg.in/yaml.v3 v3.0.1 // indirect
5564
modernc.org/libc v1.66.10 // indirect
5665
modernc.org/mathutil v1.7.1 // indirect

0 commit comments

Comments
 (0)