Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions src/httpware/middleware/resilience/circuit_breaker.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,8 +324,9 @@ def _reject(self, request: httpx2.Request, now: float, retry_after: float | None
raise _RejectionError(retry_after, self.take_events())

def _record_closed(self, request: httpx2.Request, *, failed: bool) -> None:
if self._rate_mode:
self._record_outcome(request, failed=failed)
window = self._window
if window is not None:
self._record_outcome(request, window, failed=failed)
elif failed:
self._consecutive_failures += 1
if self._consecutive_failures >= self._failure_threshold:
Expand All @@ -337,8 +338,9 @@ def _record_probe_success(self, request: httpx2.Request) -> None:
self._move_to(CircuitState.CLOSED)
self._consecutive_failures = 0
self._consecutive_successes = 0
if self._rate_mode:
self._window.clear() # ty: ignore[unresolved-attribute]
window = self._window
if window is not None:
window.clear()
attributes = {"rejected": self._episode_rejections}
self._queue(request, "circuit.closed", logging.INFO, "circuit closed — service recovered", attributes)

Expand Down Expand Up @@ -377,11 +379,10 @@ def _open_rate(self, request: httpx2.Request, *, total: int, failures: int) -> N
},
)

def _record_outcome(self, request: httpx2.Request, *, failed: bool) -> None:
# Only reached in rate mode, where _window and _failure_rate_threshold are non-None.
def _record_outcome(self, request: httpx2.Request, window: _RollingWindow, *, failed: bool) -> None:
now = self._now()
self._window.record(now, failed=failed) # ty: ignore[unresolved-attribute]
total, failures = self._window.totals(now) # ty: ignore[unresolved-attribute]
window.record(now, failed=failed)
total, failures = window.totals(now)
threshold = self._failure_rate_threshold
if threshold is not None and total >= self._minimum_calls and failures / total >= threshold:
self._open_rate(request, total=total, failures=failures)
Expand Down
8 changes: 4 additions & 4 deletions tests/test_bulkhead.py
Original file line number Diff line number Diff line change
Expand Up @@ -482,10 +482,10 @@ async def test_bulkhead_rejected_emits_observability_event(caplog: pytest.LogCap
assert len(rejected_records) == 1
record = rejected_records[0]
assert record.levelno == logging.WARNING
assert record.max_concurrent == 1 # ty: ignore[unresolved-attribute]
assert record.acquire_timeout == 0.0 # ty: ignore[unresolved-attribute]
assert record.method == "GET" # ty: ignore[unresolved-attribute]
assert "example.test/y" in record.url # ty: ignore[unresolved-attribute]
assert record.max_concurrent == 1 # ty: ignore[missing-attribute]
assert record.acquire_timeout == 0.0 # ty: ignore[missing-attribute]
assert record.method == "GET" # ty: ignore[missing-attribute]
assert "example.test/y" in record.url # ty: ignore[missing-attribute]


# ───── Single-event-loop guard ──────────────────────────────────────────────
Expand Down
40 changes: 20 additions & 20 deletions tests/test_circuit_breaker.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,13 +133,13 @@ async def test_open_emits_opened_event_and_rejects(caplog: pytest.LogCaptureFixt
opened = [r for r in records if "opened" in r.message]
rejected = [r for r in records if "rejecting" in r.message]
assert len(opened) == 1
assert opened[0].event == "circuit.opened" # ty: ignore[unresolved-attribute]
assert opened[0].failure_threshold == 2 # noqa: PLR2004 # ty: ignore[unresolved-attribute]
assert opened[0].failures == 2 # noqa: PLR2004 # ty: ignore[unresolved-attribute]
assert opened[0].event == "circuit.opened" # ty: ignore[missing-attribute]
assert opened[0].failure_threshold == 2 # noqa: PLR2004 # ty: ignore[missing-attribute]
assert opened[0].failures == 2 # noqa: PLR2004 # ty: ignore[missing-attribute]
assert len(rejected) == 1
assert rejected[0].event == "circuit.rejected" # ty: ignore[unresolved-attribute]
assert rejected[0].retry_after is not None # ty: ignore[unresolved-attribute]
assert rejected[0].method == "GET" # ty: ignore[unresolved-attribute]
assert rejected[0].event == "circuit.rejected" # ty: ignore[missing-attribute]
assert rejected[0].retry_after is not None # ty: ignore[missing-attribute]
assert rejected[0].method == "GET" # ty: ignore[missing-attribute]


async def test_success_resets_failure_streak() -> None:
Expand Down Expand Up @@ -342,8 +342,8 @@ async def test_probe_failure_reopens_circuit(caplog: pytest.LogCaptureFixture) -
r
for r in caplog.records
if r.name == "httpware.circuit_breaker"
and r.event == "circuit.opened" # ty: ignore[unresolved-attribute]
and r.failures == 1 # ty: ignore[unresolved-attribute]
and r.event == "circuit.opened" # ty: ignore[missing-attribute]
and r.failures == 1 # ty: ignore[missing-attribute]
]
assert len(reopen_records) == 1

Expand Down Expand Up @@ -664,11 +664,11 @@ async def test_rate_mode_open_event_carries_rate_attributes(caplog: pytest.LogCa
await client.get("https://example.test/x")
for _ in range(2):
await client.get("https://example.test/x")
opened = [r for r in caplog.records if r.event == "circuit.opened"] # ty: ignore[unresolved-attribute]
opened = [r for r in caplog.records if r.event == "circuit.opened"] # ty: ignore[missing-attribute]
assert opened, "expected a circuit.opened record"
rec = opened[-1]
assert rec.failure_rate_threshold == 0.5 # noqa: PLR2004 # ty: ignore[unresolved-attribute]
assert rec.observed_calls >= 4 # noqa: PLR2004 # ty: ignore[unresolved-attribute]
assert rec.failure_rate_threshold == 0.5 # noqa: PLR2004 # ty: ignore[missing-attribute]
assert rec.observed_calls >= 4 # noqa: PLR2004 # ty: ignore[missing-attribute]
assert hasattr(rec, "failure_rate")
assert not hasattr(rec, "failure_threshold") # classic attribute absent in rate mode

Expand Down Expand Up @@ -732,10 +732,10 @@ async def test_rate_mode_probe_failure_reopens_with_classic_attributes(caplog: p
with pytest.raises(InternalServerError):
await probe.get("https://example.test/x")
assert breaker.state is CircuitState.OPEN # the failed probe re-opened it
opened = [r for r in caplog.records if r.event == "circuit.opened"] # ty: ignore[unresolved-attribute]
opened = [r for r in caplog.records if r.event == "circuit.opened"] # ty: ignore[missing-attribute]
assert opened
rec = opened[-1]
assert rec.failures == 1 # ty: ignore[unresolved-attribute] # classic probe-failure shape
assert rec.failures == 1 # ty: ignore[missing-attribute] # classic probe-failure shape
assert hasattr(rec, "failure_threshold")
assert not hasattr(rec, "failure_rate") # NOT the rate shape

Expand Down Expand Up @@ -929,8 +929,8 @@ async def _unreachable(request: httpx2.Request) -> httpx2.Response: # pragma: n
clock.advance(step)
with pytest.raises(CircuitOpenError):
await breaker(httpx2.Request("GET", "https://example.test/x"), _unreachable)
rejected = [r for r in caplog.records if r.event == "circuit.rejected"] # ty: ignore[unresolved-attribute]
assert [r.suppressed for r in rejected] == [0, 4] # ty: ignore[unresolved-attribute]
rejected = [r for r in caplog.records if r.event == "circuit.rejected"] # ty: ignore[missing-attribute]
assert [r.suppressed for r in rejected] == [0, 4] # ty: ignore[missing-attribute]


# ── rejection accounting runs per outage episode ──
Expand Down Expand Up @@ -962,7 +962,7 @@ async def test_first_rejection_of_a_new_outage_is_emitted(caplog: pytest.LogCapt
await breaker(httpx2.Request("GET", "https://example.test/x"), _ok)
await _open_now(breaker)
await _reject(breaker)
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 0] # ty: ignore[unresolved-attribute]
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 0] # ty: ignore[missing-attribute]


async def test_closed_event_reports_every_rejection_of_the_outage(caplog: pytest.LogCaptureFixture) -> None:
Expand All @@ -978,8 +978,8 @@ async def test_closed_event_reports_every_rejection_of_the_outage(caplog: pytest
await _reject(breaker)
clock.advance(5.0)
await breaker(httpx2.Request("GET", "https://example.test/x"), _ok)
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 1] # ty: ignore[unresolved-attribute]
assert [r.rejected for r in _events(caplog, "circuit.closed")] == [4] # ty: ignore[unresolved-attribute]
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 1] # ty: ignore[missing-attribute]
assert [r.rejected for r in _events(caplog, "circuit.closed")] == [4] # ty: ignore[missing-attribute]


async def test_suppressed_never_counts_a_previous_outage(caplog: pytest.LogCaptureFixture) -> None:
Expand All @@ -993,7 +993,7 @@ async def test_suppressed_never_counts_a_previous_outage(caplog: pytest.LogCaptu
await breaker(httpx2.Request("GET", "https://example.test/x"), _ok)
await _open_now(breaker)
await _reject(breaker)
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 0] # ty: ignore[unresolved-attribute]
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 0] # ty: ignore[missing-attribute]


async def test_events_of_one_call_arrive_in_transition_order(caplog: pytest.LogCaptureFixture) -> None:
Expand All @@ -1004,7 +1004,7 @@ async def test_events_of_one_call_arrive_in_transition_order(caplog: pytest.LogC
await _reject(breaker)
clock.advance(1.0)
await breaker(httpx2.Request("GET", "https://example.test/x"), _ok)
assert [r.event for r in caplog.records] == [ # ty: ignore[unresolved-attribute]
assert [r.event for r in caplog.records] == [ # ty: ignore[missing-attribute]
"circuit.opened",
"circuit.rejected",
"circuit.half_open",
Expand Down
32 changes: 16 additions & 16 deletions tests/test_circuit_breaker_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,13 @@ def test_open_emits_opened_event_and_rejects(caplog: pytest.LogCaptureFixture) -
opened = [r for r in records if "opened" in r.message]
rejected = [r for r in records if "rejecting" in r.message]
assert len(opened) == 1
assert opened[0].event == "circuit.opened" # ty: ignore[unresolved-attribute]
assert opened[0].failure_threshold == 2 # noqa: PLR2004 # ty: ignore[unresolved-attribute]
assert opened[0].failures == 2 # noqa: PLR2004 # ty: ignore[unresolved-attribute]
assert opened[0].event == "circuit.opened" # ty: ignore[missing-attribute]
assert opened[0].failure_threshold == 2 # noqa: PLR2004 # ty: ignore[missing-attribute]
assert opened[0].failures == 2 # noqa: PLR2004 # ty: ignore[missing-attribute]
assert len(rejected) == 1
assert rejected[0].event == "circuit.rejected" # ty: ignore[unresolved-attribute]
assert rejected[0].retry_after is not None # ty: ignore[unresolved-attribute]
assert rejected[0].method == "GET" # ty: ignore[unresolved-attribute]
assert rejected[0].event == "circuit.rejected" # ty: ignore[missing-attribute]
assert rejected[0].retry_after is not None # ty: ignore[missing-attribute]
assert rejected[0].method == "GET" # ty: ignore[missing-attribute]


def test_success_resets_failure_streak() -> None:
Expand Down Expand Up @@ -247,8 +247,8 @@ def test_probe_failure_reopens_circuit(caplog: pytest.LogCaptureFixture) -> None
r
for r in caplog.records
if r.name == "httpware.circuit_breaker"
and r.event == "circuit.opened" # ty: ignore[unresolved-attribute]
and r.failures == 1 # ty: ignore[unresolved-attribute]
and r.event == "circuit.opened" # ty: ignore[missing-attribute]
and r.failures == 1 # ty: ignore[missing-attribute]
]
assert len(reopen_records) == 1

Expand Down Expand Up @@ -587,10 +587,10 @@ def test_rate_mode_probe_failure_reopens_with_classic_attributes(caplog: pytest.
with pytest.raises(InternalServerError):
probe.get("https://example.test/x")
assert breaker.state is CircuitState.OPEN # the failed probe re-opened it
opened = [r for r in caplog.records if r.event == "circuit.opened"] # ty: ignore[unresolved-attribute]
opened = [r for r in caplog.records if r.event == "circuit.opened"] # ty: ignore[missing-attribute]
assert opened
rec = opened[-1]
assert rec.failures == 1 # ty: ignore[unresolved-attribute] # classic probe-failure shape
assert rec.failures == 1 # ty: ignore[missing-attribute] # classic probe-failure shape
assert hasattr(rec, "failure_threshold")
assert not hasattr(rec, "failure_rate") # NOT the rate shape

Expand Down Expand Up @@ -797,8 +797,8 @@ def test_rejected_event_is_emitted_at_most_once_per_second(caplog: pytest.LogCap
clock.advance(step)
with pytest.raises(CircuitOpenError):
breaker(httpx2.Request("GET", "https://example.test/x"), _unreachable)
rejected = [r for r in caplog.records if r.event == "circuit.rejected"] # ty: ignore[unresolved-attribute]
assert [r.suppressed for r in rejected] == [0, 4] # ty: ignore[unresolved-attribute]
rejected = [r for r in caplog.records if r.event == "circuit.rejected"] # ty: ignore[missing-attribute]
assert [r.suppressed for r in rejected] == [0, 4] # ty: ignore[missing-attribute]


class _LockProbe(logging.Handler):
Expand Down Expand Up @@ -946,7 +946,7 @@ def test_first_rejection_of_a_new_outage_is_emitted(caplog: pytest.LogCaptureFix
breaker(httpx2.Request("GET", "https://example.test/x"), _ok)
_open_now(breaker)
_reject(breaker)
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 0] # ty: ignore[unresolved-attribute]
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 0] # ty: ignore[missing-attribute]


def test_closed_event_reports_every_rejection_of_the_outage(caplog: pytest.LogCaptureFixture) -> None:
Expand All @@ -962,8 +962,8 @@ def test_closed_event_reports_every_rejection_of_the_outage(caplog: pytest.LogCa
_reject(breaker)
clock.advance(5.0)
breaker(httpx2.Request("GET", "https://example.test/x"), _ok)
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 1] # ty: ignore[unresolved-attribute]
assert [r.rejected for r in _events(caplog, "circuit.closed")] == [4] # ty: ignore[unresolved-attribute]
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 1] # ty: ignore[missing-attribute]
assert [r.rejected for r in _events(caplog, "circuit.closed")] == [4] # ty: ignore[missing-attribute]


def test_suppressed_never_counts_a_previous_outage(caplog: pytest.LogCaptureFixture) -> None:
Expand All @@ -977,7 +977,7 @@ def test_suppressed_never_counts_a_previous_outage(caplog: pytest.LogCaptureFixt
breaker(httpx2.Request("GET", "https://example.test/x"), _ok)
_open_now(breaker)
_reject(breaker)
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 0] # ty: ignore[unresolved-attribute]
assert [r.suppressed for r in _events(caplog, "circuit.rejected")] == [0, 0] # ty: ignore[missing-attribute]


def test_locked_admission_into_a_circuit_closed_since_the_unlocked_read_gets_the_closed_ticket() -> None:
Expand Down
8 changes: 4 additions & 4 deletions tests/test_keyed_circuit_breaker.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,8 @@ async def test_events_carry_the_circuit_key(caplog: pytest.LogCaptureFixture) ->
with pytest.raises(CircuitOpenError):
await client.get("https://a.test/x")
records = [r for r in caplog.records if r.name == "httpware.circuit_breaker"]
assert [r.event for r in records] == ["circuit.opened", "circuit.rejected"] # ty: ignore[unresolved-attribute]
assert all(r.circuit_key == "https://a.test" for r in records) # ty: ignore[unresolved-attribute]
assert [r.event for r in records] == ["circuit.opened", "circuit.rejected"] # ty: ignore[missing-attribute]
assert all(r.circuit_key == "https://a.test" for r in records) # ty: ignore[missing-attribute]


async def test_circuit_key_is_the_normalized_origin_without_userinfo(caplog: pytest.LogCaptureFixture) -> None:
Expand All @@ -201,7 +201,7 @@ async def test_circuit_key_is_the_normalized_origin_without_userinfo(caplog: pyt
with caplog.at_level(logging.WARNING, logger="httpware.circuit_breaker"):
await _fail_n(client, "https://user:secret@A.test:443/x?q=1", 1)
await _fail_n(client, "http://a.test:8080/x", 1)
keys = [r.circuit_key for r in caplog.records if r.name == "httpware.circuit_breaker"] # ty: ignore[unresolved-attribute]
keys = [r.circuit_key for r in caplog.records if r.name == "httpware.circuit_breaker"] # ty: ignore[missing-attribute]
assert keys == ["https://a.test", "http://a.test:8080"]


Expand All @@ -211,7 +211,7 @@ async def test_circuit_key_of_a_custom_key_is_its_str(caplog: pytest.LogCaptureF
async with _client(handler, breaker=breaker) as client:
with caplog.at_level(logging.WARNING, logger="httpware.circuit_breaker"):
await _fail_n(client, "https://a.test/x", 1)
keys = [r.circuit_key for r in caplog.records if r.name == "httpware.circuit_breaker"] # ty: ignore[unresolved-attribute]
keys = [r.circuit_key for r in caplog.records if r.name == "httpware.circuit_breaker"] # ty: ignore[missing-attribute]
assert keys == ["('a.test', 1)"]


Expand Down
8 changes: 4 additions & 4 deletions tests/test_keyed_circuit_breaker_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,8 @@ def test_events_carry_the_circuit_key(caplog: pytest.LogCaptureFixture) -> None:
with pytest.raises(CircuitOpenError):
client.get("https://a.test/x")
records = [r for r in caplog.records if r.name == "httpware.circuit_breaker"]
assert [r.event for r in records] == ["circuit.opened", "circuit.rejected"] # ty: ignore[unresolved-attribute]
assert all(r.circuit_key == "https://a.test" for r in records) # ty: ignore[unresolved-attribute]
assert [r.event for r in records] == ["circuit.opened", "circuit.rejected"] # ty: ignore[missing-attribute]
assert all(r.circuit_key == "https://a.test" for r in records) # ty: ignore[missing-attribute]


def test_circuit_key_is_the_normalized_origin_without_userinfo(caplog: pytest.LogCaptureFixture) -> None:
Expand All @@ -202,7 +202,7 @@ def test_circuit_key_is_the_normalized_origin_without_userinfo(caplog: pytest.Lo
):
_fail_n(client, "https://user:secret@A.test:443/x?q=1", 1)
_fail_n(client, "http://a.test:8080/x", 1)
keys = [r.circuit_key for r in caplog.records if r.name == "httpware.circuit_breaker"] # ty: ignore[unresolved-attribute]
keys = [r.circuit_key for r in caplog.records if r.name == "httpware.circuit_breaker"] # ty: ignore[missing-attribute]
assert keys == ["https://a.test", "http://a.test:8080"]


Expand All @@ -214,7 +214,7 @@ def test_circuit_key_of_a_custom_key_is_its_str(caplog: pytest.LogCaptureFixture
caplog.at_level(logging.WARNING, logger="httpware.circuit_breaker"),
):
_fail_n(client, "https://a.test/x", 1)
keys = [r.circuit_key for r in caplog.records if r.name == "httpware.circuit_breaker"] # ty: ignore[unresolved-attribute]
keys = [r.circuit_key for r in caplog.records if r.name == "httpware.circuit_breaker"] # ty: ignore[missing-attribute]
assert keys == ["('a.test', 1)"]


Expand Down
Loading
Loading