From 13a5bea83e44d6ee72d6947b1d4bc4c9748456c0 Mon Sep 17 00:00:00 2001 From: Matthew Keeler Date: Thu, 12 Nov 2026 16:58:26 -0500 Subject: [PATCH 1/2] fix: Allow closing the client from a flag change or data store status listener Listeners run on a data system thread, and stopping the FDv2 data system joined every one of its threads, including that one. The join raised, so the data store and big segment store were left open. --- ldclient/impl/datasystem/fdv2.py | 7 ++- .../impl/datasystem/test_fdv2_datasystem.py | 36 +++++++++++++ ldclient/testing/test_ldclient_listeners.py | 50 ++++++++++++++++++- 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/ldclient/impl/datasystem/fdv2.py b/ldclient/impl/datasystem/fdv2.py index ae8281f2..63ff50f8 100644 --- a/ldclient/impl/datasystem/fdv2.py +++ b/ldclient/impl/datasystem/fdv2.py @@ -1,6 +1,6 @@ import time from queue import Queue -from threading import Event, Thread +from threading import Event, Thread, current_thread from typing import Any, Callable, Dict, List, Mapping, Optional from ldclient.config import Config, DataSourceBuilder, DataSystemConfig @@ -367,8 +367,11 @@ def stop(self): except Exception as e: log.error("Error stopping active data source: %s", e) - # Wait for all threads to complete + # Wait for all threads to complete. A listener can call stop on one of these threads, + # and a thread cannot join itself. That thread sees the stop event and exits on its own. for thread in self._threads: + if thread is current_thread(): + continue if thread.is_alive(): thread.join(timeout=5.0) # 5 second timeout if thread.is_alive(): diff --git a/ldclient/testing/impl/datasystem/test_fdv2_datasystem.py b/ldclient/testing/impl/datasystem/test_fdv2_datasystem.py index 6cc98a85..d195f009 100644 --- a/ldclient/testing/impl/datasystem/test_fdv2_datasystem.py +++ b/ldclient/testing/impl/datasystem/test_fdv2_datasystem.py @@ -153,6 +153,42 @@ def listener(flag_change: FlagChange): assert len(changes) == 0 +def test_fdv2_can_be_stopped_from_a_flag_change_listener(): + td = TestDataV2.data_source() + td.update(td.flag("feature-flag").on(True)) + data_system_config = DataSystemConfig( + initializers=None, + synchronizers=[td.builder], + ) + + set_on_ready = Event() + fdv2 = FDv2(Config(sdk_key="dummy"), data_system_config) + + stopped = Event() + errors: List[Exception] = [] + + def listener(flag_change: FlagChange): + try: + fdv2.stop() + except Exception as e: + errors.append(e) + finally: + stopped.set() + + fdv2.flag_change_listeners.add(listener) + + fdv2.start(set_on_ready) + assert set_on_ready.wait(1), "Data system did not become ready in time" + + td.update(td.flag("feature-flag").on(False)) + assert stopped.wait(1), "Flag change listener was not called in time" + assert errors == [] + + for thread in fdv2._threads: + thread.join(timeout=1) + assert not thread.is_alive(), "Thread %s did not exit" % thread.name + + def test_fdv2_data_availability_is_refreshed_with_data(): td = TestDataV2.data_source() data_system_config = DataSystemConfig( diff --git a/ldclient/testing/test_ldclient_listeners.py b/ldclient/testing/test_ldclient_listeners.py index 2ad697db..edfe3698 100644 --- a/ldclient/testing/test_ldclient_listeners.py +++ b/ldclient/testing/test_ldclient_listeners.py @@ -1,8 +1,14 @@ +import time from queue import Queue +from threading import Event +from typing import List +from ldclient import datasystem from ldclient.client import Config, LDClient from ldclient.config import BigSegmentsConfig -from ldclient.interfaces import DataSourceState +from ldclient.feature_store import InMemoryFeatureStore +from ldclient.integrations.test_datav2 import TestDataV2 +from ldclient.interfaces import DataSourceState, DataStoreMode from ldclient.testing.http_util import start_server from ldclient.testing.mock_components import MockBigSegmentStore from ldclient.testing.stub_util import ( @@ -61,3 +67,45 @@ def test_data_source_status_updates(): with LDClient(config=config) as client: assert client.data_source_status_provider.status.state == DataSourceState.VALID assert client.data_source_status_provider.status.error is None + + +def test_client_can_be_closed_from_a_flag_change_listener(): + td = TestDataV2.data_source() + td.update(td.flag('flag').on(True)) + store = _CloseTrackingStore() + data_system = datasystem.custom().synchronizers(td.builder).data_store(store, DataStoreMode.READ_WRITE).build() + client = LDClient(Config('SDK_KEY', send_events=False, datasystem_config=data_system), start_wait=5) + assert client.is_initialized() + + closed = Event() + errors: List[Exception] = [] + + def listener(flag_change): + try: + client.close() + except Exception as e: + errors.append(e) + finally: + closed.set() + + client.flag_tracker.add_listener(listener) + td.update(td.flag('flag').on(False)) + + assert closed.wait(5), "Flag change listener was not called in time" + assert errors == [] + assert store.closed + + # The listener's own thread sets the OFF status after the listener returns. + deadline = time.monotonic() + 5 + while client.data_source_status_provider.status.state != DataSourceState.OFF and time.monotonic() < deadline: + time.sleep(0.01) + assert client.data_source_status_provider.status.state == DataSourceState.OFF + + +class _CloseTrackingStore(InMemoryFeatureStore): + def __init__(self): + super().__init__() + self.closed = False + + def close(self): + self.closed = True From d17de3dbc7e9a003f599ac299f5ba34c546f8035 Mon Sep 17 00:00:00 2001 From: Matthew Keeler Date: Fri, 13 Nov 2026 11:23:42 -0500 Subject: [PATCH 2/2] fix: Allow closing the client from an override or file data source listener A flag change listener called by an override update held the override sink lock, and close waited for that lock forever. Listeners are now notified after the lock is released. The FDv1 file data source with watchdog called listeners on the observer thread, and stop joined that thread. Stop now skips the join on that thread. A close from a listener during Store.apply let apply write the change set to the closed persistent store. The store now stops persisting after close. --- ldclient/impl/datasystem/store.py | 8 ++- .../integrations/files/file_data_source.py | 6 +- ldclient/impl/overrides/sink.py | 21 ++++-- ldclient/testing/impl/test_overrides.py | 23 +++++++ ldclient/testing/test_file_data_source.py | 33 +++++++++ ldclient/testing/test_ldclient_listeners.py | 67 +++++++++++++++++-- 6 files changed, 144 insertions(+), 14 deletions(-) diff --git a/ldclient/impl/datasystem/store.py b/ldclient/impl/datasystem/store.py index 3904312b..f32101e2 100644 --- a/ldclient/impl/datasystem/store.py +++ b/ldclient/impl/datasystem/store.py @@ -440,6 +440,10 @@ def __init__( # store. Set on each successful apply from its persist flag. self._persist = False + # True after close. A listener can close the store while apply notifies it. The + # change set that apply is processing is then not written to the closed persistent store. + self._closed = False + def with_persistence( self, persistent_store: FeatureStore, @@ -518,7 +522,8 @@ def apply(self, change_set: ChangeSet, persist: bool) -> None: def _should_persist(self) -> bool: """Returns whether data should be persisted to the persistent store.""" return ( - self._persist + not self._closed + and self._persist and self._persistent_store is not None and self._persistent_store_writable ) @@ -566,6 +571,7 @@ def __mapping(data: Dict[str, ModelEntity]) -> Dict[str, Dict[str, Any]]: def close(self) -> None: """Close the store and any persistent store if configured.""" with self._lock: + self._closed = True if self._persistent_store is not None: try: # Most FeatureStore implementations don't have close methods diff --git a/ldclient/impl/integrations/files/file_data_source.py b/ldclient/impl/integrations/files/file_data_source.py index 81fb38d3..9de95fec 100644 --- a/ldclient/impl/integrations/files/file_data_source.py +++ b/ldclient/impl/integrations/files/file_data_source.py @@ -2,6 +2,7 @@ import os import time import traceback +from threading import current_thread from typing import Optional from ldclient.impl.repeating_task import RepeatingTask @@ -174,7 +175,10 @@ def on_any_event(self, event): def stop(self): self._observer.stop() - self._observer.join() + # A listener can stop the client on the observer thread, and a thread cannot join + # itself. That thread sees the stop request and exits on its own. + if self._observer is not current_thread(): + self._observer.join() # Watch for changes to data files by polling their modification times. This is used if auto-update is # on but the watchdog package is not installed. diff --git a/ldclient/impl/overrides/sink.py b/ldclient/impl/overrides/sink.py index 76c71a5a..9f0dfb4e 100644 --- a/ldclient/impl/overrides/sink.py +++ b/ldclient/impl/overrides/sink.py @@ -24,8 +24,9 @@ class OverrideSinkImpl(OverrideSink): """ Applies override layer replacements supplied by an override source, then notifies flag - change listeners of every flag whose merged-view evaluation may have changed. Calls are - serialized, so overlapping updates from a source cannot interleave. + change listeners of every flag whose merged-view evaluation may have changed. Layer updates + are serialized, so overlapping updates from a source cannot interleave. Listeners are + notified after the lock is released, so a listener can close the client. """ def __init__( @@ -56,12 +57,13 @@ def set_overrides(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None log.debug("Ignoring an override snapshot supplied after the override source was closed") return try: - self._apply(flags, segments) + affected = self._apply(flags, segments) except DeletedItemError as e: # The layer rejected the whole snapshot before changing anything, so nothing is # notified. The error also reaches the source, as for an invalid definition. log.error("Rejected an override snapshot; the overrides in effect stay: %s", e) raise + self._notify_all(affected) def close(self) -> None: """ @@ -79,13 +81,14 @@ def close_and_clear(self) -> None: """ with self._lock: self._closed = True - self._apply({}, {}) + affected = self._apply({}, {}) + self._notify_all(affected) - def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: - # Called with _lock held. + def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> Set[str]: + # Called with _lock held. Returns the keys of the affected flags. if not self._has_listeners(): self._layer.set_all(flags, segments) - return + return set() previous, current = self._layer.set_all(flags, segments) old_merged = snapshot_merged_view(self._base, previous) @@ -93,6 +96,10 @@ def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: affected = compute_affected_flags(previous, current, old_merged, new_merged) if len(affected) > 0: log.debug("Override update affected %d flag(s)", len(affected)) + return affected + + def _notify_all(self, affected: Set[str]) -> None: + # Called without _lock held. A listener can then close the client, which closes this sink. for key in sorted(affected): self._notify(key) diff --git a/ldclient/testing/impl/test_overrides.py b/ldclient/testing/impl/test_overrides.py index 1681d8e4..3cdfb7ea 100644 --- a/ldclient/testing/impl/test_overrides.py +++ b/ldclient/testing/impl/test_overrides.py @@ -3,6 +3,7 @@ snapshots and notifies flag change listeners. """ import logging +import threading from typing import Any, Dict, List, Optional import pytest @@ -502,6 +503,28 @@ def test_sink_close_and_clear_clears_the_layer_and_ignores_later_snapshots(): assert f.take_notified() == [] +def test_sink_listener_can_close_the_sink(): + # A listener that closes the client closes the sink. The sink must not hold its lock while + # it notifies, or the close waits forever. + f = SinkFixture(FakeBaseStore()) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + f.take_notified() + + def notify(key: str): + f.notified.append(key) + f.sink.close() + + f.sink._notify = notify + worker = threading.Thread(target=lambda: f.sink.set_overrides({'flag1': flag('flag1', 2)}, {}), daemon=True) + worker.start() + worker.join(5) + + assert not worker.is_alive(), "Closing the sink from a listener did not return" + assert f.take_notified() == ['flag1'] + f.sink.set_overrides({'flag2': flag('flag2')}, {}) + assert f.take_notified() == [] + + def test_sink_notifies_directly_changed_keys_when_base_read_fails(): base = FakeBaseStore(flags={'flag1': flag('flag1')}) base.fail_all = True diff --git a/ldclient/testing/test_file_data_source.py b/ldclient/testing/test_file_data_source.py index b8e3fb0b..f1a76231 100644 --- a/ldclient/testing/test_file_data_source.py +++ b/ldclient/testing/test_file_data_source.py @@ -285,6 +285,39 @@ def test_reloads_modified_file_in_polling_mode(): do_auto_update_test({'auto_update': True, 'force_polling': True, 'poll_interval': 0.1}) +@pytest.mark.parametrize('options', [ + pytest.param({'auto_update': True}, id='watchdog'), + pytest.param({'auto_update': True, 'force_polling': True, 'poll_interval': 0.1}, id='polling'), +]) +def test_client_can_be_closed_from_a_flag_change_listener_on_reload(options): + path = make_temp_file(flag_only_json) + factory = Files.new_data_source(paths=path, **options) + client = LDClient(config=Config('SDK_KEY', update_processor_class=factory, send_events=False)) + try: + closed = threading.Event() + errors: List[Exception] = [] + + def listener(_): + if closed.is_set(): + return + try: + client.close() + except Exception as e: + errors.append(e) + finally: + closed.set() + + client.flag_tracker.add_listener(listener) + time.sleep(0.5) + replace_file(path, segment_only_json) + + assert closed.wait(20), "Flag change listener was not called in time" + assert errors == [] + finally: + client.close() + os.remove(path) + + def test_evaluates_full_flag_with_client_as_expected(): path = make_temp_file(all_properties_json) try: diff --git a/ldclient/testing/test_ldclient_listeners.py b/ldclient/testing/test_ldclient_listeners.py index edfe3698..36963704 100644 --- a/ldclient/testing/test_ldclient_listeners.py +++ b/ldclient/testing/test_ldclient_listeners.py @@ -1,7 +1,7 @@ import time from queue import Queue from threading import Event -from typing import List +from typing import Any, Callable, List, Tuple from ldclient import datasystem from ldclient.client import Config, LDClient @@ -73,14 +73,44 @@ def test_client_can_be_closed_from_a_flag_change_listener(): td = TestDataV2.data_source() td.update(td.flag('flag').on(True)) store = _CloseTrackingStore() + client = _make_client(td, store) + try: + closed, errors = _close_from_listener(client, client.flag_tracker.add_listener) + td.update(td.flag('flag').on(False)) + _assert_closed_from_listener(client, store, closed, errors) + finally: + client.close() + + +def test_client_can_be_closed_from_a_data_store_status_listener(): + td = TestDataV2.data_source() + td.update(td.flag('flag').on(True)) + store = _CloseTrackingStore() + client = _make_client(td, store) + try: + closed, errors = _close_from_listener(client, client.data_store_status_provider.add_listener) + # A failed write reports the store as unavailable on the thread that applies the update. + store.fail_writes = True + td.update(td.flag('flag').on(False)) + _assert_closed_from_listener(client, store, closed, errors) + finally: + client.close() + + +def _make_client(td: TestDataV2, store: '_CloseTrackingStore') -> LDClient: data_system = datasystem.custom().synchronizers(td.builder).data_store(store, DataStoreMode.READ_WRITE).build() client = LDClient(Config('SDK_KEY', send_events=False, datasystem_config=data_system), start_wait=5) assert client.is_initialized() + return client + +def _close_from_listener(client: LDClient, add_listener: Callable[[Callable[[Any], None]], None]) -> Tuple[Event, List[Exception]]: closed = Event() errors: List[Exception] = [] - def listener(flag_change): + def listener(_): + if closed.is_set(): + return try: client.close() except Exception as e: @@ -88,10 +118,12 @@ def listener(flag_change): finally: closed.set() - client.flag_tracker.add_listener(listener) - td.update(td.flag('flag').on(False)) + add_listener(listener) + return closed, errors + - assert closed.wait(5), "Flag change listener was not called in time" +def _assert_closed_from_listener(client: LDClient, store: '_CloseTrackingStore', closed: Event, errors: List[Exception]): + assert closed.wait(5), "Listener was not called in time" assert errors == [] assert store.closed @@ -101,11 +133,36 @@ def listener(flag_change): time.sleep(0.01) assert client.data_source_status_provider.status.state == DataSourceState.OFF + # The update that called the listener is not written to the closed store. + assert store.writes_after_close == [] + class _CloseTrackingStore(InMemoryFeatureStore): def __init__(self): super().__init__() self.closed = False + self.fail_writes = False + self.writes_after_close: List[str] = [] + + def init(self, all_data): + self._record_write('init') + super().init(all_data) + + def upsert(self, kind, item): + self._record_write('upsert') + super().upsert(kind, item) + + def _record_write(self, operation: str): + if self.closed: + self.writes_after_close.append(operation) + if self.fail_writes: + raise Exception("simulated store failure") + + def is_monitoring_enabled(self) -> bool: + return True + + def is_available(self) -> bool: + return not self.fail_writes def close(self): self.closed = True