diff --git a/ldclient/impl/datasystem/fdv2.py b/ldclient/impl/datasystem/fdv2.py index ae8281f2..63ff50f8 100644 --- a/ldclient/impl/datasystem/fdv2.py +++ b/ldclient/impl/datasystem/fdv2.py @@ -1,6 +1,6 @@ import time from queue import Queue -from threading import Event, Thread +from threading import Event, Thread, current_thread from typing import Any, Callable, Dict, List, Mapping, Optional from ldclient.config import Config, DataSourceBuilder, DataSystemConfig @@ -367,8 +367,11 @@ def stop(self): except Exception as e: log.error("Error stopping active data source: %s", e) - # Wait for all threads to complete + # Wait for all threads to complete. A listener can call stop on one of these threads, + # and a thread cannot join itself. That thread sees the stop event and exits on its own. for thread in self._threads: + if thread is current_thread(): + continue if thread.is_alive(): thread.join(timeout=5.0) # 5 second timeout if thread.is_alive(): diff --git a/ldclient/impl/datasystem/store.py b/ldclient/impl/datasystem/store.py index 3904312b..f32101e2 100644 --- a/ldclient/impl/datasystem/store.py +++ b/ldclient/impl/datasystem/store.py @@ -440,6 +440,10 @@ def __init__( # store. Set on each successful apply from its persist flag. self._persist = False + # True after close. A listener can close the store while apply notifies it. The + # change set that apply is processing is then not written to the closed persistent store. + self._closed = False + def with_persistence( self, persistent_store: FeatureStore, @@ -518,7 +522,8 @@ def apply(self, change_set: ChangeSet, persist: bool) -> None: def _should_persist(self) -> bool: """Returns whether data should be persisted to the persistent store.""" return ( - self._persist + not self._closed + and self._persist and self._persistent_store is not None and self._persistent_store_writable ) @@ -566,6 +571,7 @@ def __mapping(data: Dict[str, ModelEntity]) -> Dict[str, Dict[str, Any]]: def close(self) -> None: """Close the store and any persistent store if configured.""" with self._lock: + self._closed = True if self._persistent_store is not None: try: # Most FeatureStore implementations don't have close methods diff --git a/ldclient/impl/integrations/files/file_data_source.py b/ldclient/impl/integrations/files/file_data_source.py index 81fb38d3..9de95fec 100644 --- a/ldclient/impl/integrations/files/file_data_source.py +++ b/ldclient/impl/integrations/files/file_data_source.py @@ -2,6 +2,7 @@ import os import time import traceback +from threading import current_thread from typing import Optional from ldclient.impl.repeating_task import RepeatingTask @@ -174,7 +175,10 @@ def on_any_event(self, event): def stop(self): self._observer.stop() - self._observer.join() + # A listener can stop the client on the observer thread, and a thread cannot join + # itself. That thread sees the stop request and exits on its own. + if self._observer is not current_thread(): + self._observer.join() # Watch for changes to data files by polling their modification times. This is used if auto-update is # on but the watchdog package is not installed. diff --git a/ldclient/impl/overrides/sink.py b/ldclient/impl/overrides/sink.py index 76c71a5a..9f0dfb4e 100644 --- a/ldclient/impl/overrides/sink.py +++ b/ldclient/impl/overrides/sink.py @@ -24,8 +24,9 @@ class OverrideSinkImpl(OverrideSink): """ Applies override layer replacements supplied by an override source, then notifies flag - change listeners of every flag whose merged-view evaluation may have changed. Calls are - serialized, so overlapping updates from a source cannot interleave. + change listeners of every flag whose merged-view evaluation may have changed. Layer updates + are serialized, so overlapping updates from a source cannot interleave. Listeners are + notified after the lock is released, so a listener can close the client. """ def __init__( @@ -56,12 +57,13 @@ def set_overrides(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None log.debug("Ignoring an override snapshot supplied after the override source was closed") return try: - self._apply(flags, segments) + affected = self._apply(flags, segments) except DeletedItemError as e: # The layer rejected the whole snapshot before changing anything, so nothing is # notified. The error also reaches the source, as for an invalid definition. log.error("Rejected an override snapshot; the overrides in effect stay: %s", e) raise + self._notify_all(affected) def close(self) -> None: """ @@ -79,13 +81,14 @@ def close_and_clear(self) -> None: """ with self._lock: self._closed = True - self._apply({}, {}) + affected = self._apply({}, {}) + self._notify_all(affected) - def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: - # Called with _lock held. + def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> Set[str]: + # Called with _lock held. Returns the keys of the affected flags. if not self._has_listeners(): self._layer.set_all(flags, segments) - return + return set() previous, current = self._layer.set_all(flags, segments) old_merged = snapshot_merged_view(self._base, previous) @@ -93,6 +96,10 @@ def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: affected = compute_affected_flags(previous, current, old_merged, new_merged) if len(affected) > 0: log.debug("Override update affected %d flag(s)", len(affected)) + return affected + + def _notify_all(self, affected: Set[str]) -> None: + # Called without _lock held. A listener can then close the client, which closes this sink. for key in sorted(affected): self._notify(key) diff --git a/ldclient/testing/impl/datasystem/test_fdv2_datasystem.py b/ldclient/testing/impl/datasystem/test_fdv2_datasystem.py index 6cc98a85..d195f009 100644 --- a/ldclient/testing/impl/datasystem/test_fdv2_datasystem.py +++ b/ldclient/testing/impl/datasystem/test_fdv2_datasystem.py @@ -153,6 +153,42 @@ def listener(flag_change: FlagChange): assert len(changes) == 0 +def test_fdv2_can_be_stopped_from_a_flag_change_listener(): + td = TestDataV2.data_source() + td.update(td.flag("feature-flag").on(True)) + data_system_config = DataSystemConfig( + initializers=None, + synchronizers=[td.builder], + ) + + set_on_ready = Event() + fdv2 = FDv2(Config(sdk_key="dummy"), data_system_config) + + stopped = Event() + errors: List[Exception] = [] + + def listener(flag_change: FlagChange): + try: + fdv2.stop() + except Exception as e: + errors.append(e) + finally: + stopped.set() + + fdv2.flag_change_listeners.add(listener) + + fdv2.start(set_on_ready) + assert set_on_ready.wait(1), "Data system did not become ready in time" + + td.update(td.flag("feature-flag").on(False)) + assert stopped.wait(1), "Flag change listener was not called in time" + assert errors == [] + + for thread in fdv2._threads: + thread.join(timeout=1) + assert not thread.is_alive(), "Thread %s did not exit" % thread.name + + def test_fdv2_data_availability_is_refreshed_with_data(): td = TestDataV2.data_source() data_system_config = DataSystemConfig( diff --git a/ldclient/testing/impl/test_overrides.py b/ldclient/testing/impl/test_overrides.py index 1681d8e4..3cdfb7ea 100644 --- a/ldclient/testing/impl/test_overrides.py +++ b/ldclient/testing/impl/test_overrides.py @@ -3,6 +3,7 @@ snapshots and notifies flag change listeners. """ import logging +import threading from typing import Any, Dict, List, Optional import pytest @@ -502,6 +503,28 @@ def test_sink_close_and_clear_clears_the_layer_and_ignores_later_snapshots(): assert f.take_notified() == [] +def test_sink_listener_can_close_the_sink(): + # A listener that closes the client closes the sink. The sink must not hold its lock while + # it notifies, or the close waits forever. + f = SinkFixture(FakeBaseStore()) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + f.take_notified() + + def notify(key: str): + f.notified.append(key) + f.sink.close() + + f.sink._notify = notify + worker = threading.Thread(target=lambda: f.sink.set_overrides({'flag1': flag('flag1', 2)}, {}), daemon=True) + worker.start() + worker.join(5) + + assert not worker.is_alive(), "Closing the sink from a listener did not return" + assert f.take_notified() == ['flag1'] + f.sink.set_overrides({'flag2': flag('flag2')}, {}) + assert f.take_notified() == [] + + def test_sink_notifies_directly_changed_keys_when_base_read_fails(): base = FakeBaseStore(flags={'flag1': flag('flag1')}) base.fail_all = True diff --git a/ldclient/testing/test_file_data_source.py b/ldclient/testing/test_file_data_source.py index b8e3fb0b..f1a76231 100644 --- a/ldclient/testing/test_file_data_source.py +++ b/ldclient/testing/test_file_data_source.py @@ -285,6 +285,39 @@ def test_reloads_modified_file_in_polling_mode(): do_auto_update_test({'auto_update': True, 'force_polling': True, 'poll_interval': 0.1}) +@pytest.mark.parametrize('options', [ + pytest.param({'auto_update': True}, id='watchdog'), + pytest.param({'auto_update': True, 'force_polling': True, 'poll_interval': 0.1}, id='polling'), +]) +def test_client_can_be_closed_from_a_flag_change_listener_on_reload(options): + path = make_temp_file(flag_only_json) + factory = Files.new_data_source(paths=path, **options) + client = LDClient(config=Config('SDK_KEY', update_processor_class=factory, send_events=False)) + try: + closed = threading.Event() + errors: List[Exception] = [] + + def listener(_): + if closed.is_set(): + return + try: + client.close() + except Exception as e: + errors.append(e) + finally: + closed.set() + + client.flag_tracker.add_listener(listener) + time.sleep(0.5) + replace_file(path, segment_only_json) + + assert closed.wait(20), "Flag change listener was not called in time" + assert errors == [] + finally: + client.close() + os.remove(path) + + def test_evaluates_full_flag_with_client_as_expected(): path = make_temp_file(all_properties_json) try: diff --git a/ldclient/testing/test_ldclient_listeners.py b/ldclient/testing/test_ldclient_listeners.py index 2ad697db..36963704 100644 --- a/ldclient/testing/test_ldclient_listeners.py +++ b/ldclient/testing/test_ldclient_listeners.py @@ -1,8 +1,14 @@ +import time from queue import Queue +from threading import Event +from typing import Any, Callable, List, Tuple +from ldclient import datasystem from ldclient.client import Config, LDClient from ldclient.config import BigSegmentsConfig -from ldclient.interfaces import DataSourceState +from ldclient.feature_store import InMemoryFeatureStore +from ldclient.integrations.test_datav2 import TestDataV2 +from ldclient.interfaces import DataSourceState, DataStoreMode from ldclient.testing.http_util import start_server from ldclient.testing.mock_components import MockBigSegmentStore from ldclient.testing.stub_util import ( @@ -61,3 +67,102 @@ def test_data_source_status_updates(): with LDClient(config=config) as client: assert client.data_source_status_provider.status.state == DataSourceState.VALID assert client.data_source_status_provider.status.error is None + + +def test_client_can_be_closed_from_a_flag_change_listener(): + td = TestDataV2.data_source() + td.update(td.flag('flag').on(True)) + store = _CloseTrackingStore() + client = _make_client(td, store) + try: + closed, errors = _close_from_listener(client, client.flag_tracker.add_listener) + td.update(td.flag('flag').on(False)) + _assert_closed_from_listener(client, store, closed, errors) + finally: + client.close() + + +def test_client_can_be_closed_from_a_data_store_status_listener(): + td = TestDataV2.data_source() + td.update(td.flag('flag').on(True)) + store = _CloseTrackingStore() + client = _make_client(td, store) + try: + closed, errors = _close_from_listener(client, client.data_store_status_provider.add_listener) + # A failed write reports the store as unavailable on the thread that applies the update. + store.fail_writes = True + td.update(td.flag('flag').on(False)) + _assert_closed_from_listener(client, store, closed, errors) + finally: + client.close() + + +def _make_client(td: TestDataV2, store: '_CloseTrackingStore') -> LDClient: + data_system = datasystem.custom().synchronizers(td.builder).data_store(store, DataStoreMode.READ_WRITE).build() + client = LDClient(Config('SDK_KEY', send_events=False, datasystem_config=data_system), start_wait=5) + assert client.is_initialized() + return client + + +def _close_from_listener(client: LDClient, add_listener: Callable[[Callable[[Any], None]], None]) -> Tuple[Event, List[Exception]]: + closed = Event() + errors: List[Exception] = [] + + def listener(_): + if closed.is_set(): + return + try: + client.close() + except Exception as e: + errors.append(e) + finally: + closed.set() + + add_listener(listener) + return closed, errors + + +def _assert_closed_from_listener(client: LDClient, store: '_CloseTrackingStore', closed: Event, errors: List[Exception]): + assert closed.wait(5), "Listener was not called in time" + assert errors == [] + assert store.closed + + # The listener's own thread sets the OFF status after the listener returns. + deadline = time.monotonic() + 5 + while client.data_source_status_provider.status.state != DataSourceState.OFF and time.monotonic() < deadline: + time.sleep(0.01) + assert client.data_source_status_provider.status.state == DataSourceState.OFF + + # The update that called the listener is not written to the closed store. + assert store.writes_after_close == [] + + +class _CloseTrackingStore(InMemoryFeatureStore): + def __init__(self): + super().__init__() + self.closed = False + self.fail_writes = False + self.writes_after_close: List[str] = [] + + def init(self, all_data): + self._record_write('init') + super().init(all_data) + + def upsert(self, kind, item): + self._record_write('upsert') + super().upsert(kind, item) + + def _record_write(self, operation: str): + if self.closed: + self.writes_after_close.append(operation) + if self.fail_writes: + raise Exception("simulated store failure") + + def is_monitoring_enabled(self) -> bool: + return True + + def is_available(self) -> bool: + return not self.fail_writes + + def close(self): + self.closed = True