Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions ldclient/impl/datasystem/fdv2.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import time
from queue import Queue
from threading import Event, Thread
from threading import Event, Thread, current_thread
from typing import Any, Callable, Dict, List, Mapping, Optional

from ldclient.config import Config, DataSourceBuilder, DataSystemConfig
Expand Down Expand Up @@ -367,8 +367,11 @@ def stop(self):
except Exception as e:
log.error("Error stopping active data source: %s", e)

# Wait for all threads to complete
# Wait for all threads to complete. A listener can call stop on one of these threads,
# and a thread cannot join itself. That thread sees the stop event and exits on its own.
for thread in self._threads:
if thread is current_thread():
continue
if thread.is_alive():
thread.join(timeout=5.0) # 5 second timeout
if thread.is_alive():
Expand Down
8 changes: 7 additions & 1 deletion ldclient/impl/datasystem/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,10 @@ def __init__(
# store. Set on each successful apply from its persist flag.
self._persist = False

# True after close. A listener can close the store while apply notifies it. The
# change set that apply is processing is then not written to the closed persistent store.
self._closed = False

def with_persistence(
self,
persistent_store: FeatureStore,
Expand Down Expand Up @@ -518,7 +522,8 @@ def apply(self, change_set: ChangeSet, persist: bool) -> None:
def _should_persist(self) -> bool:
"""Returns whether data should be persisted to the persistent store."""
return (
self._persist
not self._closed
and self._persist
and self._persistent_store is not None
and self._persistent_store_writable
)
Expand Down Expand Up @@ -566,6 +571,7 @@ def __mapping(data: Dict[str, ModelEntity]) -> Dict[str, Dict[str, Any]]:
def close(self) -> None:
"""Close the store and any persistent store if configured."""
with self._lock:
self._closed = True
if self._persistent_store is not None:
try:
# Most FeatureStore implementations don't have close methods
Expand Down
6 changes: 5 additions & 1 deletion ldclient/impl/integrations/files/file_data_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import os
import time
import traceback
from threading import current_thread
from typing import Optional

from ldclient.impl.repeating_task import RepeatingTask
Expand Down Expand Up @@ -174,7 +175,10 @@ def on_any_event(self, event):

def stop(self):
self._observer.stop()
self._observer.join()
# A listener can stop the client on the observer thread, and a thread cannot join
# itself. That thread sees the stop request and exits on its own.
if self._observer is not current_thread():
self._observer.join()

# Watch for changes to data files by polling their modification times. This is used if auto-update is
# on but the watchdog package is not installed.
Expand Down
21 changes: 14 additions & 7 deletions ldclient/impl/overrides/sink.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@
class OverrideSinkImpl(OverrideSink):
"""
Applies override layer replacements supplied by an override source, then notifies flag
change listeners of every flag whose merged-view evaluation may have changed. Calls are
serialized, so overlapping updates from a source cannot interleave.
change listeners of every flag whose merged-view evaluation may have changed. Layer updates
are serialized, so overlapping updates from a source cannot interleave. Listeners are
notified after the lock is released, so a listener can close the client.
"""

def __init__(
Expand Down Expand Up @@ -56,12 +57,13 @@ def set_overrides(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None
log.debug("Ignoring an override snapshot supplied after the override source was closed")
return
try:
self._apply(flags, segments)
affected = self._apply(flags, segments)
except DeletedItemError as e:
# The layer rejected the whole snapshot before changing anything, so nothing is
# notified. The error also reaches the source, as for an invalid definition.
log.error("Rejected an override snapshot; the overrides in effect stay: %s", e)
raise
self._notify_all(affected)

def close(self) -> None:
"""
Expand All @@ -79,20 +81,25 @@ def close_and_clear(self) -> None:
"""
with self._lock:
self._closed = True
self._apply({}, {})
affected = self._apply({}, {})
self._notify_all(affected)

def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None:
# Called with _lock held.
def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> Set[str]:
# Called with _lock held. Returns the keys of the affected flags.
if not self._has_listeners():
self._layer.set_all(flags, segments)
return
return set()

previous, current = self._layer.set_all(flags, segments)
old_merged = snapshot_merged_view(self._base, previous)
new_merged = snapshot_merged_view(self._base, current)
affected = compute_affected_flags(previous, current, old_merged, new_merged)
if len(affected) > 0:
log.debug("Override update affected %d flag(s)", len(affected))
return affected

def _notify_all(self, affected: Set[str]) -> None:
# Called without _lock held. A listener can then close the client, which closes this sink.
for key in sorted(affected):
self._notify(key)

Expand Down
36 changes: 36 additions & 0 deletions ldclient/testing/impl/datasystem/test_fdv2_datasystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,42 @@ def listener(flag_change: FlagChange):
assert len(changes) == 0


def test_fdv2_can_be_stopped_from_a_flag_change_listener():
td = TestDataV2.data_source()
td.update(td.flag("feature-flag").on(True))
data_system_config = DataSystemConfig(
initializers=None,
synchronizers=[td.builder],
)

set_on_ready = Event()
fdv2 = FDv2(Config(sdk_key="dummy"), data_system_config)

stopped = Event()
errors: List[Exception] = []

def listener(flag_change: FlagChange):
try:
fdv2.stop()
except Exception as e:
errors.append(e)
finally:
stopped.set()

fdv2.flag_change_listeners.add(listener)

fdv2.start(set_on_ready)
assert set_on_ready.wait(1), "Data system did not become ready in time"

td.update(td.flag("feature-flag").on(False))
assert stopped.wait(1), "Flag change listener was not called in time"
assert errors == []

for thread in fdv2._threads:
thread.join(timeout=1)
assert not thread.is_alive(), "Thread %s did not exit" % thread.name


def test_fdv2_data_availability_is_refreshed_with_data():
td = TestDataV2.data_source()
data_system_config = DataSystemConfig(
Expand Down
23 changes: 23 additions & 0 deletions ldclient/testing/impl/test_overrides.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
snapshots and notifies flag change listeners.
"""
import logging
import threading
from typing import Any, Dict, List, Optional

import pytest
Expand Down Expand Up @@ -502,6 +503,28 @@ def test_sink_close_and_clear_clears_the_layer_and_ignores_later_snapshots():
assert f.take_notified() == []


def test_sink_listener_can_close_the_sink():
# A listener that closes the client closes the sink. The sink must not hold its lock while
# it notifies, or the close waits forever.
f = SinkFixture(FakeBaseStore())
f.sink.set_overrides({'flag1': flag('flag1')}, {})
f.take_notified()

def notify(key: str):
f.notified.append(key)
f.sink.close()

f.sink._notify = notify
worker = threading.Thread(target=lambda: f.sink.set_overrides({'flag1': flag('flag1', 2)}, {}), daemon=True)
worker.start()
worker.join(5)

assert not worker.is_alive(), "Closing the sink from a listener did not return"
assert f.take_notified() == ['flag1']
f.sink.set_overrides({'flag2': flag('flag2')}, {})
assert f.take_notified() == []


def test_sink_notifies_directly_changed_keys_when_base_read_fails():
base = FakeBaseStore(flags={'flag1': flag('flag1')})
base.fail_all = True
Expand Down
33 changes: 33 additions & 0 deletions ldclient/testing/test_file_data_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,39 @@ def test_reloads_modified_file_in_polling_mode():
do_auto_update_test({'auto_update': True, 'force_polling': True, 'poll_interval': 0.1})


@pytest.mark.parametrize('options', [
pytest.param({'auto_update': True}, id='watchdog'),
pytest.param({'auto_update': True, 'force_polling': True, 'poll_interval': 0.1}, id='polling'),
])
def test_client_can_be_closed_from_a_flag_change_listener_on_reload(options):
path = make_temp_file(flag_only_json)
factory = Files.new_data_source(paths=path, **options)
client = LDClient(config=Config('SDK_KEY', update_processor_class=factory, send_events=False))
try:
closed = threading.Event()
errors: List[Exception] = []

def listener(_):
if closed.is_set():
return
try:
client.close()
except Exception as e:
errors.append(e)
finally:
closed.set()

client.flag_tracker.add_listener(listener)
time.sleep(0.5)
replace_file(path, segment_only_json)

assert closed.wait(20), "Flag change listener was not called in time"
assert errors == []
finally:
client.close()
os.remove(path)


def test_evaluates_full_flag_with_client_as_expected():
path = make_temp_file(all_properties_json)
try:
Expand Down
107 changes: 106 additions & 1 deletion ldclient/testing/test_ldclient_listeners.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
import time
from queue import Queue
from threading import Event
from typing import Any, Callable, List, Tuple

from ldclient import datasystem
from ldclient.client import Config, LDClient
from ldclient.config import BigSegmentsConfig
from ldclient.interfaces import DataSourceState
from ldclient.feature_store import InMemoryFeatureStore
from ldclient.integrations.test_datav2 import TestDataV2
from ldclient.interfaces import DataSourceState, DataStoreMode
from ldclient.testing.http_util import start_server
from ldclient.testing.mock_components import MockBigSegmentStore
from ldclient.testing.stub_util import (
Expand Down Expand Up @@ -61,3 +67,102 @@ def test_data_source_status_updates():
with LDClient(config=config) as client:
assert client.data_source_status_provider.status.state == DataSourceState.VALID
assert client.data_source_status_provider.status.error is None


def test_client_can_be_closed_from_a_flag_change_listener():
td = TestDataV2.data_source()
td.update(td.flag('flag').on(True))
store = _CloseTrackingStore()
client = _make_client(td, store)
try:
closed, errors = _close_from_listener(client, client.flag_tracker.add_listener)
td.update(td.flag('flag').on(False))
_assert_closed_from_listener(client, store, closed, errors)
finally:
client.close()


def test_client_can_be_closed_from_a_data_store_status_listener():
td = TestDataV2.data_source()
td.update(td.flag('flag').on(True))
store = _CloseTrackingStore()
client = _make_client(td, store)
try:
closed, errors = _close_from_listener(client, client.data_store_status_provider.add_listener)
# A failed write reports the store as unavailable on the thread that applies the update.
store.fail_writes = True
td.update(td.flag('flag').on(False))
_assert_closed_from_listener(client, store, closed, errors)
finally:
client.close()


def _make_client(td: TestDataV2, store: '_CloseTrackingStore') -> LDClient:
data_system = datasystem.custom().synchronizers(td.builder).data_store(store, DataStoreMode.READ_WRITE).build()
client = LDClient(Config('SDK_KEY', send_events=False, datasystem_config=data_system), start_wait=5)
assert client.is_initialized()
return client


def _close_from_listener(client: LDClient, add_listener: Callable[[Callable[[Any], None]], None]) -> Tuple[Event, List[Exception]]:
closed = Event()
errors: List[Exception] = []

def listener(_):
if closed.is_set():
return
try:
client.close()
except Exception as e:
errors.append(e)
finally:
closed.set()

add_listener(listener)
return closed, errors


def _assert_closed_from_listener(client: LDClient, store: '_CloseTrackingStore', closed: Event, errors: List[Exception]):
assert closed.wait(5), "Listener was not called in time"
assert errors == []
assert store.closed

# The listener's own thread sets the OFF status after the listener returns.
deadline = time.monotonic() + 5
while client.data_source_status_provider.status.state != DataSourceState.OFF and time.monotonic() < deadline:
time.sleep(0.01)
assert client.data_source_status_provider.status.state == DataSourceState.OFF

# The update that called the listener is not written to the closed store.
assert store.writes_after_close == []


class _CloseTrackingStore(InMemoryFeatureStore):
def __init__(self):
super().__init__()
self.closed = False
self.fail_writes = False
self.writes_after_close: List[str] = []

def init(self, all_data):
self._record_write('init')
super().init(all_data)

def upsert(self, kind, item):
self._record_write('upsert')
super().upsert(kind, item)

def _record_write(self, operation: str):
if self.closed:
self.writes_after_close.append(operation)
if self.fail_writes:
raise Exception("simulated store failure")

def is_monitoring_enabled(self) -> bool:
return True

def is_available(self) -> bool:
return not self.fail_writes

def close(self):
self.closed = True
Loading