diff --git a/contract-tests/async_client_entity.py b/contract-tests/async_client_entity.py index 30708349..488830e1 100644 --- a/contract-tests/async_client_entity.py +++ b/contract-tests/async_client_entity.py @@ -27,6 +27,7 @@ ) from ldclient.impl.util import Result from ldclient.integrations import Redis +from ldclient.integrations.overrides import FileOverrideSourceBuilder from ldclient.interfaces import DataStoreMode from ldclient.migrations import ( AsyncMigratorBuilder, @@ -82,8 +83,10 @@ async def start(self): opts['events_uri'] = endpoints['events'] datasystem_config = config_params.get('dataSystem') + if config_params.get('overrides') is not None and datasystem_config is None: + raise ValueError("flag overrides require the data system to be configured") if datasystem_config is not None: - opts["datasystem_config"] = _build_async_data_system(datasystem_config, opts) + opts["datasystem_config"] = _build_async_data_system(datasystem_config, opts, config_params.get('overrides')) elif config_params.get("streaming") is not None: streaming = config_params["streaming"] if streaming.get("baseUri") is not None: @@ -320,13 +323,13 @@ def _set_optional_value(params_in: dict, name_in: str, func: Callable[[Any], Any func(params_in[name_in]) -def _build_async_data_system(datasystem_config: dict, opts: dict) -> AsyncDataSystemConfig: +def _build_async_data_system(datasystem_config: dict, opts: dict, overrides_config: Optional[dict] = None) -> AsyncDataSystemConfig: """Build an AsyncDataSystemConfig from the harness's dataSystem config. Wires the FDv2 initializers, the ordered synchronizer chain, the FDv1 - fallback synchronizer, the payload filter, and an optional async - persistent store. The async client injects its shared aiohttp session - into these builders when it starts. + fallback synchronizer, the payload filter, an optional async persistent + store, and the optional file-based override source. The async client + injects its shared aiohttp session into these builders when it starts. """ initializers: Optional[list] = None init_configs = datasystem_config.get('initializers') @@ -379,6 +382,9 @@ def _build_async_data_system(datasystem_config: dict, opts: dict) -> AsyncDataSy "fdv1_fallback_synchronizer": fdv1_fallback_synchronizer, } + if overrides_config is not None: + ds_kwargs["override_source"] = _make_override_source(overrides_config) + store_config = datasystem_config.get("store") if store_config is not None: persistent_store_config = store_config.get("persistentDataStore") @@ -393,6 +399,18 @@ def _build_async_data_system(datasystem_config: dict, opts: dict) -> AsyncDataSy return AsyncDataSystemConfig(**ds_kwargs) +def _make_override_source(params: dict) -> FileOverrideSourceBuilder: + """ + Builds the file-based override source from the harness's overrides configuration. The + files are written by the harness on a file system shared with this service. + """ + builder = FileOverrideSourceBuilder(params["filePaths"]) + _set_optional_value(params, "duplicateKeysHandling", builder.duplicate_keys_handling) + _set_optional_value(params, "changeDetection", builder.change_detection) + _set_optional_time(params, "pollIntervalMs", builder.poll_interval) + return builder + + def _create_async_persistent_store(persistent_store_config: dict): """Create an async persistent feature store from the harness config. diff --git a/contract-tests/async_service.py b/contract-tests/async_service.py index ac87e64b..dae59b19 100644 --- a/contract-tests/async_service.py +++ b/contract-tests/async_service.py @@ -69,6 +69,7 @@ async def handle_status(request: aiohttp.web.Request) -> aiohttp.web.Response: 'migrations', 'persistent-data-store-redis', 'fdv1-fallback', + 'flag-overrides', 'retry-conformance-fdv1-streaming', 'retry-conformance-fdv1-polling', 'service-endpoints', diff --git a/contract-tests/client_entity.py b/contract-tests/client_entity.py index cb8caf79..394b83a4 100644 --- a/contract-tests/client_entity.py +++ b/contract-tests/client_entity.py @@ -28,6 +28,7 @@ from ldclient.feature_store import CacheConfig from ldclient.impl.datasourcev2.polling import PollingDataSourceBuilder from ldclient.integrations import Consul, DynamoDB, Redis +from ldclient.integrations.overrides import FileOverrideSourceBuilder from ldclient.interfaces import DataStoreMode @@ -69,6 +70,8 @@ def __init__(self, tag, config): opts['events_uri'] = endpoints['events'] datasystem_config = config.get('dataSystem') + if config.get('overrides') is not None and datasystem_config is None: + raise ValueError("flag overrides require the data system to be configured") if datasystem_config is not None: datasystem = custom() @@ -132,6 +135,10 @@ def __init__(self, tag, config): datasystem.data_store(store, store_mode) + overrides_config = config.get('overrides') + if overrides_config is not None: + datasystem.overrides(_make_override_source(overrides_config)) + opts["datasystem_config"] = datasystem.build() elif config.get("streaming") is not None: @@ -352,6 +359,18 @@ def _set_optional_value(params_in: dict, name_in: str, func: Callable[[Any], Any func(params_in[name_in]) +def _make_override_source(params: dict) -> FileOverrideSourceBuilder: + """ + Builds the file-based override source from the harness's overrides configuration. The + files are written by the harness on a file system shared with this service. + """ + builder = FileOverrideSourceBuilder(params["filePaths"]) + _set_optional_value(params, "duplicateKeysHandling", builder.duplicate_keys_handling) + _set_optional_value(params, "changeDetection", builder.change_detection) + _set_optional_time(params, "pollIntervalMs", builder.poll_interval) + return builder + + def _create_persistent_store(persistent_store_config: dict): """ Creates a persistent store instance based on the configuration. diff --git a/contract-tests/service.py b/contract-tests/service.py index 4631fdee..eb977734 100644 --- a/contract-tests/service.py +++ b/contract-tests/service.py @@ -86,6 +86,7 @@ def status(): 'flag-change-listeners', 'flag-value-change-listeners', 'fdv1-fallback', + 'flag-overrides', 'retry-conformance-fdv1-streaming', 'retry-conformance-fdv1-polling', 'service-endpoints', diff --git a/docs/api-integrations.rst b/docs/api-integrations.rst index 8d8146ff..594d7ed8 100644 --- a/docs/api-integrations.rst +++ b/docs/api-integrations.rst @@ -8,3 +8,13 @@ ldclient.integrations module :members: :special-members: __init__ :show-inheritance: + +ldclient.integrations.overrides module +-------------------------------------- + +The entry point for this feature is :class:`ldclient.integrations.overrides.FileOverrideSourceBuilder`. + +.. automodule:: ldclient.integrations.overrides + :members: + :special-members: __init__ + :show-inheritance: diff --git a/ldclient/async_client.py b/ldclient/async_client.py index 08656245..9c9f975d 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -4,7 +4,7 @@ import asyncio import traceback -from typing import Any, Callable, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple from uuid import uuid4 from ldclient.async_config import AsyncConfig @@ -28,7 +28,7 @@ ) from ldclient.impl.client_common import secure_mode_hash as _secure_mode_hash from ldclient.impl.datasystem import AsyncDataSystem, DataAvailability -from ldclient.impl.evaluator_common import error_reason +from ldclient.impl.evaluator_common import error_reason, is_override_definition from ldclient.impl.events.async_event_processor import ( DefaultAsyncEventProcessor ) @@ -95,6 +95,7 @@ def __init__(self, config: AsyncConfig): # these flags, so plain booleans are safe. self._eval_cached_data_warned = False self._all_flags_cached_data_warned = False + self._all_flags_overrides_only_warned = False # Build the object graph here (loop-free). start() supplies the loop-bound # resources: the HTTP session (created lazily), the data source, the @@ -485,7 +486,12 @@ async def evaluate(): tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker}) - detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE')) + # The type mismatch replaces the reason. The evaluation read the same definitions, + # so the new reason keeps the override-affected marking. + reason = error_reason('WRONG_TYPE') + if detail.reason.get('overrideAffected') is True: + reason['overrideAffected'] = True + detail = EvaluationDetail(default_stage.value, None, reason) tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker}) @@ -498,6 +504,13 @@ async def _evaluate_internal(self, key: str, context: Context, default: Any, eve if self._config.offline: return EvaluationDetail(default, None, error_reason('CLIENT_NOT_READY')), None + def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: + log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) + reason = error_reason('CLIENT_NOT_READY') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None + + flag: Any = None availability = await self._data_system.data_availability() if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -505,23 +518,28 @@ async def _evaluate_internal(self, key: str, context: Context, default: Any, eve self._eval_cached_data_warned = True log.warning("Feature Flag evaluation attempted before client has initialized - using last known values from feature store for feature key: " + key + ". This message is logged once.") else: - log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) - reason = error_reason('CLIENT_NOT_READY') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + # No data from LaunchDarkly is available. A flag that the override layer holds + # is served anyway. A flag the layer does not hold gets the not-ready default, + # as it does without an override source. + layer = self._data_system.override_layer + if layer is not None: + flag = layer.get(FEATURES, key) + if flag is None: + return not_ready() if not context.valid: log.warning("Context was invalid for flag evaluation (%s); returning default value" % context.error) return EvaluationDetail(default, None, error_reason('USER_NOT_SPECIFIED')), None - try: - flag = await self._data_system.store.get(FEATURES, key) - except Exception as e: - log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) - log.debug(traceback.format_exc()) - reason = error_reason('EXCEPTION') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + if flag is None: + try: + flag = await self._data_system.store.get(FEATURES, key) + except Exception as e: + log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) + log.debug(traceback.format_exc()) + reason = error_reason('EXCEPTION') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None if not flag: reason = error_reason('FLAG_NOT_FOUND') self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) @@ -534,13 +552,19 @@ async def _evaluate_internal(self, key: str, context: Context, default: Any, eve detail = result.detail if detail.is_default_value(): detail = EvaluationDetail(default, None, detail.reason) - self._send_event(event_factory.new_eval_event(flag, context, detail, default)) + self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected)) return detail, flag except Exception as e: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') - self._send_event(event_factory.new_default_event(flag, context, default, reason)) + # The evaluation read the flag definition before it failed. A definition from + # the override layer marks the result, so the failure is reported in the + # summary counters only, like any other override-affected evaluation. + override_affected = is_override_definition(flag) + if override_affected: + reason['overrideAffected'] = True + self._send_event(event_factory.new_default_event(flag, context, default, reason, override_affected)) return EvaluationDetail(default, None, reason), flag async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: @@ -572,6 +596,7 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState log.warning("all_flags_state() called, but client is in offline mode. Returning empty state") return FeatureFlagsState(False) + override_flags: Optional[Dict[str, Any]] = None availability = await self._data_system.data_availability() if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -579,25 +604,49 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState self._all_flags_cached_data_warned = True log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") else: - log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") - return FeatureFlagsState(False) + # No data from LaunchDarkly is available. When the override layer holds flags, + # the state contains only those, read from the layer itself: a store that is not + # initialized can still hold LaunchDarkly data, which the state must not include. + # Without override flags the state is unavailable, as it is without an override + # source. + layer = self._data_system.override_layer + override_flags = layer.all(FEATURES) if layer is not None else {} + if len(override_flags) == 0: + log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") + return FeatureFlagsState(False) if not context.valid: log.warning("Context was invalid for all_flags_state (%s); returning default value" % context.error) return FeatureFlagsState(False) - state = FeatureFlagsState(True) + valid = True client_only = kwargs.get('client_side_only', False) with_reasons = kwargs.get('with_reasons', False) details_only_if_tracked = kwargs.get('details_only_for_tracked_flags', False) - try: - flags_map = await self._data_system.store.all(FEATURES) - if flags_map is None: - raise ValueError("feature store error") - except Exception as e: - log.error("Unable to read flags for all_flag_state: %s" % repr(e)) - return FeatureFlagsState(False) - + if override_flags is not None: + if not self._all_flags_overrides_only_warned: + self._all_flags_overrides_only_warned = True + log.warning("all_flags_state() called before client has finished initializing! Returning only flags from the override layer. This message is logged once.") + flags_map = override_flags + else: + try: + flags_map = await self._data_system.store.all(FEATURES) + if flags_map is None: + raise ValueError("feature store error") + except Exception as e: + log.error("Unable to read flags for all_flag_state: %s" % repr(e)) + # Overrides only add to the state. Without them the state is invalid and empty + # here, so it holds the override layer's flags and stays invalid. A client-side + # SDK bootstrapped from the state uses its flags whatever $valid says (the + # JavaScript client SDKs 3.x and 4.x only warn), so the overridden values still + # reach it. + layer = self._data_system.override_layer + flags_map = layer.all(FEATURES) if layer is not None else {} + if len(flags_map) == 0: + return FeatureFlagsState(False) + valid = False + + state = FeatureFlagsState(valid) for key, flag in flags_map.items(): if client_only and not flag.get('clientSide', False): continue @@ -605,13 +654,29 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState result = await self._evaluator.evaluate(flag, context, self._event_factory_default) detail = result.detail prerequisites = result.prerequisites + override_affected = result.override_affected except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) - reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + reason = error_reason('EXCEPTION') + # The evaluation read the flag definition before it failed, so a definition from + # the override layer marks the result. + override_affected = is_override_definition(flag) + if override_affected: + reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) + track_events = flag.get('trackEvents', False) or requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = flag.get('debugEventsUntilDate', None) + if override_affected: + # A consumer of this state sends individual events according to these fields. + # An override-affected evaluation produces no individual events, so the state + # turns them off for this flag. The flag, its value, and its reason stay. + track_events = False + track_reason = False + debug_events_until_date = None flag_state = { 'key': flag['key'], 'value': detail.value, @@ -619,9 +684,9 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState 'reason': detail.reason, 'version': flag['version'], 'prerequisites': prerequisites, - 'trackEvents': flag.get('trackEvents', False) or requires_experiment_data, - 'trackReason': requires_experiment_data, - 'debugEventsUntilDate': flag.get('debugEventsUntilDate', None), + 'trackEvents': track_events, + 'trackReason': track_reason, + 'debugEventsUntilDate': debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) diff --git a/ldclient/async_config.py b/ldclient/async_config.py index 5090d9e3..43c632ed 100644 --- a/ldclient/async_config.py +++ b/ldclient/async_config.py @@ -23,6 +23,7 @@ DataSourceBuilder, DataSourceBuilderConfig, HTTPConfig, + OverrideSourceBuilder, PrivateAttributesConfig ) from ldclient.hook import Hook @@ -129,6 +130,20 @@ class AsyncDataSystemConfig: fdv1_fallback_synchronizer: Optional[DataSourceBuilder[AsyncSynchronizer]] = None """An optional fallback synchronizer that will read from FDv1""" + override_source: Optional[OverrideSourceBuilder] = None + """ + An optional override source. Its flag and segment definitions take precedence over + LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject + to change. + + When the client is offline, the source is not built or started: no overrides are served, and + a configuration error in the source is not reported. The SDK logs a warning instead. + + When the client uses a persistent data store, a change to an override notifies the flag change + listeners of the overridden flags, but not of LaunchDarkly flags that depend on them through a + prerequisite or a segment. + """ + class AsyncConfig(DataSourceBuilderConfig, PrivateAttributesConfig): """Advanced configuration options for the async SDK client. diff --git a/ldclient/client.py b/ldclient/client.py index c8941e57..a9b5a2a9 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -28,6 +28,7 @@ from ldclient.impl.datasystem import DataAvailability, DataSystem from ldclient.impl.datasystem.fdv2 import FDv2 from ldclient.impl.evaluator import Evaluator, error_reason +from ldclient.impl.evaluator_common import is_override_definition from ldclient.impl.events.diagnostics import ( _DiagnosticAccumulator, create_diagnostic_id @@ -89,6 +90,7 @@ def __init__(self, config: Config, start_wait: float = 5): self._cached_data_warning_lock = threading.Lock() self._eval_cached_data_warned = False self._all_flags_cached_data_warned = False + self._all_flags_overrides_only_warned = False self._owner_pid = os.getpid() self._fork_warned_pids: Dict[int, object] = {} @@ -211,7 +213,13 @@ def __start_up(self, start_wait: float): self.__register_plugins(environment_metadata) update_processor_ready = threading.Event() - self._data_system.start(update_processor_ready) + try: + self._data_system.start(update_processor_ready) + except BaseException: + # A data system that fails to start fails the constructor. The components that + # are already running stop here, so the failure leaves no threads behind. + self._close_components() + raise if not self._config.offline and not self._config.use_ldd: if start_wait > 60: @@ -281,6 +289,24 @@ def close(self): self._data_system.stop() self.__big_segment_store_manager.stop() + def _close_components(self): + """ + Stops the SDK components after a failed start-up. The event processor is last because + it may still be sending events the other components generated. A component that fails to + stop is logged and the others are still stopped, so the caller receives the start-up + failure rather than the failure to stop. + """ + components = ( + ("data system", self._data_system.stop), + ("big segment store manager", self.__big_segment_store_manager.stop), + ("event processor", self._event_processor.stop), + ) + for name, stop in components: + try: + stop() + except Exception as e: + log.warning("Error stopping the %s after a failed start: %s", name, e) + # These magic methods allow a client object to be automatically cleaned up by the "with" scope operator def __enter__(self): return self @@ -455,7 +481,12 @@ def evaluate(): tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker}) - detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE')) + # The type mismatch replaces the reason. The evaluation read the same definitions, + # so the new reason keeps the override-affected marking. + reason = error_reason('WRONG_TYPE') + if detail.reason.get('overrideAffected') is True: + reason['overrideAffected'] = True + detail = EvaluationDetail(default_stage.value, None, reason) tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker}) @@ -468,6 +499,13 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac if self._config.offline: return EvaluationDetail(default, None, error_reason('CLIENT_NOT_READY')), None + def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: + log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) + reason = error_reason('CLIENT_NOT_READY') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None + + flag: Any = None availability = self._data_system.data_availability if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -477,23 +515,28 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac self._eval_cached_data_warned = True log.warning("Feature Flag evaluation attempted before client has initialized - using last known values from feature store for feature key: " + key + ". This message is logged once.") else: - log.warning("Feature Flag evaluation attempted before client has initialized! Feature store unavailable - returning default: " + str(default) + " for feature key: " + key) - reason = error_reason('CLIENT_NOT_READY') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + # No data from LaunchDarkly is available. A flag that the override layer holds + # is served anyway. A flag the layer does not hold gets the not-ready default, + # as it does without an override source. + layer = self._data_system.override_layer + if layer is not None: + flag = layer.get(FEATURES, key) + if flag is None: + return not_ready() if not context.valid: log.warning("Context was invalid for flag evaluation (%s); returning default value" % context.error) return EvaluationDetail(default, None, error_reason('USER_NOT_SPECIFIED')), None - try: - flag = self._data_system.store.get(FEATURES, key) - except Exception as e: - log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) - log.debug(traceback.format_exc()) - reason = error_reason('EXCEPTION') - self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) - return EvaluationDetail(default, None, reason), None + if flag is None: + try: + flag = self._data_system.store.get(FEATURES, key) + except Exception as e: + log.error("Unexpected error while retrieving feature flag \"%s\": %s" % (key, repr(e))) + log.debug(traceback.format_exc()) + reason = error_reason('EXCEPTION') + self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) + return EvaluationDetail(default, None, reason), None if not flag: reason = error_reason('FLAG_NOT_FOUND') self._send_event(event_factory.new_unknown_flag_event(key, context, default, reason)) @@ -506,13 +549,19 @@ def _evaluate_internal(self, key: str, context: Context, default: Any, event_fac detail = result.detail if detail.is_default_value(): detail = EvaluationDetail(default, None, detail.reason) - self._send_event(event_factory.new_eval_event(flag, context, detail, default)) + self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected)) return detail, flag except Exception as e: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') - self._send_event(event_factory.new_default_event(flag, context, default, reason)) + # The evaluation read the flag definition before it failed. A definition from + # the override layer marks the result, so the failure is reported in the + # summary counters only, like any other override-affected evaluation. + override_affected = is_override_definition(flag) + if override_affected: + reason['overrideAffected'] = True + self._send_event(event_factory.new_default_event(flag, context, default, reason, override_affected)) return EvaluationDetail(default, None, reason), flag def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: @@ -546,6 +595,7 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: self._check_forked() + override_flags: Optional[Dict[str, Any]] = None availability = self._data_system.data_availability if availability != DataAvailability.REFRESHED: if availability == DataAvailability.CACHED: @@ -555,25 +605,51 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: self._all_flags_cached_data_warned = True log.warning("all_flags_state() called before client has finished initializing! Using last known values from feature store. This message is logged once.") else: - log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") - return FeatureFlagsState(False) + # No data from LaunchDarkly is available. When the override layer holds flags, + # the state contains only those, read from the layer itself: a store that is not + # initialized can still hold LaunchDarkly data, which the state must not include. + # Without override flags the state is unavailable, as it is without an override + # source. + layer = self._data_system.override_layer + override_flags = layer.all(FEATURES) if layer is not None else {} + if len(override_flags) == 0: + log.warning("all_flags_state() called before client has finished initializing! Feature store unavailable - returning empty state") + return FeatureFlagsState(False) if not context.valid: log.warning("Context was invalid for all_flags_state (%s); returning default value" % context.error) return FeatureFlagsState(False) - state = FeatureFlagsState(True) + valid = True client_only = kwargs.get('client_side_only', False) with_reasons = kwargs.get('with_reasons', False) details_only_if_tracked = kwargs.get('details_only_for_tracked_flags', False) - try: - flags_map = self._data_system.store.all(FEATURES, lambda x: x) - if flags_map is None: - raise ValueError("feature store error") - except Exception as e: - log.error("Unable to read flags for all_flag_state: %s" % repr(e)) - return FeatureFlagsState(False) - + if override_flags is not None: + if not self._all_flags_overrides_only_warned: + with self._cached_data_warning_lock: + if not self._all_flags_overrides_only_warned: + self._all_flags_overrides_only_warned = True + log.warning("all_flags_state() called before client has finished initializing! Returning only flags from the override layer. This message is logged once.") + flags_map = override_flags + else: + try: + flags_map = self._data_system.store.all(FEATURES, lambda x: x) + if flags_map is None: + raise ValueError("feature store error") + except Exception as e: + log.error("Unable to read flags for all_flag_state: %s" % repr(e)) + # Overrides only add to the state. Without them the state is invalid and empty + # here, so it holds the override layer's flags and stays invalid. A client-side + # SDK bootstrapped from the state uses its flags whatever $valid says (the + # JavaScript client SDKs 3.x and 4.x only warn), so the overridden values still + # reach it. + layer = self._data_system.override_layer + flags_map = layer.all(FEATURES) if layer is not None else {} + if len(flags_map) == 0: + return FeatureFlagsState(False) + valid = False + + state = FeatureFlagsState(valid) for key, flag in flags_map.items(): if client_only and not flag.get('clientSide', False): continue @@ -581,14 +657,30 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: result = self._evaluator.evaluate(flag, context, self._event_factory_default) detail = result.detail prerequisites = result.prerequisites + override_affected = result.override_affected except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) - reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + reason = error_reason('EXCEPTION') + # The evaluation read the flag definition before it failed, so a definition from + # the override layer marks the result. + override_affected = is_override_definition(flag) + if override_affected: + reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) + track_events = flag.get('trackEvents', False) or requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = flag.get('debugEventsUntilDate', None) + if override_affected: + # A consumer of this state sends individual events according to these fields. + # An override-affected evaluation produces no individual events, so the state + # turns them off for this flag. The flag, its value, and its reason stay. + track_events = False + track_reason = False + debug_events_until_date = None flag_state = { 'key': flag['key'], 'value': detail.value, @@ -596,9 +688,9 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: 'reason': detail.reason, 'version': flag['version'], 'prerequisites': prerequisites, - 'trackEvents': flag.get('trackEvents', False) or requires_experiment_data, - 'trackReason': requires_experiment_data, - 'debugEventsUntilDate': flag.get('debugEventsUntilDate', None), + 'trackEvents': track_events, + 'trackReason': track_reason, + 'debugEventsUntilDate': debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) diff --git a/ldclient/config.py b/ldclient/config.py index 9e7bd263..678bca3f 100644 --- a/ldclient/config.py +++ b/ldclient/config.py @@ -24,6 +24,7 @@ EventProcessor, FeatureStore, Initializer, + OverrideSource, Synchronizer, UpdateProcessor ) @@ -260,6 +261,23 @@ def build(self, config: DataSourceBuilderConfig) -> T_co: raise NotImplementedError +class OverrideSourceBuilder(Protocol): # pylint: disable=too-few-public-methods + """ + Protocol for building an override source. Flag overrides are currently experimental and + subject to change. + """ + + def build(self, config: DataSourceBuilderConfig) -> OverrideSource: + """ + Builds the override source. Invalid configuration raises an exception, which the client + constructor propagates. + + :param config: the SDK configuration + :return: the built override source + """ + raise NotImplementedError + + @dataclass(frozen=True) class DataSystemConfig: """Configuration for LaunchDarkly's data acquisition strategy.""" @@ -283,6 +301,16 @@ class DataSystemConfig: fdv1_fallback_synchronizer: Optional[DataSourceBuilder[Synchronizer]] = None """An optional fallback synchronizer that will read from FDv1""" + override_source: Optional[OverrideSourceBuilder] = None + """ + An optional override source. Its flag and segment definitions take precedence over + LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject + to change. + + When the client is offline, the source is not built or started: no overrides are served, and + a configuration error in the source is not reported. The SDK logs a warning instead. + """ + class Config(DataSourceBuilderConfig, PrivateAttributesConfig): """Advanced configuration options for the SDK client. @@ -719,4 +747,4 @@ def _validate(self): log.warning("Missing or blank SDK key") -__all__ = ['Config', 'BigSegmentsConfig', 'DataSourceBuilder', 'DataSystemConfig', 'HTTPConfig'] +__all__ = ['Config', 'BigSegmentsConfig', 'DataSourceBuilder', 'DataSystemConfig', 'HTTPConfig', 'OverrideSourceBuilder'] diff --git a/ldclient/datasystem.py b/ldclient/datasystem.py index d3a61a54..0e36ec34 100644 --- a/ldclient/datasystem.py +++ b/ldclient/datasystem.py @@ -4,7 +4,11 @@ from typing import List, Optional -from ldclient.config import DataSourceBuilder, DataSystemConfig +from ldclient.config import ( + DataSourceBuilder, + DataSystemConfig, + OverrideSourceBuilder +) from ldclient.impl.datasourcev2.polling import ( FallbackToFDv1PollingDataSourceBuilder, PollingDataSourceBuilder @@ -32,6 +36,7 @@ def __init__(self) -> None: self._fdv1_fallback_synchronizer: Optional[DataSourceBuilder[Synchronizer]] = None self._store_mode: DataStoreMode = DataStoreMode.READ_ONLY self._data_store: Optional[FeatureStore] = None + self._override_source: Optional[OverrideSourceBuilder] = None def initializers(self, initializers: Optional[List[DataSourceBuilder[Initializer]]]) -> "ConfigBuilder": """ @@ -80,6 +85,38 @@ def data_store(self, data_store: FeatureStore, store_mode: DataStoreMode) -> "Co self._store_mode = store_mode return self + def overrides(self, source: OverrideSourceBuilder) -> "ConfigBuilder": + """ + Configures the SDK with an override source. Flag overrides are currently experimental + and subject to change. + + The source supplies flag and segment definitions that take precedence over data + received from LaunchDarkly on a per-key basis. Overrides let an operator force one or + more flags to a known state on a running client, whether or not the client can reach + LaunchDarkly. Flags not present in the override data are unaffected. + + The override source is not a data source. It has no effect on the client's + initialization status or data source status. Configuring it changes nothing until the + source actually supplies an override. At most one override source is accepted. + + When the client is offline, the source is not built or started: no overrides are served, + and a configuration error in the source is not reported. The SDK logs a warning instead. + + Example, with the file-based source and the SDK's default data sources: + :: + + from ldclient import datasystem + from ldclient.integrations.overrides import FileOverrideSourceBuilder + + source = FileOverrideSourceBuilder(['/etc/launchdarkly/overrides.json']) + config = Config(sdk_key, datasystem_config=datasystem.default().overrides(source).build()) + + :param source: a builder for the override source, such as + :class:`ldclient.integrations.overrides.FileOverrideSourceBuilder` + """ + self._override_source = source + return self + def build(self) -> DataSystemConfig: """ Builds the data system configuration. @@ -90,6 +127,7 @@ def build(self) -> DataSystemConfig: fdv1_fallback_synchronizer=self._fdv1_fallback_synchronizer, data_store_mode=self._store_mode, data_store=self._data_store, + override_source=self._override_source, ) diff --git a/ldclient/evaluation.py b/ldclient/evaluation.py index 18aa9e29..a65b9c83 100644 --- a/ldclient/evaluation.py +++ b/ldclient/evaluation.py @@ -57,6 +57,12 @@ def reason(self) -> dict: * ``inExperiment``: This is true if the evaluation resulted in an experiment rollout and served one of the variations in the experiment. Otherwise it is false. + * ``overrideAffected``: This is true if a flag override affected the evaluation, directly + or indirectly. It is present only when true. An evaluation is affected when the flag + itself, a prerequisite flag at any depth, or a segment consulted during rule matching + came from the SDK's override store. Flag overrides are currently experimental and + subject to change. + * ``bigSegmentsStatus``: describes the validity of Big Segment information, if and only if the flag evaluation required querying at least one Big Segment; otherwise it returns None. Allowable values are defined in :class:`BigSegmentsStatus`. For more information, read the diff --git a/ldclient/impl/async_evaluator.py b/ldclient/impl/async_evaluator.py index ead01e08..31f3466c 100644 --- a/ldclient/impl/async_evaluator.py +++ b/ldclient/impl/async_evaluator.py @@ -15,7 +15,8 @@ _match_clause_by_kind, _match_single_context_value, _maybe_negate, - check_targets + check_targets, + mark_override_affected ) from ldclient.impl.events.types import EventFactory from ldclient.impl.model import * @@ -56,15 +57,23 @@ def __init__( async def evaluate(self, flag: FeatureFlag, context: Context, event_factory: EventFactory) -> EvalResult: state = EvalResult() state.original_flag_key = flag.key + # Reading the flag's own definition is the first read of the evaluation, so the marking + # starts from the flag's override marker. + state.override_affected = flag.is_override try: state.detail = await self._evaluate(flag, context, state, event_factory) except EvaluationException as e: if self.__logger is not None: self.__logger.error('Could not evaluate flag "%s": %s' % (flag.key, e.message)) state.detail = EvaluationDetail(None, None, {'kind': 'ERROR', 'errorKind': e.error_kind}) + # An evaluation that fails is still marked when it read an override definition. + if state.override_affected: + mark_override_affected(state.detail) return state if state.big_segments_status is not None: state.detail.reason['bigSegmentsStatus'] = state.big_segments_status + if state.override_affected: + mark_override_affected(state.detail) return state async def _evaluate(self, flag: FeatureFlag, context: Context, state: EvalResult, event_factory: EventFactory) -> EvaluationDetail: @@ -120,14 +129,26 @@ async def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: log.warning("Missing prereq flag: " + prereq_key) failed_prereq = prereq else: - state.depth += 1 - prereq_res = await self._evaluate(prereq_flag, context, state, event_factory) - state.depth -= 1 + # The prerequisite is an evaluation in its own right. Its marking starts from + # its own definition and covers only the definitions its subtree reads. The + # parent's marking is restored and merged afterward, so the marking propagates + # upward only. A failure inside the subtree still merges what was read. + parent_override_affected = state.override_affected + state.override_affected = prereq_flag.is_override + try: + state.depth += 1 + prereq_res = await self._evaluate(prereq_flag, context, state, event_factory) + state.depth -= 1 + finally: + prereq_override_affected = state.override_affected + state.override_affected = parent_override_affected or prereq_override_affected + if prereq_override_affected: + mark_override_affected(prereq_res) # Note that if the prerequisite flag is off, we don't consider it a match no matter what its # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: failed_prereq = prereq - event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag) + event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected) state.add_event(event) if failed_prereq: return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key} @@ -146,7 +167,14 @@ async def _clause_matches_context(self, clause: Clause, context: Context, state: if clause.op == 'segmentMatch': for seg_key in clause.values: segment = await self.__get_segment(seg_key) - if segment is not None and await self._segment_matches_context(segment, context, state): + if segment is None: + continue + # The segment definition was read at this point, so an override segment marks the + # evaluation whether or not it matches. A segment that does not match still shapes + # the outcome, for example through a negated clause. + if segment.is_override: + state.override_affected = True + if await self._segment_matches_context(segment, context, state): return _maybe_negate(clause, True) return _maybe_negate(clause, False) diff --git a/ldclient/impl/datasystem/__init__.py b/ldclient/impl/datasystem/__init__.py index 26be520d..6799351f 100644 --- a/ldclient/impl/datasystem/__init__.py +++ b/ldclient/impl/datasystem/__init__.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from ldclient.impl.aio.concurrency import AsyncEvent + from ldclient.impl.overrides.layer import OverrideLayer from ldclient.impl.listeners import Listeners from ldclient.interfaces import ( @@ -158,6 +159,17 @@ def environment_id(self) -> Optional[str]: """ raise NotImplementedError + @property + @abstractmethod + def override_layer(self) -> Optional["OverrideLayer"]: + """ + Returns the override layer when the data system was built with an override source, + and None otherwise. The store served by :attr:`store` overlays this layer. The client + consults the layer directly before it has LaunchDarkly data, so that a flag the layer + holds is served and a flag it does not hold gets the not-ready handling. + """ + raise NotImplementedError + class AsyncDataSystem(Protocol): """ @@ -232,6 +244,15 @@ def store(self) -> AsyncReadOnlyStore: """ raise NotImplementedError + @property + @abstractmethod + def override_layer(self) -> Optional["OverrideLayer"]: + """ + Returns the override layer when the data system was built with an override source, + and None otherwise. See :attr:`DataSystem.override_layer`. + """ + raise NotImplementedError + class DiagnosticAccumulator(Protocol): def record_stream_init(self, timestamp, duration, failed): diff --git a/ldclient/impl/datasystem/async_fdv1.py b/ldclient/impl/datasystem/async_fdv1.py index 96fbc660..a7bd1455 100644 --- a/ldclient/impl/datasystem/async_fdv1.py +++ b/ldclient/impl/datasystem/async_fdv1.py @@ -124,6 +124,11 @@ async def stop(self): def store(self) -> AsyncReadOnlyStore: return self._store_view + @property + def override_layer(self) -> None: + # Overrides are an option of the FDv2 data system only. + return None + def set_diagnostic_accumulator(self, diagnostic_accumulator: DiagnosticAccumulator): """ Sets the diagnostic accumulator for streaming initialization metrics. diff --git a/ldclient/impl/datasystem/async_fdv2.py b/ldclient/impl/datasystem/async_fdv2.py index d6b6c3e3..f190a67e 100644 --- a/ldclient/impl/datasystem/async_fdv2.py +++ b/ldclient/impl/datasystem/async_fdv2.py @@ -39,6 +39,11 @@ ) from ldclient.impl.datasystem.store import _decode from ldclient.impl.listeners import Listeners +from ldclient.impl.overrides import ( + AsyncOverrideStoreView, + OverrideLayer, + OverrideSinkImpl +) from ldclient.impl.util import _LD_FD_FALLBACK_HEADER, _Fail, log from ldclient.interfaces import ( AsyncFeatureStore, @@ -48,7 +53,9 @@ DataSourceErrorKind, DataSourceState, DataStoreMode, - DataStoreStatus + DataStoreStatus, + FlagChange, + OverrideSource ) from ldclient.versioned_data_kind import VersionedDataKind @@ -271,6 +278,21 @@ def __init__( self._store_view = _AsyncReadOnlyStoreView(self._store) + # Set only when an override source is configured. See the sync FDv2 for the roles of + # the layer, the overlay, and the source. Building the source here makes an invalid + # source configuration a construction error. + self._override_source: Optional[OverrideSource] = None + self._override_layer: Optional[OverrideLayer] = None + self._override_sink: Optional[OverrideSinkImpl] = None + self._overlay: Optional[AsyncOverrideStoreView] = None + if data_system_config.override_source is not None and not self._disabled: + self._override_source = data_system_config.override_source.build(config) + self._override_layer = OverrideLayer() + self._overlay = AsyncOverrideStoreView(self._store_view, self._override_layer) + self._store.set_external_dependencies(self._override_layer) + elif data_system_config.override_source is not None: + log.warning("Offline mode is enabled; the override source is not started and no overrides are served") + # Concurrency self._stop_event = AsyncEvent() self._lock = AsyncLock() @@ -315,11 +337,50 @@ def start(self, set_on_ready: AsyncEvent): self._stop_event.clear() + if self._override_source is not None and self._override_layer is not None: + # The source runs on its own threads. Flag change notifications are delivered on + # the event loop, like every other change notification of the async client. The + # merged view for the change computation reads the in-memory store, which is the + # only store that can be read synchronously. While a persistent store is the + # active store, the dependency fan-out is limited to the in-memory data, and the + # directly changed keys are still notified. + loop = asyncio.get_running_loop() + listeners = self._flag_change_listeners + + def notify(key: str) -> None: + try: + loop.call_soon_threadsafe(listeners.notify, FlagChange(key)) + except RuntimeError: + # A reload can finish after the client closed. The loop is closed then, + # and the notification has nobody left to reach. + log.debug("Dropped the flag change notification for %s because the event loop is closed", key) + + self._override_sink = OverrideSinkImpl(self._override_layer, self._store._memory_store, notify, listeners.has_listeners) + try: + self._override_source.start(self._override_sink) + except Exception as e: + # The SDK's own file-based source does not raise from start; it logs a failed + # load and retries it. A custom source could, and a client in an incident is + # better served by running without its overrides than by not starting. + log.error("Unable to start the override source; the client runs without overrides: %s", e) + self._close_override_source("Unable to close the override source after it failed to start") + # The source may have supplied a snapshot before it failed. Clearing the layer + # removes it, so the client runs without overrides as logged, and a late + # snapshot from the closed source is ignored. + self._override_sink.close_and_clear() + # Start the main coordination loop self._runner.spawn("AsyncFDv2-main", lambda: self._run_main_loop(set_on_ready)) async def stop(self): """Stop the AsyncFDv2 data system and all the work it is coordinating.""" + if self._override_source is not None: + # The sink stops accepting snapshots first, so a snapshot that the source supplies + # while it shuts down cannot update the layer or notify listeners. + if self._override_sink is not None: + self._override_sink.close() + self._close_override_source("Unable to close the override source") + self._stop_event.set() async with self._lock: @@ -335,6 +396,18 @@ async def stop(self): # Close the store await self._store.close() + def _close_override_source(self, failure_message: str) -> None: + # A custom source can raise from close. That is logged and not raised, so the rest of + # the client still stops, or still starts after a failed start. + source = self._override_source + self._override_source = None + if source is None: + return + try: + source.close() + except Exception as e: + log.error("%s: %s", failure_message, e) + async def _run_main_loop(self, set_on_ready: AsyncEvent): """Main coordination loop that manages initializers and synchronizers.""" try: @@ -631,9 +704,18 @@ async def reader(): @property def store(self) -> AsyncReadOnlyStore: - """Get the underlying store for flag evaluation.""" + """ + Get the store for flag evaluation. When an override source is configured, this is the + overlay that serves override entries in preference to LaunchDarkly data. + """ + if self._overlay is not None: + return self._overlay return self._store_view + @property + def override_layer(self) -> Optional[OverrideLayer]: + return self._override_layer + async def data_availability(self) -> DataAvailability: # type: ignore[override] """Reports what form of data is currently available, awaiting the store's readiness so a persistent store populated by another process is recognized diff --git a/ldclient/impl/datasystem/fdv1.py b/ldclient/impl/datasystem/fdv1.py index 09ca6939..9d0008a1 100644 --- a/ldclient/impl/datasystem/fdv1.py +++ b/ldclient/impl/datasystem/fdv1.py @@ -233,6 +233,11 @@ def stop(self): def store(self) -> ReadOnlyStore: return self._store_view + @property + def override_layer(self) -> None: + # Overrides are an option of the FDv2 data system only. + return None + @property def environment_id(self) -> Optional[str]: return self._data_source_update_sink.environment_id diff --git a/ldclient/impl/datasystem/fdv2.py b/ldclient/impl/datasystem/fdv2.py index 1fbcbd33..ae8281f2 100644 --- a/ldclient/impl/datasystem/fdv2.py +++ b/ldclient/impl/datasystem/fdv2.py @@ -17,6 +17,11 @@ ) from ldclient.impl.datasystem.store import Store, _decode from ldclient.impl.listeners import Listeners +from ldclient.impl.overrides import ( + OverrideLayer, + OverrideSinkImpl, + OverrideStoreView +) from ldclient.impl.repeating_task import RepeatingTask from ldclient.impl.rwlock import ReadWriteLock from ldclient.impl.util import _LD_FD_FALLBACK_HEADER, _Fail, log @@ -27,6 +32,8 @@ DataStoreMode, DataStoreStatus, FeatureStore, + FlagChange, + OverrideSource, ReadOnlyStore, Synchronizer ) @@ -258,6 +265,27 @@ def __init__( self._store_view = _ReadOnlyStoreView(self._store) + # The following are set only when an override source is configured. The layer holds the + # override entries. The overlay serves them in preference to the store's data at the + # store read boundary. The source populates the layer at runtime. The override system + # has no effect on initialization status or data availability. Building the source + # here makes an invalid source configuration a construction error, like any other + # invalid component configuration. + self._override_source: Optional[OverrideSource] = None + self._override_layer: Optional[OverrideLayer] = None + self._override_sink: Optional[OverrideSinkImpl] = None + self._overlay: Optional[OverrideStoreView] = None + if data_system_config.override_source is not None and not self._disabled: + self._override_source = data_system_config.override_source.build(config) + self._override_layer = OverrideLayer() + self._overlay = OverrideStoreView(self._store_view, self._override_layer) + # A LaunchDarkly change to a flag or segment that an override entry depends on + # must notify that entry's flag as well. The store reaches those entries through + # the layer's dependency graph. + self._store.set_external_dependencies(self._override_layer) + elif data_system_config.override_source is not None: + log.warning("Offline mode is enabled; the override source is not started and no overrides are served") + def _create_store(self, flag_change_listeners: Listeners, change_set_listeners: Listeners) -> Store: return Store(flag_change_listeners, change_set_listeners) @@ -289,6 +317,28 @@ def start(self, set_on_ready: Event): self._stop_event.clear() + if self._override_source is not None and self._override_layer is not None: + # The source starts before the main thread, so a source that loads synchronously + # has its overrides in place before the client begins evaluating. + self._override_sink = OverrideSinkImpl( + self._override_layer, + self._store_view, + lambda key: self._flag_change_listeners.notify(FlagChange(key)), + self._flag_change_listeners.has_listeners, + ) + try: + self._override_source.start(self._override_sink) + except Exception as e: + # The SDK's own file-based source does not raise from start; it logs a failed + # load and retries it. A custom source could, and a client in an incident is + # better served by running without its overrides than by not starting. + log.error("Unable to start the override source; the client runs without overrides: %s", e) + self._close_override_source("Unable to close the override source after it failed to start") + # The source may have supplied a snapshot before it failed. Clearing the layer + # removes it, so the client runs without overrides as logged, and a late + # snapshot from the closed source is ignored. + self._override_sink.close_and_clear() + # Start the main coordination thread main_thread = Thread( target=self._run_main_loop, @@ -301,6 +351,13 @@ def start(self, set_on_ready: Event): def stop(self): """Stop the FDv2 data system and all associated threads.""" + if self._override_source is not None: + # The sink stops accepting snapshots first, so a snapshot that the source supplies + # while it shuts down cannot update the layer or notify listeners. + if self._override_sink is not None: + self._override_sink.close() + self._close_override_source("Unable to close the override source") + self._stop_event.set() with self._lock.write(): @@ -628,6 +685,18 @@ def _record_environment_id(self, environment_id: Optional[str]): with self._lock.write(): self._environment_id = environment_id + def _close_override_source(self, failure_message: str) -> None: + # A custom source can raise from close. That is logged and not raised, so the rest of + # the client still stops, or still starts after a failed start. + source = self._override_source + self._override_source = None + if source is None: + return + try: + source.close() + except Exception as e: + log.error("%s: %s", failure_message, e) + @property def environment_id(self) -> Optional[str]: """Get the environment ID reported by LaunchDarkly, if known.""" @@ -636,9 +705,18 @@ def environment_id(self) -> Optional[str]: @property def store(self) -> ReadOnlyStore: - """Get the underlying store for flag evaluation.""" + """ + Get the store for flag evaluation. When an override source is configured, this is the + overlay that serves override entries in preference to LaunchDarkly data. + """ + if self._overlay is not None: + return self._overlay return self._store_view + @property + def override_layer(self) -> Optional[OverrideLayer]: + return self._override_layer + __all__ = [ 'ConditionDirective', diff --git a/ldclient/impl/datasystem/store.py b/ldclient/impl/datasystem/store.py index b6532605..3904312b 100644 --- a/ldclient/impl/datasystem/store.py +++ b/ldclient/impl/datasystem/store.py @@ -10,7 +10,11 @@ from collections import defaultdict from typing import Any, Callable, Dict, List, Optional, Set -from ldclient.impl.dependency_tracker import DependencyTracker, KindAndKey +from ldclient.impl.dependency_tracker import ( + DependencyTracker, + ExternalDependencies, + KindAndKey +) from ldclient.impl.listeners import Listeners from ldclient.impl.model.entity import ModelEntity from ldclient.impl.rwlock import ReadWriteLock @@ -182,6 +186,10 @@ def __init__( # Used to track dependencies between items in the store self._dependency_tracker = DependencyTracker() + # A second dependency graph, such as the override layer's, whose items may depend on + # items in this store. Set once at construction of the data system, before any data. + self._external_dependencies: Optional[ExternalDependencies] = None + # Listeners for events self._flag_change_listeners = flag_change_listeners self._change_set_listeners = change_set_listeners @@ -195,6 +203,40 @@ def __init__( # Thread synchronization self._lock = threading.RLock() + def set_external_dependencies(self, dependencies: Optional[ExternalDependencies]) -> None: + """ + Registers a second dependency graph whose items may depend on items in this store. A + change to an item in this store then also notifies the external items that depend on + it, directly or through any chain of dependencies in either graph. + """ + with self._lock: + self._external_dependencies = dependencies + + def _add_affected_items(self, items_out: Set[KindAndKey], initial_modified_item: KindAndKey) -> None: + """ + Adds the item and everything that depends on it to the set, following this store's + dependency graph and the external one. Every item reached through either graph is + expanded through both, so a dependency chain may cross between them any number of times. + """ + external = self._external_dependencies + if external is None: + self._dependency_tracker.add_affected_items(items_out, initial_modified_item) + return + + pending: List[KindAndKey] = [initial_modified_item] + while pending: + current = pending.pop() + # Each graph gets its own working set: a tracker does not expand an item that is + # already present, so sharing one set would stop the second graph at the start. + via_store: Set[KindAndKey] = set() + self._dependency_tracker.add_affected_items(via_store, current) + via_external: Set[KindAndKey] = set() + external.add_affected_items(via_external, current) + for reached in via_store | via_external: + if reached not in items_out: + items_out.add(reached) + pending.append(reached) + def selector(self) -> Selector: """Returns the current selector.""" with self._lock: @@ -282,7 +324,7 @@ def _apply_delta( kind, key, item ) if has_listeners: - self._dependency_tracker.add_affected_items( + self._add_affected_items( affected_items, KindAndKey(kind=kind, key=key) ) @@ -353,11 +395,11 @@ def _compute_changed_items_for_full_data_set( # If either is missing or versions differ, it's a change if old_item is None or new_item is None: - self._dependency_tracker.add_affected_items( + self._add_affected_items( affected_items, KindAndKey(kind=kind, key=key) ) elif old_item.get("version", 0) != new_item.get("version", 0): - self._dependency_tracker.add_affected_items( + self._add_affected_items( affected_items, KindAndKey(kind=kind, key=key) ) diff --git a/ldclient/impl/dependency_tracker.py b/ldclient/impl/dependency_tracker.py index 23d6b0d5..10601862 100644 --- a/ldclient/impl/dependency_tracker.py +++ b/ldclient/impl/dependency_tracker.py @@ -1,6 +1,7 @@ -from typing import Dict, List, NamedTuple, Optional, Set, Union +from typing import Dict, List, NamedTuple, Optional, Protocol, Set, Union from ldclient.impl.model.clause import Clause +from ldclient.impl.model.entity import ModelEntity from ldclient.impl.model.feature_flag import FeatureFlag from ldclient.impl.model.segment import Segment from ldclient.versioned_data_kind import FEATURES, SEGMENTS, VersionedDataKind @@ -11,6 +12,20 @@ class KindAndKey(NamedTuple): key: str +class ExternalDependencies(Protocol): + """ + A second dependency graph whose items may depend on items held elsewhere, such as the + override layer, whose entries may depend on LaunchDarkly data. A store that is handed one + extends its change notifications to the external items that depend on a changed item. + """ + + def add_affected_items(self, items_out: Set[KindAndKey], initial_modified_item: KindAndKey) -> None: + """ + Adds the given item and every item in this graph that directly or indirectly depends on + it to the set. Items already in the set are not expanded again. + """ + + class DependencyTracker: """ The DependencyTracker is responsible for tracking both up and downstream @@ -23,7 +38,7 @@ def __init__(self): self.__children: Dict[KindAndKey, Set[KindAndKey]] = {} self.__parents: Dict[KindAndKey, Set[KindAndKey]] = {} - def update_dependencies_from(self, from_kind: VersionedDataKind, from_key: str, from_item: Optional[Union[dict, FeatureFlag, Segment]]): + def update_dependencies_from(self, from_kind: VersionedDataKind, from_key: str, from_item: Optional[Union[dict, ModelEntity]]): """ Updates the dependency graph when an item has changed. @@ -83,7 +98,7 @@ def reset(self): self.__parents.clear() @staticmethod - def compute_dependencies_from(from_kind: VersionedDataKind, from_item: Optional[Union[dict, FeatureFlag, Segment]]) -> Set[KindAndKey]: + def compute_dependencies_from(from_kind: VersionedDataKind, from_item: Optional[Union[dict, ModelEntity]]) -> Set[KindAndKey]: """ @param from_kind [String] @param from_item [LaunchDarkly::Impl::Model::FeatureFlag, LaunchDarkly::Impl::Model::Segment] diff --git a/ldclient/impl/evaluator.py b/ldclient/impl/evaluator.py index 77a811d4..bbce2260 100644 --- a/ldclient/impl/evaluator.py +++ b/ldclient/impl/evaluator.py @@ -17,7 +17,8 @@ _maybe_negate, _variation_index_for_context, check_targets, - error_reason + error_reason, + mark_override_affected ) from ldclient.impl.events.types import EventFactory from ldclient.impl.model import * @@ -57,15 +58,23 @@ def __init__( def evaluate(self, flag: FeatureFlag, context: Context, event_factory: EventFactory) -> EvalResult: state = EvalResult() state.original_flag_key = flag.key + # Reading the flag's own definition is the first read of the evaluation, so the marking + # starts from the flag's override marker. + state.override_affected = flag.is_override try: state.detail = self._evaluate(flag, context, state, event_factory) except EvaluationException as e: if self.__logger is not None: self.__logger.error('Could not evaluate flag "%s": %s' % (flag.key, e.message)) state.detail = EvaluationDetail(None, None, {'kind': 'ERROR', 'errorKind': e.error_kind}) + # An evaluation that fails is still marked when it read an override definition. + if state.override_affected: + mark_override_affected(state.detail) return state if state.big_segments_status is not None: state.detail.reason['bigSegmentsStatus'] = state.big_segments_status + if state.override_affected: + mark_override_affected(state.detail) return state def _evaluate(self, flag: FeatureFlag, context: Context, state: EvalResult, event_factory: EventFactory) -> EvaluationDetail: @@ -118,14 +127,26 @@ def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: EvalR log.warning("Missing prereq flag: " + prereq_key) failed_prereq = prereq else: - state.depth += 1 - prereq_res = self._evaluate(prereq_flag, context, state, event_factory) - state.depth -= 1 + # The prerequisite is an evaluation in its own right. Its marking starts from + # its own definition and covers only the definitions its subtree reads. The + # parent's marking is restored and merged afterward, so the marking propagates + # upward only. A failure inside the subtree still merges what was read. + parent_override_affected = state.override_affected + state.override_affected = prereq_flag.is_override + try: + state.depth += 1 + prereq_res = self._evaluate(prereq_flag, context, state, event_factory) + state.depth -= 1 + finally: + prereq_override_affected = state.override_affected + state.override_affected = parent_override_affected or prereq_override_affected + if prereq_override_affected: + mark_override_affected(prereq_res) # Note that if the prerequisite flag is off, we don't consider it a match no matter what its # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: failed_prereq = prereq - event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag) + event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected) state.add_event(event) if failed_prereq: return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key} @@ -144,7 +165,14 @@ def _clause_matches_context(self, clause: Clause, context: Context, state: EvalR if clause.op == 'segmentMatch': for seg_key in clause.values: segment = self.__get_segment(seg_key) - if segment is not None and self._segment_matches_context(segment, context, state): + if segment is None: + continue + # The segment definition was read at this point, so an override segment marks the + # evaluation whether or not it matches. A segment that does not match still shapes + # the outcome, for example through a negated clause. + if segment.is_override: + state.override_affected = True + if self._segment_matches_context(segment, context, state): return _maybe_negate(clause, True) return _maybe_negate(clause, False) diff --git a/ldclient/impl/evaluator_common.py b/ldclient/impl/evaluator_common.py index 04dbebfd..7385f4e8 100644 --- a/ldclient/impl/evaluator_common.py +++ b/ldclient/impl/evaluator_common.py @@ -25,7 +25,7 @@ # prerequisite evaluations, and the cached state of any Big Segments query that we may have # ended up having to do for the context. class EvalResult: - __slots__ = ['detail', 'events', 'big_segments_status', 'big_segments_membership', 'original_flag_key', 'prereq_stack', 'segment_stack', 'depth', 'prerequisites'] + __slots__ = ['detail', 'events', 'big_segments_status', 'big_segments_membership', 'original_flag_key', 'prereq_stack', 'segment_stack', 'depth', 'prerequisites', 'override_affected'] def __init__(self): self.detail = None @@ -37,6 +37,11 @@ def __init__(self): self.segment_stack = None # type: Optional[List[str]] self.depth = 0 self.prerequisites = [] # type: List[str] + # True when the evaluation in progress has read a flag or segment definition that carries + # the override marker. While a prerequisite is evaluated, this holds the marking of the + # prerequisite's own subtree. The evaluator merges it into the parent's marking afterward, + # so the marking propagates upward only. + self.override_affected = False def record_prerequisite(self, key: str): if self.depth == 0: @@ -255,3 +260,22 @@ def check_targets(flag: FeatureFlag, context: Context) -> Optional[EvaluationDet def error_reason(error_kind: str) -> dict: return {'kind': 'ERROR', 'errorKind': error_kind} + + +def is_override_definition(item: Any) -> bool: + """ + Reports whether a flag or segment read from the store came from the override layer. The + data systems decode the dicts a store holds into model objects before the client reads + them, but an error path reads the marker this way so that an unexpected item cannot make + it fail too. + """ + return getattr(item, 'is_override', False) is True + + +def mark_override_affected(detail: EvaluationDetail) -> None: + """ + Sets the override-affected indicator on the evaluation reason. The indicator is present + only when true, so a reason that was not affected by an override is left untouched and + serializes exactly as before. + """ + detail.reason['overrideAffected'] = True diff --git a/ldclient/impl/events/event_processor_common.py b/ldclient/impl/events/event_processor_common.py index 95ea278f..75f634b1 100644 --- a/ldclient/impl/events/event_processor_common.py +++ b/ldclient/impl/events/event_processor_common.py @@ -181,7 +181,7 @@ def make_summary_event(self, summary: EventSummary): } counters = [] for ckey, cval in flag_data.counters.items(): - variation, version = ckey + variation, version, override_affected = ckey counter = {'count': cval.count, 'value': cval.value} if variation is not None: counter['variation'] = variation @@ -189,6 +189,9 @@ def make_summary_event(self, summary: EventSummary): counter['unknown'] = True else: counter['version'] = version + # The marker is present only when true, like the unknown marker. + if override_affected: + counter['overrideAffected'] = True counters.append(counter) flag_data_out['counters'] = counters flags_out[key] = flag_data_out @@ -266,9 +269,11 @@ def _process_event(self, event: EventInput): context = event.context if not event.exclude_from_summaries: self._outbox.add_to_summary(event) - if event.track_events: + # An override-affected evaluation appears only in the summary counters. It produces + # no individual feature event and no debug event, even when the flag requests them. + if event.track_events and not event.override_affected: full_event = event - if self._should_debug_event(event): + if not event.override_affected and self._should_debug_event(event): debug_event = DebugEvent(event) elif isinstance(event, EventInputIdentify): if self._omit_anonymous_contexts: diff --git a/ldclient/impl/events/event_summarizer.py b/ldclient/impl/events/event_summarizer.py index acf3a88f..ca2b5b5e 100644 --- a/ldclient/impl/events/event_summarizer.py +++ b/ldclient/impl/events/event_summarizer.py @@ -22,10 +22,16 @@ def __repr__(self) -> str: # used only in test debugging return "EventSummaryCounter(%d, %s)" % (self.count, self.value) +# A counter is keyed by variation index, flag version, and whether the evaluations it counts +# were override-affected. The marker is part of the key, so override-affected and ordinary +# evaluations of the same flag, variation, and version accumulate into separate counters. +CounterKey = Tuple[Optional[int], Optional[int], bool] + + class EventSummaryFlag: __slots__ = ['context_kinds', 'default', 'counters'] - def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[Tuple[Optional[int], Optional[int]], EventSummaryCounter]): + def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[CounterKey, EventSummaryCounter]): self.context_kinds = context_kinds self.counters = counters self.default = default @@ -71,7 +77,7 @@ def summarize_event(self, event: EventInputEvaluation): if c is not None: flag_data.context_kinds.add(c.kind) - counter_key = (event.variation, None if event.flag is None else event.flag.version) + counter_key = (event.variation, None if event.flag is None else event.flag.version, event.override_affected) counter = flag_data.counters.get(counter_key) if counter is None: counter = EventSummaryCounter(1, event.value) diff --git a/ldclient/impl/events/types.py b/ldclient/impl/events/types.py index d0883c47..6f754a23 100644 --- a/ldclient/impl/events/types.py +++ b/ldclient/impl/events/types.py @@ -52,6 +52,7 @@ class EventInputEvaluation(EventInput): "track_events", "sampling_ratio", "exclude_from_summaries", + "override_affected", ] def __init__( @@ -66,6 +67,7 @@ def __init__( default_value: Any, prereq_of: Optional[FeatureFlag] = None, track_events: bool = False, + override_affected: bool = False, ): super().__init__(timestamp, context, 1 if flag is None else flag.sampling_ratio) self.key = key @@ -79,6 +81,11 @@ def __init__( self.exclude_from_summaries = ( False if flag is None else flag.exclude_from_summaries ) + # True when a flag override affected this evaluation, directly or through a + # prerequisite or segment. The event processor keys on this scalar alone, never on the + # reason: such an evaluation produces no individual feature or debug event and is + # counted in a summary counter that carries the override-affected marker. + self.override_affected = override_affected def to_debugging_dict(self) -> dict: return { @@ -94,6 +101,7 @@ def to_debugging_dict(self) -> dict: "track_events": self.track_events, "exclude_from_summaries": self.exclude_from_summaries, "sampling_ratio": self.sampling_ratio, + "override_affected": self.override_affected, } @@ -155,6 +163,7 @@ def new_eval_event( detail: EvaluationDetail, default_value: Any, prereq_of_flag: Optional[FeatureFlag] = None, + override_affected: bool = False, ) -> EventInputEvaluation: add_experiment_data = self.is_experiment(flag, detail.reason) return EventInputEvaluation( @@ -168,6 +177,7 @@ def new_eval_event( default_value, prereq_of_flag, flag.track_events or add_experiment_data, + override_affected, ) def new_default_event( @@ -176,6 +186,7 @@ def new_default_event( context: Context, default_value: Any, reason: Optional[dict], + override_affected: bool = False, ) -> EventInputEvaluation: return EventInputEvaluation( self._timestamp_fn(), @@ -188,6 +199,7 @@ def new_default_event( default_value, None, flag.track_events, + override_affected, ) def new_unknown_flag_event( diff --git a/ldclient/impl/integrations/files/filedata.py b/ldclient/impl/integrations/files/filedata.py new file mode 100644 index 00000000..347d4cc9 --- /dev/null +++ b/ldclient/impl/integrations/files/filedata.py @@ -0,0 +1,1303 @@ +""" +File reading, parsing, and merging logic for components that load flag and segment data from +local files. The file-based override source is built on it. The existing file data sources +keep their own implementation and behavior. + +A data file is a JSON or YAML document with optional ``flags``, ``flagValues``, and +``segments`` members. ``flags`` and ``segments`` hold full definitions keyed by key, and a +definition marked as deleted fails the load of its file. ``flagValues`` maps a flag key to a +single value. It expands into a full flag definition that returns that value for every +context. The file name extension decides the format: a file whose name ends in ``.yaml`` or +``.yml`` is YAML, and any other file is JSON. +""" + +import codecs +import hashlib +import json +import os +import threading +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Callable, Dict, List, Optional, Set, Tuple + +from ldclient.impl.model import FeatureFlag, Segment +from ldclient.impl.repeating_task import RepeatingTask +from ldclient.impl.util import log + +have_yaml = False +try: + import yaml + + have_yaml = True +except ImportError: + pass + +have_watchdog = False +try: + import watchdog + import watchdog.events + import watchdog.observers + + have_watchdog = True +except ImportError: + pass + + +# A settle window long enough to coalesce the burst of change notifications produced by a +# single file edit, and short enough to stay responsive. +DEFAULT_DEBOUNCE_DELAY = 0.1 + +# Bounds how long a failed reload can go uncorrected when no further change notification +# arrives, for example when the failure came from reading a file mid-write. Reading a local +# file is cheap, so this can be short. +DEFAULT_RETRY_DELAY = 1.0 + +# The interval between attempts to watch a directory that could not be watched, for example +# because it does not exist yet. +_WATCH_RETRY_INTERVAL = 1.0 + +# The file name extensions of YAML files, compared without regard to case. Any other file is JSON. +YAML_EXTENSIONS = frozenset([".yaml", ".yml"]) + +# The watchdog event types that can change a file's content or presence. Opening or reading a +# file also produces events, and a reload reads the files, so those must not count as changes. +# watchdog 7 reports a change of a file's metadata, such as a new modification time from a +# touch, as "attrib", where earlier versions reported "modified". A read does not produce it. +_CHANGE_EVENT_TYPES = frozenset(["created", "modified", "moved", "deleted", "closed", "attrib"]) + +# How long, in seconds, the watchdog observer waits for further notifications before it delivers +# the ones it has. watchdog 7 on Windows keeps waiting while notifications arrive, so a change is +# reported this late, and not at all while another file in the same directory changes more often +# than this. The watchdog default is one second. +_OBSERVER_TIMEOUT = 0.1 + + +class DuplicateKeysHandling(str, Enum): + """ + Determines what happens when the same flag or segment key appears in more than one file. + Flag overrides are currently experimental and subject to change. + """ + + FAIL = "fail" + """A duplicated key causes the load to fail.""" + + IGNORE = "ignore" + """Only the first occurrence of a duplicated key is used, in the order the files were given.""" + + +class FileDataError(Exception): + """Base class for the errors raised while loading file data.""" + + +class FileReadError(FileDataError): + """ + Indicates that one of the source files could not be read or parsed. It distinguishes a + per-file failure from a failure to merge the files' contents. + """ + + def __init__(self, path: str, message: str): + super().__init__("%s [%s]" % (message, path)) + self.path = path + + +class DeletedEntryError(FileReadError): + """ + Indicates that a file holds a flag or segment entry marked as deleted. An override is removed + by removing its entry, so such a file fails to load like one that cannot be parsed. + """ + + def __init__(self, path: str, kind_name: str, key: str): + FileDataError.__init__(self, '%s "%s" in %s is marked as deleted; remove the entry to remove its override' % (kind_name, key, path)) + self.path = path + self.kind_name = kind_name + self.key = key + + +class DuplicateKeyError(FileDataError): + """Indicates that the same key appears in more than one file and the handling is FAIL.""" + + +@dataclass +class Document: + """The parsed form of a single data file.""" + + flags: Dict[str, FeatureFlag] = field(default_factory=dict) + flag_values: Dict[str, Any] = field(default_factory=dict) + segments: Dict[str, Segment] = field(default_factory=dict) + + +@dataclass +class DocumentSummary: + """Counts the entries the merge kept from one document.""" + + flags: int = 0 + segments: int = 0 + + +@dataclass +class FileSummary: + """Describes one configured file after a load.""" + + path: str + present: bool = False + """False when the file does not exist and missing files are skipped.""" + flags: int = 0 + segments: int = 0 + + +@dataclass +class MergeResult: + """ + The merged items from one or more documents. The dictionaries preserve document order: + all of one document's items precede the next document's. + """ + + flags: Dict[str, FeatureFlag] = field(default_factory=dict) + segments: Dict[str, Segment] = field(default_factory=dict) + documents: List[DocumentSummary] = field(default_factory=list) + """For each input document in order, the number of entries the merge kept from it.""" + files: List[FileSummary] = field(default_factory=list) + """Set by the loader. Describes each configured file in order.""" + + +def abs_file_paths(paths: List[str]) -> List[str]: + """Converts each of the given paths to an absolute path.""" + return [os.path.abspath(p) for p in paths] + + +def make_flag_with_value(key: str, value: Any) -> FeatureFlag: + """ + Expands a flag-key-to-value entry into a full flag definition that returns the given value + for every context. The flag is on, has the value as its only variation, and serves that + variation as its fallthrough. + """ + return FeatureFlag({"key": key, "version": 1, "on": True, "fallthrough": {"variation": 0}, "variations": [value]}) + + +def is_yaml_path(path: str) -> bool: + """ + Whether a data file is a YAML document. The end of the file name decides: a name that ends in + ``.yaml`` or ``.yml``, in any case, is YAML, including a name that is only that, such as + ``.yaml``. Any other file, including one with no extension, is JSON. + """ + return os.path.basename(path).lower().endswith(tuple(YAML_EXTENSIONS)) + + +def read_file(path: str) -> Document: + """Reads and parses a single data file, in the format its file name extension selects.""" + try: + with open(path, "rb") as f: + raw = f.read() + except OSError as e: + raise FileReadError(path, "unable to read file: %s" % e) + try: + document = parse_document(raw, is_yaml_path(path)) + except Exception as e: + raise FileReadError(path, "error parsing file: %s" % e) + _reject_deleted_entries(path, document) + return document + + +def has_content(raw: bytes) -> bool: + """Whether the raw content of a file holds anything other than white space and a byte order mark.""" + return raw.removeprefix(codecs.BOM_UTF8).strip() != b"" + + +def parse_document(raw: bytes, is_yaml: bool = False) -> Document: + """ + Parses the raw content of a data file as YAML when ``is_yaml`` is true, which requires the + ``pyyaml`` package, and as JSON otherwise. The file name extension decides the format; see + :func:`is_yaml_path`. A byte order mark at the start is ignored. + + Whether a file with no content is valid depends on its format, like the rest of its content. + A JSON file with no content, or only white space, is not a JSON document, so it fails to + parse. A YAML file with no content, or with only comments or a document marker, is a + document with no entries. + """ + document = _parse_content(raw, is_yaml) + return Document() if document is None else document + + +def _parse_content(raw: bytes, is_yaml: bool) -> Optional[Document]: + """ + The parse behind :func:`parse_document`. Returns None when the YAML parser reads the content + as nothing, as it does for a file with no content, only comments, or ``null``, rather than as + a document, even one with no members such as ``{}``. A JSON document must be an object. + """ + text = raw.decode("utf-8-sig") + if is_yaml: + if not have_yaml: + raise ValueError("the pyyaml package is required to read YAML") + parsed = yaml.safe_load(text) + else: + if not has_content(raw): + # Not a JSON document. A file that is rewritten in place can be empty for a moment, + # and the failure keeps the last good data until the file is complete. + raise ValueError("the file has no content, which is not a JSON document") + parsed = json.loads(text) + if not isinstance(parsed, dict): + raise ValueError("the document must be an object") + if parsed is None: + return None + if not isinstance(parsed, dict): + raise ValueError("the document must be an object") + + document = Document() + for key, item in _member_items(parsed, "flags").items(): + document.flags[key] = FeatureFlag(_definition(item, key, "flag")) + for key, value in _member_items(parsed, "flagValues").items(): + document.flag_values[key] = value + for key, item in _member_items(parsed, "segments").items(): + document.segments[key] = Segment(_definition(item, key, "segment")) + return document + + +def _member_items(parsed: dict, name: str) -> Dict[str, Any]: + member = parsed.get(name) + if member is None: + return {} + if not isinstance(member, dict): + raise ValueError('"%s" must be an object' % name) + for key in member: + if not isinstance(key, str): + raise ValueError('"%s" has a key that is not a string: %r' % (name, key)) + return member + + +def _definition(item: Any, key: str, kind_name: str) -> dict: + """ + Validates the shape of a flag or segment entry. The map key is the entry's key for every + purpose: the entry is stored under it, duplicates are detected by it, and the definition's + own key is set to it, so that evaluations report the flag by that key. The file format + allows a definition to omit its version, which the model requires, so a version of 1 is + filled in. + """ + if not isinstance(item, dict): + raise ValueError('%s "%s" must be an object' % (kind_name, key)) + item["key"] = key + if "version" not in item: + item["version"] = 1 + return item + + +def merge(documents: List[Document], duplicate_keys_handling: DuplicateKeysHandling) -> MergeResult: + """ + Combines the items of the given documents in order, expanding flag-value entries into full + flag definitions and applying the given duplicate keys handling. + """ + result = MergeResult() + seen_flags: Set[str] = set() + seen_segments: Set[str] = set() + + def insert(items: Dict[str, Any], seen: Set[str], kind_name: str, key: str, item: Any) -> bool: + if key in seen: + if duplicate_keys_handling == DuplicateKeysHandling.IGNORE: + return False + raise DuplicateKeyError("%s '%s' is specified by multiple files" % (kind_name, key)) + items[key] = item + seen.add(key) + return True + + for document in documents: + summary = DocumentSummary() + for key, flag in document.flags.items(): + if insert(result.flags, seen_flags, "flag", key, flag): + summary.flags += 1 + for key, value in document.flag_values.items(): + if insert(result.flags, seen_flags, "flag", key, make_flag_with_value(key, value)): + summary.flags += 1 + for key, segment in document.segments.items(): + if insert(result.segments, seen_segments, "segment", key, segment): + summary.segments += 1 + result.documents.append(summary) + return result + + +def load_files(paths: List[str], duplicate_keys_handling: DuplicateKeysHandling, skip_missing_paths: bool = False) -> MergeResult: + """ + Reads, parses, and merges all of the given files in order. Raises :class:`FileReadError` + when a file cannot be read or parsed, :class:`DuplicateKeyError` when a key is duplicated + and the handling is FAIL. When ``skip_missing_paths`` is true, a file that does not exist + contributes no entries instead of failing the load. + """ + merged, _ = _parse_files(_read_files(paths, skip_missing_paths), duplicate_keys_handling) + return merged + + +@dataclass +class _FileRead: + """What one load read from one configured file.""" + + path: str + raw: Optional[bytes] + """The raw content, or None when the file does not exist and missing files are skipped.""" + + +def _read_files(paths: List[str], skip_missing_paths: bool) -> List[_FileRead]: + """ + Reads the raw content of all of the given files in order. Raises :class:`FileReadError` + when a file cannot be read, including a missing file unless ``skip_missing_paths`` is true. + """ + return [_read_file(path, skip_missing_paths) for path in paths] + + +def _read_file(path: str, skip_missing_paths: bool) -> _FileRead: + """Reads the raw content of one file, as :func:`_read_files` does.""" + try: + with open(path, "rb") as f: + return _FileRead(path, f.read()) + except FileNotFoundError: + if skip_missing_paths: + log.debug("File %s does not exist; it contributes no data", path) + return _FileRead(path, None) + raise FileReadError(path, "unable to read file: the file does not exist") + except OSError as e: + raise FileReadError(path, "unable to read file: %s" % e) + + +def _parse_file(path: str, raw: bytes) -> Optional[Document]: + """Parses what was read from a file that exists. Returns None when the parser read nothing.""" + is_yaml = is_yaml_path(path) + try: + document = _parse_content(raw, is_yaml) + except Exception as e: + raise FileReadError(path, "error parsing file as %s, the format of its name: %s" % ("YAML" if is_yaml else "JSON", e)) + if document is not None: + _reject_deleted_entries(path, document) + return document + + +def _reject_deleted_entries(path: str, document: Document) -> None: + """ + Raises :class:`DeletedEntryError` for the first flag or segment entry of a parsed file that is + marked as deleted. The override layer never holds a deleted item, and an override is removed + by removing its entry, so such an entry is a mistake in the file that fails its load. The file + data sources do not use this module and keep accepting such entries. + """ + for kind_name, items in (("flag", document.flags), ("segment", document.segments)): + for key, item in items.items(): + if item.deleted: + raise DeletedEntryError(path, kind_name, key) + + +def _parse_files(contents: List[_FileRead], duplicate_keys_handling: DuplicateKeysHandling) -> Tuple[MergeResult, bytes]: + """Parses and merges the content that :func:`_read_files` read.""" + return _merge_files([(read, None if read.raw is None else _parse_file(read.path, read.raw)) for read in contents], duplicate_keys_handling) + + +def _merge_files(parsed: List[Tuple[_FileRead, Optional[Document]]], duplicate_keys_handling: DuplicateKeysHandling) -> Tuple[MergeResult, bytes]: + """ + Merges the parsed files. It also returns a digest of the files that were read, so a caller + can tell whether a later load read the same. One read feeds both the digest and the parse, + so the digest can never disagree with the content that was parsed. The digest covers each + path, whether its file exists, and its bytes: the same bytes in a file with another name can + be in another format, and parse differently. + """ + documents: List[Document] = [] + files: List[FileSummary] = [] + hasher = hashlib.sha256() + for read, document in parsed: + path = os.fsencode(read.path) + hasher.update(len(path).to_bytes(8, "big")) + hasher.update(path) + if read.raw is None: + hasher.update(b"-") + files.append(FileSummary(path=read.path)) + continue + hasher.update(b"+") + hasher.update(len(read.raw).to_bytes(8, "big")) + hasher.update(read.raw) + documents.append(Document() if document is None else document) + files.append(FileSummary(path=read.path, present=True)) + + merged = merge(documents, duplicate_keys_handling) + # The documents are the present files in order. Copy their counts onto the file summaries. + next_document = 0 + for summary in files: + if summary.present: + summary.flags = merged.documents[next_document].flags + summary.segments = merged.documents[next_document].segments + next_document += 1 + merged.files = files + return merged, hasher.digest() + + +FileState = Optional[Tuple[int, int]] + + +def _observe(path: str) -> FileState: + """ + Observes the state of a file: its modification time and size, through any symbolic link, or + None when it does not exist or cannot be examined. + """ + try: + info = os.stat(path) + return (info.st_mtime_ns, info.st_size) + except OSError: + return None + + +def _observe_all(paths: List[str]) -> List[FileState]: + """Observes the state of each file, as :func:`_observe` does.""" + return [_observe(path) for path in paths] + + +# A file is held back for at most this many retry delays after a load first held it back, until +# a load holds nothing back. +_MAX_HOLD_RETRY_DELAYS = 5 + + +@dataclass +class _EmptyFile: + """A file whose content is in effect and that loads have found with no entries.""" + + since: Optional[float] = None + """ + Since when the file has had no entries without being written, on the reloader's clock, or + None when the last load read it with content, even content that failed to parse, or found + it missing. A read that fails keeps it, as nothing is known about the content then. + """ + state: FileState = None + """The file's state then: its modification time and size, through any symbolic link.""" + first_held: Optional[float] = None + """ + When a load first held the file back, or None. It bounds how long the file is held back. + It is kept while loads hold files back, so files emptied in turn stay bounded, and cleared by + any load that holds nothing back, including one that fails. + """ + + +class Reloader: + """ + Owns the reload cycle for a set of data files. It serializes reloads, debounces change + signals, retains the last good result on failure by not calling ``apply``, retries after + failures, and skips applications that would change nothing. + + A file can be rewritten in place: a writer truncates it and then writes the new content, and a + load in between finds it empty. When retries are enabled, a load that finds a file with no + entries, because it is empty or holds only what its parser reads as nothing (a YAML comment or + document marker), while that file's content is in effect, holds the file back: it keeps the + current data without reporting a failure, and the other files' changes wait too, since a load + applies all files or none. The file is held back until it has stayed so, without being written, + for the retry delay; a change of its modification time or size starts the delay over. Each file + is held back for at most five retry delays after a load first held it back, until a load holds + nothing back, so a file that keeps being rewritten is applied in the end without cutting short + another file's rewrite. A file whose content is not in effect, such as a placeholder that has + always been empty, is never held back. Every file is read and parsed, even after another file + fails, before any is held back, so files emptied together wait together, and a file that cannot + be read or parsed fails the load, with the first failure, even while another is held back. The + clock is read after each file's own read, so a slow read of another file does not shorten a + file's delay. + + The worker thread starts on the first :meth:`reload_now` or :meth:`trigger` call, so a + reloader that is constructed but never used does not leak a thread. + """ + + def __init__( + self, + paths: List[str], + duplicate_keys_handling: DuplicateKeysHandling, + apply: Callable[[MergeResult], None], + on_error: Optional[Callable[[Exception], None]] = None, + skip_missing_paths: bool = False, + debounce_delay: float = 0.0, + retry_delay: float = 0.0, + skip_unchanged: bool = False, + clock: Callable[[], float] = time.monotonic, + ): + """ + :param paths: the files to load, in order. The order determines which file wins under + the duplicate keys handling. + :param duplicate_keys_handling: what to do when the same key appears in more than one file + :param apply: receives each successfully merged result. Calls are serialized. An + exception from it is reported like a load failure and retried, and the result it + rejected is not remembered as the last good one. + :param on_error: receives each distinct failure. Repeats of an identical failure do not + call it again until a success re-arms it. Failures are also logged here. An exception + from it is logged and does not prevent the retry. + :param skip_missing_paths: when true, a configured file that does not exist contributes + no entries. When false, a missing file fails the load like any other read error. + :param debounce_delay: how long to wait after a trigger for further triggers to settle + before reloading. Zero reloads on every trigger. + :param retry_delay: how long to wait after a failed reload before retrying it + automatically, and how long a file whose content is in effect must have no entries, + without being written, before that is applied. Zero disables the automatic retry, and + an empty file is then parsed as it is. + :param skip_unchanged: when true, a load whose raw file contents are identical to the + last applied contents does not call ``apply`` + :param clock: the monotonic clock, in seconds, that times how long files are held back. + Tests replace it; the timers always use ``time.monotonic``. + """ + self._paths = list(paths) + self._duplicate_keys_handling = duplicate_keys_handling + self._apply = apply + self._on_error = on_error + self._skip_missing_paths = skip_missing_paths + self._debounce_delay = debounce_delay + self._retry_delay = retry_delay + self._skip_unchanged = skip_unchanged + self._clock = clock + + # Guards the deadlines, the closed flag, and the worker start. + self._cond = threading.Condition() + self._closed = False + self._started = False + self._debounce_deadline: Optional[float] = None + self._retry_deadline: Optional[float] = None + # Whether the pending retry was armed by a load that held files back. + self._retry_after_hold = False + + # Serializes the load work between reload_now and the worker thread. The fields below are + # only touched while holding it. + self._reload_lock = threading.Lock() + self._last_good_digest: Optional[bytes] = None + self._last_error_message: Optional[str] = None + # Whether each configured file contributed content to the data in effect. A missing file, + # or one with no entries, contributed none. + self._applied_had_content: Dict[str, bool] = {} + # The files whose content is in effect and that loads have found with no entries. All are + # forgotten when data is applied or a load is skipped as unchanged, and a load that holds + # nothing back, even one that fails, starts every file's bound over. + self._empty_files: Dict[str, _EmptyFile] = {} + # Whether the last load held files back, which the log line of the retry it arms names. + self._held_back = False + # Whether the current load has held a file back, even if it then fails. + self._holding_this_load = False + # The delay of the last retry that was armed, for tests. + self._last_requested_retry_delay: Optional[float] = None + + def reload_now(self) -> None: + """ + Loads the files synchronously and applies the result or reports the failure. A failure + arms the same automatic retry as a failed triggered reload. + """ + self._ensure_started() + retry_after, held = self._reload() + if retry_after is not None: + self._arm_retry(retry_after, keep_sooner=True, after_hold=held) + + def trigger(self) -> None: + """ + Signals that the files may have changed. A reload happens after the debounce delay. + Signals that arrive while a reload is pending extend the settle window. + """ + self._ensure_started() + with self._cond: + if self._closed: + return + self._debounce_deadline = time.monotonic() + max(self._debounce_delay, 0.0) + self._cond.notify() + + def close(self) -> None: + """ + Stops the reloader. It does not wait for a reload that is already in progress, so such + a reload may still deliver its result shortly after this returns. A reload that has not + yet reached its callbacks does not invoke them. + """ + with self._cond: + self._closed = True + self._cond.notify_all() + + def _arm_retry(self, retry_after: float, keep_sooner: bool, after_hold: bool) -> None: + """ + Arms the automatic retry for the given time from now, unless retries are off. With + ``keep_sooner``, a retry that is already due sooner stays as it is. ``after_hold`` says + that the load held files back, which the retry's log line names. + """ + if self._retry_delay <= 0: + return + with self._cond: + if self._closed: + return + deadline = time.monotonic() + retry_after + if keep_sooner and self._retry_deadline is not None and self._retry_deadline <= deadline: + return + self._retry_deadline = deadline + self._retry_after_hold = after_hold + self._last_requested_retry_delay = retry_after + self._cond.notify() + + def _ensure_started(self) -> None: + with self._cond: + if self._closed or self._started: + return + self._started = True + thread = threading.Thread(target=self._run, name="ldclient.filedata.reloader", daemon=True) + thread.start() + + def _run(self) -> None: + while True: + is_retry = False + after_hold = False + with self._cond: + while True: + if self._closed: + return + now = time.monotonic() + deadlines = [d for d in (self._debounce_deadline, self._retry_deadline) if d is not None] + if len(deadlines) == 0: + self._cond.wait() + continue + next_deadline = min(deadlines) + if next_deadline > now: + self._cond.wait(next_deadline - now) + continue + if self._debounce_deadline is not None and self._debounce_deadline <= now: + # A triggered reload supersedes a pending retry. It either succeeds, or + # it fails or holds files back and arms a fresh retry below. + self._debounce_deadline = None + self._retry_deadline = None + is_retry = False + else: + self._retry_deadline = None + is_retry = True + after_hold = self._retry_after_hold + break + + if is_retry and after_hold: + log.debug("Reading flag data again after a file was found empty") + elif is_retry: + log.debug("Retrying flag data load after earlier failure") + else: + log.info("Reloading flag data after detecting a change") + try: + retry_after, held = self._reload() + except Exception as e: + log.exception("Unexpected error while reloading flag data: %s", e) + retry_after, held = None, False + if retry_after is not None: + self._arm_retry(retry_after, keep_sooner=False, after_hold=held) + + def _is_closed(self) -> bool: + with self._cond: + return self._closed + + def _reload(self) -> Tuple[Optional[float], bool]: + """ + Performs one full load of all configured files. Returns the time after which to load + again, or None when the load succeeded, including a skipped no-op application: the retry + delay after a failure, or the time a hold lasts. Also returns whether the load held files + back. + """ + with self._reload_lock: + if self._is_closed(): + return None, False + self._held_back = False + self._holding_this_load = False + reached: List[str] = [] + retry_after = self._load(reached) + if not self._holding_this_load: + # A load that holds nothing back, whether it applies, is skipped or fails, starts the + # bound over for every file that it read or tried to read, including one that + # failed to be read or parsed: such a file is no longer held back, so its next + # rewrite is a new one. + for path in reached: + empty = self._empty_files.get(path) + if empty is not None: + empty.first_held = None + return retry_after, self._held_back + + def _load(self, reached: List[str]) -> Optional[float]: + """ + The body of :meth:`_reload`. It records in ``reached`` each file that it reads or tries to + read, which is every configured file, since a load reads all of them even after a failure. + """ + # Every file is read, parsed and classified, even after another file fails, before the + # load fails, holds files back or merges them. So every file that is found with no + # entries starts its delay in this load, and every file is reached, so a load that holds + # nothing back starts every file's bound over. The load fails with the first failure. + failure: Optional[Exception] = None + + def note_failure(e: Exception) -> None: + nonlocal failure + if failure is None: + failure = e + else: + # The load fails with the first failure. The others are logged, so that a second + # broken file is visible too. + log.debug("Unable to load flag data, besides the first failure: %s", e) + + held_back: List[str] = [] + parsed: List[Tuple[_FileRead, Optional[Document]]] = [] + had_content: Dict[str, bool] = {} + for path in self._paths: + reached.append(path) + try: + read = _read_file(path, self._skip_missing_paths) + except Exception as e: + # Nothing is known about the content, so the file's empty state, if any, stays. + note_failure(e) + continue + if read.raw is None: + # A missing file is not an empty one. + self._end_empty(read.path) + had_content[read.path] = False + parsed.append((read, None)) + continue + # A file with no content, or with content that its parser reads as nothing, such as + # YAML with only a comment, has no document and so no entries. A read can see that + # much of a file that is being written. Content with no document is invalid JSON, so + # it is only parsed when the file is not held back. {} is a document. + document: Optional[Document] = None + has_document = has_content(read.raw) + if has_document: + try: + document = _parse_file(read.path, read.raw) + except FileReadError as e: + # Content that cannot be parsed is not an empty file either. + self._end_empty(read.path) + note_failure(e) + continue + has_document = document is not None + if has_document: + self._end_empty(read.path) + # The clock is read after this file's own read, so that a slow read of another file + # does not shorten its delay. + elif self._should_hold_back(read.path, self._clock()): + held_back.append(read.path) + self._holding_this_load = True + continue + # The file was read and is not held back, even if it then fails to parse. + had_content[read.path] = has_document + if not has_content(read.raw): + try: + document = _parse_file(read.path, read.raw) + except FileReadError as e: + note_failure(e) + continue + parsed.append((read, document)) + if failure is not None: + return self._fail(failure) + if len(held_back) > 0: + return self._hold_back(held_back, self._clock()) + + try: + merged, digest = _merge_files(parsed, self._duplicate_keys_handling) + except Exception as e: + return self._fail(e) + + # A close may have happened while the files were being read. Deliver nothing then. + if self._is_closed(): + return None + + # A success right after a failure applies even when the content is unchanged since the + # last success. The consumer heard about the failure and only an application tells it + # that things are good again. + recovering = self._last_error_message is not None + if self._skip_unchanged and not recovering and digest == self._last_good_digest: + # The data in effect is confirmed, as by an application, so it is remembered the same + # way, and the bounds start over. + self._remember_applied(had_content) + return None + + # Nothing is remembered until the consumer has accepted the result. A result the + # consumer rejects must not become the baseline that skip-unchanged compares against, + # and must not count as a recovery. + try: + self._apply(merged) + except Exception as e: + return self._fail(e) + self._last_error_message = None + self._last_good_digest = digest + self._remember_applied(had_content) + return None + + def _end_empty(self, path: str) -> None: + """ + Ends the empty state of a file that a load read with content, even content that cannot be + parsed, or found missing, so that it starts a new delay when it is found empty again. Its + bound is kept until a load holds nothing back. + """ + empty = self._empty_files.get(path) + if empty is not None: + empty.since = None + + def _remember_applied(self, had_content: Dict[str, bool]) -> None: + """Records what each file contributed to the data in effect, and forgets the empty files.""" + self._applied_had_content.update(had_content) + self._empty_files.clear() + + def _should_hold_back(self, path: str, now: float) -> bool: + """ + Decides whether a file that a load found with no entries is held back. A file whose + content is in effect and that has none now may be in the middle of an in-place rewrite. + It is held back until it has stayed so, without being written, for the retry delay. A + change of its state, its modification time and size read through any symbolic link, + means that it was written since a load last found it so, for example rewritten and + truncated again between two loads, so the delay starts over. A file that has stayed so + for the whole delay is not held back. Each file is held back for at most five retry + delays after a load first held it back, so that a file that keeps being written while + every load finds it empty, or files that are emptied in turn, are not held back without + end. The bound starts over only at a load that holds nothing back, for every file, whether + the load applies, is skipped or fails. A load reads every file even after one fails. A file whose content is not in effect + is parsed right away, and so is any file when retries are off, since nothing would read it + again. + """ + if self._retry_delay <= 0 or not self._applied_had_content.get(path, False): + return False + state = _observe(path) + empty = self._empty_files.get(path) + if empty is None: + empty = _EmptyFile() + self._empty_files[path] = empty + if empty.since is None or empty.state != state: + empty.since = now + empty.state = state + if now - empty.since >= self._retry_delay: + # The file stayed empty for the whole delay. Its bound starts over only at a load that + # holds nothing back, so that files emptied in turn do not keep each other held back. + return False + if empty.first_held is not None and now - empty.first_held >= _MAX_HOLD_RETRY_DELAYS * self._retry_delay: + return False + if empty.first_held is None: + empty.first_held = now + return True + + def _hold_back(self, held_back: List[str], now: float) -> Optional[float]: + """ + Keeps the current data for a load that holds files back. Whatever the files' format, + nothing is applied or reported. The returned time arms the retry for when the first of + the files has stayed so for the retry delay, or has been held back for as long as it may + be, plus a tenth of the retry delay, so that a timer that fires a little early still + finds the time over. The change signal for a write starts a load sooner. + """ + if self._is_closed(): + return None + until = float("inf") + for path in held_back: + log.debug("File %s has no entries, possibly while it is being rewritten; waiting for it to be written before applying it", path) + empty = self._empty_files[path] + since = now if empty.since is None else empty.since + first_held = now if empty.first_held is None else empty.first_held + until = min(until, since + self._retry_delay, first_held + _MAX_HOLD_RETRY_DELAYS * self._retry_delay) + self._held_back = True + return max(until - now, 0.0) + self._retry_delay / 10 + + def _fail(self, err: Exception) -> Optional[float]: + if self._is_closed(): + return None + # With automatic retries, a persistent failure would repeat the same log entry and the + # same callback on every attempt. Repeats of an identical failure are logged at debug + # level and do not call on_error again. + message = str(err) + if message == self._last_error_message: + log.debug("Unable to load flag data: %s", err) + return self._retry_delay + self._last_error_message = message + log.error("Unable to load flag data: %s", err) + if self._on_error is not None: + try: + self._on_error(err) + except Exception as e: + # A consumer that raises while it reports a failure must not stop the retry. + log.error("Error while reporting a flag data load failure: %s", e) + return self._retry_delay + + +class Poller: + """ + Detects changes to a set of files by examining them on a fixed interval. A change to the + modification time or the size of any file invokes the callback. A file that appears or + disappears is also a change. Use it where file system change notifications are not + available or not reliable. + + Detection is generous. The callback can run for a change that does not alter the + effective data. Feed it into a :class:`Reloader`, whose debouncing and skip-unchanged + handling absorb the excess. + """ + + def __init__(self, paths: List[str], interval: float, on_change: Callable[[], None]): + self._paths = list(paths) + self._on_change = on_change + # The files are examined once here, so only later changes invoke the callback. + self._last = _observe_all(self._paths) + self._task = RepeatingTask.at_interval("ldclient.filedata.poll", interval, interval, self._poll) + + def start(self) -> None: + """Starts the polling thread.""" + self._task.start() + + def close(self) -> None: + """ + Stops the poller. It does not wait for an examination or a callback that is in + progress, so the callback can run once more shortly after this returns. + """ + self._task.stop() + + def _poll(self) -> None: + current = _observe_all(self._paths) + changed = current != self._last + self._last = current + if changed: + self._on_change() + + +# The most symbolic links followed from one configured file, as in the operating system's own +# limit, so that a loop of links ends. +_MAX_LINKS = 40 + + +def _link_target(path: str) -> Optional[str]: + """ + The path of the file that a configured symbolic link finally points to, with its directory + resolved to a real path, or None when the configured file is not a link, the target's + directory does not exist, or the links loop. The target file itself need not exist, so that + a link created ahead of its target is followed once the target appears. + + A relative link is relative to the real directory that holds it, so it is joined to that + directory's real path. Normalizing the path as written would take a ".." to the wrong place + when a directory in that path is itself a link. + """ + try: + if not os.path.islink(path): + return None + target = path + for _ in range(_MAX_LINKS): + directory = os.path.dirname(target) + if not os.path.isdir(directory): + return None + real_directory = os.path.realpath(directory) + if not os.path.islink(target): + return os.path.join(real_directory, os.path.basename(target)) + target = os.path.join(real_directory, os.readlink(target)) + return None + except (OSError, ValueError): + return None + + +def _event_path(value: Any) -> Optional[str]: + """The path carried by a watchdog event as text, or None when the event has none.""" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, str): + return value + return None + + +class Watcher: + """ + Detects changes to a set of files through file system change notifications, using the + ``watchdog`` package. The directory of each file is watched, so a file that does not exist + yet is picked up when it appears. + + A directory that cannot be watched, for example because it does not exist yet or because + the notification mechanism cannot be started, is attempted again on an interval. A watched + directory that is deleted has its watch dropped and set up again once the directory exists. + When a retry sets up a watch, the callback runs once, so that changes made while the watch + was not in place are picked up. + + A configured file can be a symbolic link to a file in another directory. The directory of + the link's target is watched as well, for the target's path, so that an edit to the target + is reported. The links are resolved again after every reported change, after a retry sets up + watches, and when a directory that holds a target is deleted, as happens when a Kubernetes + ConfigMap volume is updated. A link that cannot be resolved, for example because its target's + directory does not exist yet or was deleted, is resolved again on the retry interval until + its target is watched, and the callback then runs once. + + A change that none of those events reports is not followed: a link further along the chain + that is pointed elsewhere while its old target stays, a configured directory that is itself + a link and is pointed elsewhere, or a configured directory that is renamed and replaced. + Deployments that switch files that way, such as a ``current`` link to a release directory, + should use polling, which reads every configured path on each interval. + + Notifications for the watched paths invoke the callback. The callback can run several times + for one logical edit, so feed it into a :class:`Reloader`. + """ + + def __init__(self, paths: List[str], on_change: Callable[[], None]): + if not have_watchdog: + raise RuntimeError("the watchdog package is required to watch files for changes") + self._on_change = on_change + self._paths = [os.path.abspath(path) for path in paths] + self._watched_paths: Set[str] = set() + self._directories: Set[str] = set() + for absolute in self._paths: + real_directory = os.path.realpath(os.path.dirname(absolute)) + self._watched_paths.add(os.path.join(real_directory, os.path.basename(absolute))) + self._directories.add(real_directory) + + watcher = self + + class _Handler(watchdog.events.FileSystemEventHandler): + def on_any_event(self, event): + watcher._handle_event(event) + + self._handler = _Handler() + self._observer = watchdog.observers.Observer(timeout=_OBSERVER_TIMEOUT) + + # Guards the state below. It is never held while the observer is called, because the + # observer holds its own lock while it dispatches an event to the handler. + self._lock = threading.Lock() + self._closed = False + self._observer_running = False + # The watch on each directory that currently has one. + self._watches: Dict[str, Any] = {} + # The directories that have no watch yet. The retry schedule runs while this is not empty. + self._pending_directories: Set[str] = set(self._directories) + self._retry_task: Optional[RepeatingTask] = None + # Serializes the watch setup against close, so that a setup in progress cannot start + # anything after close has stopped the observer. + self._setup_lock = threading.Lock() + # The files that the configured symbolic links point to, and the directories that are + # watched only because a target is in them. A directory that is also configured is in + # _directories and keeps its own watch. + self._target_paths: Set[str] = set() + self._target_directories: Set[str] = set() + # True while a configured link's target is not watched, because the link cannot be + # resolved or its target's directory cannot be watched. The retry schedule runs while it + # is true. + self._links_unresolved = False + # The last reason logged for each directory that cannot be watched yet, so that a retry + # that fails the same way logs at debug level. A directory leaves it once it is watched. + self._unwatchable_messages: Dict[str, str] = {} + + self._set_up_watches(is_retry=False) + self._refresh_links() + + def _set_up_watches(self, is_retry: bool) -> None: + """ + Starts the observer when it is not running, then sets up a watch on each pending + directory. Whatever cannot be set up stays pending and is attempted again on the retry + schedule. When a retry sets up a watch, the callback runs once, because the files may + have changed while the watch was not in place. + """ + with self._setup_lock: + with self._lock: + if self._closed: + return + running = self._observer_running + if not running: + try: + self._observer.start() + except Exception as e: + log.error("Unable to start watching files for changes: %s", e) + self._ensure_retry_scheduled() + return + with self._lock: + self._observer_running = True + + with self._lock: + pending = sorted(self._pending_directories) + armed = False + for directory in pending: + watch = self._schedule(directory) + if watch is None: + continue + with self._lock: + self._pending_directories.discard(directory) + self._watches[directory] = watch + armed = True + + with self._lock: + if len(self._pending_directories) > 0: + self._ensure_retry_scheduled_locked() + if is_retry and armed: + self._refresh_links() + self._on_change() + + def _schedule(self, directory: str) -> Optional[Any]: + """Sets up a watch on a directory. Returns the watch, or None when the directory cannot be watched yet.""" + # The observer accepts a watch on a directory that does not exist and fails later when + # it starts the watch, so the check happens here first. + if not os.path.isdir(directory): + self._log_unwatchable(directory, 'Cannot watch directory "%s" for changes yet because it does not exist' % directory) + return None + try: + watch = self._observer.schedule(self._handler, directory, recursive=False) + except Exception as e: + self._log_unwatchable(directory, 'Cannot watch directory "%s" for changes yet: %s' % (directory, e)) + return None + # watchdog 6 on Windows starts a watch's reader thread and returns before the thread has + # asked for its first notification, and a change made before then is not reported. + # Yielding lets the thread make that request before the setup goes on, for example to + # the initial load. + time.sleep(0) + with self._lock: + self._unwatchable_messages.pop(directory, None) + return watch + + def _log_unwatchable(self, directory: str, message: str) -> None: + """ + Logs why a directory cannot be watched. The retry attempts it again every interval, so + an unchanged reason for the same directory is logged at debug level after the first time, + until the directory is watched. + """ + with self._lock: + repeated = self._unwatchable_messages.get(directory) == message + self._unwatchable_messages[directory] = message + if repeated: + log.debug(message) + else: + log.warning(message) + + def _ensure_retry_scheduled(self) -> None: + with self._lock: + self._ensure_retry_scheduled_locked() + + def _ensure_retry_scheduled_locked(self) -> None: + if self._closed or self._retry_task is not None: + return + self._retry_task = RepeatingTask.at_interval("ldclient.filedata.watch-retry", _WATCH_RETRY_INTERVAL, _WATCH_RETRY_INTERVAL, self._retry_pending) + self._retry_task.start() + + def _retry_pending(self) -> None: + self._set_up_watches(is_retry=True) + with self._lock: + links_unresolved = self._links_unresolved + if links_unresolved and self._refresh_links(): + # The target that a retry watches may have changed while it was not watched. + self._on_change() + with self._lock: + # The schedule ends once everything is watched, the targets of the configured links + # included. It is set up again when a directory becomes pending or a link cannot be + # resolved later. The check and the stop happen under the lock, so a directory that + # becomes pending at the same time either keeps this schedule or starts a new one. + if len(self._pending_directories) == 0 and not self._links_unresolved and self._observer_running and self._retry_task is not None: + self._retry_task.stop() + self._retry_task = None + + def _directory_lost(self, directory: str) -> None: + """ + Drops the watch on a directory that no longer exists and arms the retry that sets it up + again once the directory exists. + """ + with self._lock: + if self._closed: + return + stale = self._watches.pop(directory, None) + self._pending_directories.add(directory) + self._ensure_retry_scheduled_locked() + log.warning('Directory "%s" no longer exists. It is watched again when it exists.', directory) + self._unschedule(directory, stale) + with self._lock: + held_target = any(os.path.dirname(target) == directory for target in self._target_paths) + if held_target: + # The file that a configured link names has changed. + self._refresh_links() + self._on_change() + + def _target_directory_lost(self, directory: str) -> None: + """ + Drops the watch on a directory that held only link targets, resolves the links again, + and runs the callback, since the file that a configured link names has changed. + """ + with self._lock: + if self._closed: + return + stale = self._watches.pop(directory, None) + self._target_directories.discard(directory) + log.debug('Directory "%s" of a linked data file no longer exists. The links are resolved again.', directory) + self._unschedule(directory, stale) + self._refresh_links() + self._on_change() + + def _unschedule(self, directory: str, watch: Optional[Any]) -> None: + if watch is None: + return + # The observer keeps a watch whose directory is gone, and a later watch on the same path + # would reuse it, so it is removed. + try: + self._observer.unschedule(watch) + except Exception as e: + log.debug('Unable to remove the watch on directory "%s": %s', directory, e) + + def _refresh_links(self) -> bool: + """ + Resolves each configured file that is a symbolic link and watches the directory of each + target that is not a configured directory, which has a watch of its own. A directory that + no longer holds a target of a configured link loses its watch. A link that cannot be + resolved, for example because its target's directory does not exist yet, or whose + target's directory cannot be watched, keeps the retry schedule running, which resolves + the links again. Returns True if it set up a watch or if a link now leads to a target it + did not lead to before, which can be in a directory that is already watched. + + This runs on the observer's thread too, which holds the observer's lock, so it takes only + the watcher's own lock and never while the observer is called. + """ + targets = set() + unresolved = [] + for path in self._paths: + target = _link_target(path) + if target is not None: + targets.add(target) + elif os.path.islink(path): + unresolved.append(path) + directories = {os.path.dirname(target) for target in targets} - self._directories + with self._lock: + if self._closed: + return False + new_targets = targets - self._target_paths + self._target_paths = targets + added = sorted(d for d in directories if d not in self._watches) + removed = [d for d in self._target_directories if d not in directories] + self._target_directories = set(directories) + stale = [(d, self._watches.pop(d, None)) for d in removed] + for directory, watch in stale: + self._unschedule(directory, watch) + watched_any = False + unwatched = False + for directory in added: + watch = self._schedule(directory) + if watch is None: + unwatched = True + continue + with self._lock: + stored = self._watches.get(directory) + keep = not self._closed and directory in self._target_directories and stored is None + if keep: + self._watches[directory] = watch + if keep: + watched_any = True + elif stored is None or stored != watch: + # The observer shares one watch per directory, so a watch that another thread + # stored for this directory meanwhile is this one, and removing it would end it. + self._unschedule(directory, watch) + if unresolved: + log.debug('Cannot resolve the target of %s yet. The links are resolved again shortly.', ", ".join(unresolved)) + with self._lock: + if self._closed: + return watched_any + self._links_unresolved = bool(unresolved) or unwatched + if self._links_unresolved: + self._ensure_retry_scheduled_locked() + return watched_any or len(new_targets) > 0 + + def _handle_event(self, event) -> None: + event_type = getattr(event, "event_type", None) + if event_type not in _CHANGE_EVENT_TYPES: + return + src_path = _event_path(getattr(event, "src_path", None)) + if event_type in ("deleted", "moved") and getattr(event, "is_directory", False): + if src_path in self._directories: + self._directory_lost(src_path) + return + with self._lock: + is_target_directory = src_path is not None and src_path in self._target_directories + if src_path is not None and is_target_directory: + self._target_directory_lost(src_path) + return + with self._lock: + target_paths = self._target_paths + for candidate in (src_path, _event_path(getattr(event, "dest_path", None))): + if candidate in self._watched_paths or candidate in target_paths: + # The change can be to a configured link itself, so the links are resolved again + # first. + self._refresh_links() + self._on_change() + return + + def close(self) -> None: + """Stops watching and waits briefly for the observer thread to finish.""" + with self._setup_lock: + with self._lock: + self._closed = True + retry_task = self._retry_task + self._retry_task = None + running = self._observer_running + if retry_task is not None: + retry_task.stop() + self._observer.stop() + # A thread that never started cannot be joined. + if running: + self._observer.join(timeout=5) diff --git a/ldclient/impl/integrations/overrides/__init__.py b/ldclient/impl/integrations/overrides/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ldclient/impl/integrations/overrides/file_override_source.py b/ldclient/impl/integrations/overrides/file_override_source.py new file mode 100644 index 00000000..852e4ffb --- /dev/null +++ b/ldclient/impl/integrations/overrides/file_override_source.py @@ -0,0 +1,153 @@ +""" +The file-based override source. It reads flag and segment overrides from one or more local +files and reloads them when the files change. +""" + +import threading +from enum import Enum +from typing import List, Optional, Union + +from ldclient.impl.integrations.files.filedata import ( + DEFAULT_DEBOUNCE_DELAY, + DEFAULT_RETRY_DELAY, + DuplicateKeysHandling, + FileSummary, + MergeResult, + Poller, + Reloader, + Watcher +) +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSink, OverrideSource + + +class ChangeDetection(str, Enum): + """ + Selects how the file-based override source learns that a file changed. The two modes are + alternatives. Flag overrides are currently experimental and subject to change. + """ + + POLLING = "polling" + """ + The source examines the files on a fixed interval and reloads when the modification time + or the size of a file changes. Polling works on every file system, including network + mounts and directories whose contents are swapped through symbolic links, as Kubernetes + does for mounted ConfigMaps. It is the default. + """ + + WATCHING = "watching" + """ + The source reloads in response to file system change notifications, using the ``watchdog`` + package. It reacts faster than polling. It depends on notifications, which some file + systems do not deliver reliably. + """ + + +class _FileOverrideSource(OverrideSource): + """ + Reads overrides from the configured files and supplies each successful load to the sink as + a full snapshot. A configured file that does not exist contributes no overrides. A file + that cannot be read or parsed, or that holds an entry marked as deleted, fails that load, + the last good snapshot stays in effect, and the load is retried. A file whose content is in + effect and that is found with no entries is held back until it has stayed so for the retry + delay, because it may be in the middle of an in-place rewrite. + """ + + def __init__(self, paths: List[str], duplicate_keys_handling: DuplicateKeysHandling, change_detection: ChangeDetection, poll_interval: float): + self._paths = paths + self._duplicate_keys_handling = duplicate_keys_handling + self._change_detection = change_detection + self._poll_interval = poll_interval + self._reloader: Optional[Reloader] = None + self._change_detector: Optional[Union[Poller, Watcher]] = None + self._lock = threading.Lock() + self._closed = False + + def start(self, sink: OverrideSink) -> None: + def apply(merged: MergeResult) -> None: + sink.set_overrides(merged.flags, merged.segments) + _log_overrides_in_effect(merged) + + reloader = Reloader( + self._paths, + self._duplicate_keys_handling, + apply=apply, + skip_missing_paths=True, + debounce_delay=DEFAULT_DEBOUNCE_DELAY, + retry_delay=DEFAULT_RETRY_DELAY, + skip_unchanged=True, + ) + # Change detection is set up before the initial load, so an edit made while the files + # are first read is picked up by the reload it triggers instead of waiting for a later + # change. + with self._lock: + if self._closed: + return + self._reloader = reloader + if self._change_detection == ChangeDetection.WATCHING: + self._change_detector = Watcher(self._paths, reloader.trigger) + else: + poller = Poller(self._paths, self._poll_interval, reloader.trigger) + poller.start() + self._change_detector = poller + + # The initial load runs synchronously, so overrides present in the files are in effect + # by the time the client constructor returns. A file that does not exist yet + # contributes no overrides. A file that cannot be read or parsed is not fatal: the + # client runs with no overrides, the failure is logged, and the retry recovers once + # the file is readable. + reloader.reload_now() + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + change_detector = self._change_detector + reloader = self._reloader + self._change_detector = None + self._reloader = None + try: + if change_detector is not None: + change_detector.close() + finally: + # The reloader stops even when the change detector fails to close, so no reload + # can run after close. + if reloader is not None: + reloader.close() + + +def _log_overrides_in_effect(merged: MergeResult) -> None: + """ + Reports the overrides now in effect and what each configured file supplied. The reloader + applies a snapshot only when the content changed, so this logs each change once. + """ + details = "; ".join(_file_summary_text(summary) for summary in merged.files) + if len(merged.flags) == 0 and len(merged.segments) == 0: + log.info("Flag overrides: none in effect (%s)", details) + return + log.info("Flag overrides in effect: %s (%s)", _counts_text(len(merged.flags), len(merged.segments)), details) + + +def _file_summary_text(summary: FileSummary) -> str: + if not summary.present: + return "%s: absent" % summary.path + if summary.flags == 0 and summary.segments == 0: + return "%s: no entries" % summary.path + return "%s: %s" % (summary.path, _counts_text(summary.flags, summary.segments)) + + +def _counts_text(flags: int, segments: int) -> str: + """Formats flag and segment counts, for example "2 flags, 1 segment".""" + parts = [] + if flags > 0: + parts.append(_pluralize(flags, "flag")) + if segments > 0: + parts.append(_pluralize(segments, "segment")) + return ", ".join(parts) + + +def _pluralize(count: int, noun: str) -> str: + if count == 1: + return "1 %s" % noun + return "%d %ss" % (count, noun) diff --git a/ldclient/impl/model/feature_flag.py b/ldclient/impl/model/feature_flag.py index ae0b6f75..f7c03f4d 100644 --- a/ldclient/impl/model/feature_flag.py +++ b/ldclient/impl/model/feature_flag.py @@ -1,3 +1,4 @@ +import copy from typing import Any, Dict, List, Optional, Set, Union from ldclient.impl.model.clause import Clause @@ -96,10 +97,12 @@ class FeatureFlag(ModelEntity): '_salt', '_track_events', '_debug_events_until_date', + '_is_override', ] def __init__(self, data: dict): super().__init__(data) + self._is_override = False # In the following logic, we're being somewhat lenient in terms of allowing most properties to # be absent even if they are really required in the schema. That's for backward compatibility # with test logic that constructed incomplete JSON, and also with the file data source which @@ -202,3 +205,24 @@ def exclude_from_summaries(self) -> bool: @property def sampling_ratio(self) -> Optional[int]: return self._sampling_ratio + + @property + def is_override(self) -> bool: + """ + True if this flag definition was supplied by an SDK override source rather than by + LaunchDarkly data. The marker is never part of the JSON representation. Only the SDK + components that manage override entries set it. Components that read a definition + through the store may treat a marked definition the same as any other. + + Flag overrides are currently experimental and subject to change. + """ + return self._is_override + + def with_override_marker(self) -> 'FeatureFlag': + """ + Returns a shallow copy of this flag with the override marker set. The copy shares the + underlying data with this instance, which is left unmarked. + """ + marked = copy.copy(self) + marked._is_override = True + return marked diff --git a/ldclient/impl/model/segment.py b/ldclient/impl/model/segment.py index 6ed1b6a8..709518a2 100644 --- a/ldclient/impl/model/segment.py +++ b/ldclient/impl/model/segment.py @@ -1,3 +1,4 @@ +import copy from typing import Any, List, Optional, Set from ldclient.impl.model.attribute_ref import ( @@ -65,10 +66,12 @@ class Segment(ModelEntity): '_unbounded', '_unbounded_context_kind', '_generation', + '_is_override', ] def __init__(self, data: dict): super().__init__(data) + self._is_override = False # In the following logic, we're being somewhat lenient in terms of allowing most properties to # be absent even if they are really required in the schema. That's for backward compatibility # with test logic that constructed incomplete JSON, and also with the file data source which @@ -137,3 +140,24 @@ def unbounded_context_kind(self) -> Optional[str]: @property def generation(self) -> Optional[int]: return self._generation + + @property + def is_override(self) -> bool: + """ + True if this segment definition was supplied by an SDK override source rather than by + LaunchDarkly data. The marker is never part of the JSON representation. Only the SDK + components that manage override entries set it. Components that read a definition + through the store may treat a marked definition the same as any other. + + Flag overrides are currently experimental and subject to change. + """ + return self._is_override + + def with_override_marker(self) -> 'Segment': + """ + Returns a shallow copy of this segment with the override marker set. The copy shares the + underlying data with this instance, which is left unmarked. + """ + marked = copy.copy(self) + marked._is_override = True + return marked diff --git a/ldclient/impl/overrides/__init__.py b/ldclient/impl/overrides/__init__.py new file mode 100644 index 00000000..b3e7126e --- /dev/null +++ b/ldclient/impl/overrides/__init__.py @@ -0,0 +1,15 @@ +""" +The flag and segment override layer. The layer is a runtime-mutable collection of flag and +segment definitions supplied by an override source. Those definitions take precedence over +LaunchDarkly data at evaluation time. Flag overrides are currently experimental and subject to +change. +""" + +from ldclient.impl.overrides.layer import OverrideLayer +from ldclient.impl.overrides.overlay import ( + AsyncOverrideStoreView, + OverrideStoreView +) +from ldclient.impl.overrides.sink import OverrideSinkImpl + +__all__ = ['AsyncOverrideStoreView', 'OverrideLayer', 'OverrideSinkImpl', 'OverrideStoreView'] diff --git a/ldclient/impl/overrides/layer.py b/ldclient/impl/overrides/layer.py new file mode 100644 index 00000000..e5fa49ae --- /dev/null +++ b/ldclient/impl/overrides/layer.py @@ -0,0 +1,120 @@ +""" +The override store. It holds the entries an override source has loaded, keyed by flag key or +segment key, and is replaced wholesale on each update. +""" + +from typing import Any, Dict, Mapping, Optional, Set, Tuple + +from ldclient.impl.dependency_tracker import DependencyTracker, KindAndKey +from ldclient.impl.model import ModelEntity +from ldclient.impl.rwlock import ReadWriteLock +from ldclient.versioned_data_kind import FEATURES, SEGMENTS, VersionedDataKind + +LayerContents = Dict[VersionedDataKind, Dict[str, ModelEntity]] + + +def _empty_contents() -> LayerContents: + return {FEATURES: {}, SEGMENTS: {}} + + +class DeletedItemError(ValueError): + """ + Indicates that a snapshot holds a flag or segment marked as deleted. The layer never holds a + deleted item, so it rejects the whole snapshot. An override is removed by leaving it out of + the snapshot. + """ + + def __init__(self, kind_name: str, key: str): + super().__init__('%s "%s" is marked as deleted; an override is removed by leaving it out of the snapshot' % (kind_name, key)) + self.kind_name = kind_name + self.key = key + + +class OverrideLayer: + """ + A thread-safe store of override entries, replaced wholesale on each update from an override + source. Each entry is a marked shallow copy of the definition the source supplied, so the + source's own objects are never marked and a source may retain and resupply them. + """ + + def __init__(self): + self._lock = ReadWriteLock() + self._contents: LayerContents = _empty_contents() + # The dependency graph of the override entries, rebuilt with each replacement. The store + # consults it so that a LaunchDarkly change to a prerequisite or a segment also notifies + # the override entries that depend on it. + self._dependencies = DependencyTracker() + # Read without the lock on every evaluation. A single attribute read is atomic, so the + # per-evaluation cost of a configured but unpopulated override layer is negligible. + self._non_empty = False + + def set_all(self, flags: Mapping[str, Any], segments: Mapping[str, Any]) -> Tuple[LayerContents, LayerContents]: + """ + Atomically replaces the entire layer contents. Empty mappings clear the layer. + + Values may be model objects or their JSON dictionary form. A dictionary is decoded with + the model constructor, which raises ``ValueError`` for an invalid definition. A + definition whose key differs from its key in the mapping raises ``ValueError`` too, and a + flag or segment marked as deleted, in either form, raises :class:`DeletedItemError`, + which is a ``ValueError``. Nothing is replaced in any of these cases. + + :return: the previous and the new contents. The returned dictionaries must not be modified. + """ + decoded: Dict[VersionedDataKind, Dict[str, Any]] = { + FEATURES: {key: FEATURES.decode(item) for key, item in flags.items()}, + SEGMENTS: {key: SEGMENTS.decode(item) for key, item in segments.items()}, + } + for kind, items in decoded.items(): + for key, item in items.items(): + if item.deleted: + raise DeletedItemError("flag" if kind == FEATURES else "segment", key) + # Evaluation reads an entry by its mapping key, while the all-flags report and + # analytics events use the definition's key. A mismatch would serve one key's + # override under another, so the whole snapshot is rejected. + if item.key != key: + raise ValueError('the override %s entry "%s" has the key "%s"; the key of each definition must be its key in the snapshot' + % ("flag" if kind == FEATURES else "segment", key, item.key)) + replacement: LayerContents = {kind: {key: item.with_override_marker() for key, item in items.items()} for kind, items in decoded.items()} + count = len(replacement[FEATURES]) + len(replacement[SEGMENTS]) + dependencies = DependencyTracker() + for kind, items in replacement.items(): + for key, item in items.items(): + try: + dependencies.update_dependencies_from(kind, key, item) + except Exception: + # An entry whose dependency fields cannot be read contributes no edges. It is + # still served, and the remaining entries are still tracked. + pass + with self._lock.write(): + previous = self._contents + self._contents = replacement + self._dependencies = dependencies + self._non_empty = count > 0 + return previous, replacement + + def add_affected_items(self, items_out: Set[KindAndKey], initial_modified_item: KindAndKey) -> None: + """ + Adds the given item and every override entry that directly or indirectly depends on it + to the set. The item itself need not be an override entry: a LaunchDarkly flag or + segment that an override entry references is a valid starting point. + """ + with self._lock.read(): + dependencies = self._dependencies + dependencies.add_affected_items(items_out, initial_modified_item) + + def get(self, kind: VersionedDataKind, key: str) -> Optional[ModelEntity]: + """Returns the override entry for a key, or None when the layer has none.""" + if not self._non_empty: + return None + with self._lock.read(): + return self._contents.get(kind, {}).get(key) + + def all(self, kind: VersionedDataKind) -> Dict[str, ModelEntity]: + """Returns the entries of the given kind. The returned dictionary must not be modified.""" + with self._lock.read(): + return self._contents.get(kind, {}) + + @property + def is_empty(self) -> bool: + """True when the layer holds no entries.""" + return not self._non_empty diff --git a/ldclient/impl/overrides/overlay.py b/ldclient/impl/overrides/overlay.py new file mode 100644 index 00000000..c1fd0829 --- /dev/null +++ b/ldclient/impl/overrides/overlay.py @@ -0,0 +1,73 @@ +""" +The overlay at the store read boundary. A read for a key returns the override entry when one +exists and the LaunchDarkly entry otherwise. Targeting rules, prerequisites, and segment matches +behave identically for overridden and ordinary data, because they are the same reads through +the same boundary. +""" + +from typing import Any, Callable, Dict, Optional + +from ldclient.impl.overrides.layer import OverrideLayer +from ldclient.interfaces import AsyncReadOnlyStore, ReadOnlyStore +from ldclient.versioned_data_kind import VersionedDataKind + + +def _merge_all(base_items: Dict[str, Any], override_items: Dict[str, Any]) -> Dict[str, Any]: + """The union of the base items and the override items. The override entry wins for a key present in both.""" + if len(override_items) == 0: + return base_items + merged = dict(base_items) + merged.update(override_items) + return merged + + +class OverrideStoreView(ReadOnlyStore): + """ + Merges an override layer over a base read-only store. + + A per-key read serves an override entry whatever the state of the base store, so an + uninitialized base still serves overrides. An enumeration is the union of the base items + and the layer's items, and the override entry wins for a key present in both, including a + key the base holds as a deleted item. A failure of the base enumeration propagates, as it + does without the overlay: the layer's entries alone would look like a complete result. + """ + + def __init__(self, base: ReadOnlyStore, layer: OverrideLayer): + self._base = base + self._layer = layer + + def get(self, kind: VersionedDataKind, key: str, callback: Callable[[Any], Any] = lambda x: x) -> Any: + item = self._layer.get(kind, key) + if item is not None: + return callback(item) + return self._base.get(kind, key, callback) + + def all(self, kind: VersionedDataKind, callback: Callable[[Any], Any] = lambda x: x) -> Any: + base_items = self._base.all(kind, lambda x: x) + if base_items is None: + # A store returns no result when it cannot read. That stays a failure rather than + # becoming the layer's entries alone. + return callback(None) + return callback(_merge_all(base_items, self._layer.all(kind))) + + @property + def initialized(self) -> bool: + # The override layer never affects initialization status or data availability. + return self._base.initialized + + +class AsyncOverrideStoreView(AsyncReadOnlyStore): + """The async counterpart of :class:`OverrideStoreView`, over an async base store.""" + + def __init__(self, base: AsyncReadOnlyStore, layer: OverrideLayer): + self._base = base + self._layer = layer + + async def get(self, kind: VersionedDataKind, key: str) -> Optional[Any]: + item = self._layer.get(kind, key) + if item is not None: + return item + return await self._base.get(kind, key) + + async def all(self, kind: VersionedDataKind) -> Dict[str, Any]: + return _merge_all(await self._base.all(kind), self._layer.all(kind)) diff --git a/ldclient/impl/overrides/sink.py b/ldclient/impl/overrides/sink.py new file mode 100644 index 00000000..76c71a5a --- /dev/null +++ b/ldclient/impl/overrides/sink.py @@ -0,0 +1,183 @@ +""" +The sink that applies override snapshots to the layer and notifies flag change listeners of the +flags whose evaluation may have changed. +""" + +import threading +from typing import Any, Callable, Dict, Optional, Set + +from ldclient.impl.dependency_tracker import DependencyTracker, KindAndKey +from ldclient.impl.overrides.layer import ( + DeletedItemError, + LayerContents, + OverrideLayer +) +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSink, ReadOnlyStore +from ldclient.versioned_data_kind import FEATURES, SEGMENTS, VersionedDataKind + +MergedView = Dict[VersionedDataKind, Dict[str, Any]] + +_DIFF_KINDS = (FEATURES, SEGMENTS) + + +class OverrideSinkImpl(OverrideSink): + """ + Applies override layer replacements supplied by an override source, then notifies flag + change listeners of every flag whose merged-view evaluation may have changed. Calls are + serialized, so overlapping updates from a source cannot interleave. + """ + + def __init__( + self, + layer: OverrideLayer, + base: ReadOnlyStore, + notify: Callable[[str], None], + has_listeners: Callable[[], bool], + ): + """ + :param layer: the override layer to write to + :param base: the store holding LaunchDarkly data, without the overlay. Merged-view + snapshots for change computation are built from it plus the layer. + :param notify: receives the key of each affected flag + :param has_listeners: reports whether anything listens for flag changes, so the change + computation can be skipped when nothing does + """ + self._layer = layer + self._base = base + self._notify = notify + self._has_listeners = has_listeners + self._lock = threading.Lock() + self._closed = False # guarded by _lock + + def set_overrides(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: # type: ignore[override] + with self._lock: + if self._closed: + log.debug("Ignoring an override snapshot supplied after the override source was closed") + return + try: + self._apply(flags, segments) + except DeletedItemError as e: + # The layer rejected the whole snapshot before changing anything, so nothing is + # notified. The error also reaches the source, as for an invalid definition. + log.error("Rejected an override snapshot; the overrides in effect stay: %s", e) + raise + + def close(self) -> None: + """ + Stops accepting snapshots. Used when the client is closed: a snapshot that the source + supplies while it shuts down is ignored. The layer keeps its contents. + """ + with self._lock: + self._closed = True + + def close_and_clear(self) -> None: + """ + Stops accepting snapshots and clears the layer. Used when the source failed to start: a + snapshot the source supplied before failing is removed, so the client runs without + overrides, and a late snapshot from the closed source is ignored. + """ + with self._lock: + self._closed = True + self._apply({}, {}) + + def _apply(self, flags: Dict[str, Any], segments: Dict[str, Any]) -> None: + # Called with _lock held. + if not self._has_listeners(): + self._layer.set_all(flags, segments) + return + + previous, current = self._layer.set_all(flags, segments) + old_merged = snapshot_merged_view(self._base, previous) + new_merged = snapshot_merged_view(self._base, current) + affected = compute_affected_flags(previous, current, old_merged, new_merged) + if len(affected) > 0: + log.debug("Override update affected %d flag(s)", len(affected)) + for key in sorted(affected): + self._notify(key) + + +def snapshot_merged_view(base: ReadOnlyStore, overrides: LayerContents) -> MergedView: + """ + Captures the merged view of a base store and a layer snapshot: base data with the override + entries overlaid. A base read failure for a kind yields just the overrides for that kind. + This degrades the dependency fan-out but never loses the directly changed keys. + """ + view: MergedView = {} + for kind in _DIFF_KINDS: + items: Dict[str, Any] = {} + try: + base_items = base.all(kind, lambda x: x) + if base_items is not None: + items.update(base_items) + except Exception as e: + log.debug("Unable to read %s for override change computation: %s", kind.namespace, e) + items.update(overrides.get(kind, {})) + view[kind] = items + return view + + +def compute_affected_flags( + old_overrides: LayerContents, + new_overrides: LayerContents, + old_merged: MergedView, + new_merged: MergedView, +) -> Set[str]: + """ + Returns the keys of all flags whose merged-view evaluation may have changed when the + override layer was replaced. The result includes the flags whose override entries were + added, removed, or changed. Dependency fan-out adds every flag that depends, directly or + transitively, on any added, removed, or changed entry of either kind. + """ + seeds = diff_overrides(old_overrides, new_overrides) + if len(seeds) == 0: + return set() + + # Dependency edges are computed over both the old and the new merged views, because a + # replacement can rewire dependencies. For example, removing a flag override restores the + # prerequisite edges of the LaunchDarkly definition. Flags that depended on the override's + # references exist as dependents only in the old view. + old_tracker = _tracker_from_view(old_merged) + new_tracker = _tracker_from_view(new_merged) + affected: Set[KindAndKey] = set() + for seed in seeds: + old_tracker.add_affected_items(affected, seed) + new_tracker.add_affected_items(affected, seed) + return {item.key for item in affected if item.kind == FEATURES} + + +def diff_overrides(old_overrides: LayerContents, new_overrides: LayerContents) -> Set[KindAndKey]: + """ + Returns a key for each entry whose override differs between the two layer snapshots. An + added or removed entry is always a change, even when its content matches the underlying + LaunchDarkly data, because the override marker alone changes the served entry. Entries + present in both snapshots are compared by version and definition. The layer is rebuilt + wholesale on every update, so identity comparison would report every retained entry as + changed. + """ + seeds: Set[KindAndKey] = set() + for kind in _DIFF_KINDS: + old_items = old_overrides.get(kind, {}) + new_items = new_overrides.get(kind, {}) + for key, old_item in old_items.items(): + new_item = new_items.get(key) + if new_item is None or not _items_equal(old_item, new_item): + seeds.add(KindAndKey(kind=kind, key=key)) + for key in new_items: + if key not in old_items: + seeds.add(KindAndKey(kind=kind, key=key)) + return seeds + + +def _items_equal(a: Any, b: Any) -> bool: + if a.version != b.version: + return False + return a.to_json_dict() == b.to_json_dict() + + +def _tracker_from_view(view: MergedView) -> DependencyTracker: + tracker = DependencyTracker() + for kind in _DIFF_KINDS: + for key, item in view.get(kind, {}).items(): + tracker.update_dependencies_from(kind, key, item) + return tracker diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py new file mode 100644 index 00000000..0923c2de --- /dev/null +++ b/ldclient/integrations/overrides.py @@ -0,0 +1,222 @@ +""" +Override sources for the SDK's flag override capability. Flag overrides are currently +experimental and subject to change. + +Overrides are flag and segment definitions that take precedence over data received from +LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an +incident. An operator can force one or more flags to a known state on a running application, +whether or not the application can reach LaunchDarkly. The override stays in effect until the +operator removes it. Flags not present in the override data are completely unaffected. + +This module currently provides one source: :class:`FileOverrideSourceBuilder`, which reads +overrides from local files and reloads them as the files change. Configure it with the data +system builder: +:: + + from ldclient import Config, datasystem + from ldclient.integrations.overrides import FileOverrideSourceBuilder + + source = FileOverrideSourceBuilder(['/etc/launchdarkly/overrides.json']) + config = Config(sdk_key, datasystem_config=datasystem.default().overrides(source).build()) + +An evaluation that an override affects is marked. The marking is direct or transitive. It +applies when the evaluated flag, a prerequisite at any depth, or a segment read during the +evaluation came from the override layer. The evaluation reason's ``overrideAffected`` property +reports the marking. Marked evaluations appear in analytics summary events only, under separate +counters, so LaunchDarkly can distinguish them. They produce no individual evaluation events. +""" + +import math +import os +import threading +from typing import List, Optional, Sequence, Union + +from ldclient.config import DataSourceBuilderConfig, OverrideSourceBuilder +from ldclient.impl.integrations.files.filedata import ( + DuplicateKeysHandling, + abs_file_paths, + have_watchdog, + have_yaml, + is_yaml_path +) +from ldclient.impl.integrations.overrides.file_override_source import ( + ChangeDetection, + _FileOverrideSource +) +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSource + + +class FileOverrideSourceBuilder(OverrideSourceBuilder): + """ + A builder for a file-based override source. Flag overrides are currently experimental and + subject to change. + + The source reads flag and segment overrides from one or more local files and reloads them + as the files change. The files use the same document format as the file data source: + each file is a JSON or YAML document with optional ``flags``, ``flagValues``, and + ``segments`` members. ``flagValues`` entries are expanded into full flag definitions that + return the given value for every context. A ``flagValues`` entry is not available to + client-side SDKs, so :meth:`ldclient.client.LDClient.all_flags_state` with + ``client_side_only=True`` leaves it out, even when the flag it overrides is available to them. + To override such a flag, define it in ``flags`` with ``"clientSide": true``. + + The file name decides the format of each file: a file whose name ends in ``.yaml`` or + ``.yml`` is YAML, which requires the ``pyyaml`` package, and any other file is JSON. Files + must be UTF-8. + + When multiple files are configured, their entries are combined in the configured order. + The order determines which file wins under the duplicate keys handling. A reload replaces + the entire override set, so removing an entry from the files removes the override. A + configured file that does not exist contributes no overrides: deleting a file removes its + overrides, and deleting every file removes them all. A file that exists but cannot be read + or parsed makes that whole reload fail. The previously loaded overrides stay in effect, the + source logs the failure, retries after a short delay, and recovers on its own once the file + is readable again. A file that contains an entry marked as deleted fails to load, and the + overrides in effect stay; to remove an override, remove its entry or delete the file. + + Files can be rewritten in place: if the source finds a file empty while it is being + rewritten, it keeps the overrides in effect until the new content is written. To remove a + file's overrides, delete the file or write a document with no entries, such as ``{}``. An + empty JSON file is not a valid document. A partly written YAML file can still parse, so for a + YAML file, replacing its contents in one operation, such as by renaming a fully written + temporary file over it, avoids briefly applying a partly written file. + + Whenever the set of overrides in effect changes, including at startup, the source logs the + overrides in effect and what each configured file supplied, at Info level. + + By default the source polls the files for changes once per second. See + :meth:`change_detection` and :meth:`poll_interval`. + """ + + DEFAULT_POLL_INTERVAL = 1.0 + """ + The interval, in seconds, at which the source examines the files for changes in polling + mode when no interval was specified. Because the source reads local files rather than + contacting a service, a short interval keeps an override responsive during an incident at + negligible cost. + """ + + MINIMUM_POLL_INTERVAL = 1.0 + """ + The shortest allowed polling interval, in seconds. A configured interval below this is + raised to it. The minimum exists only to prevent a pathological tight loop over the file + system. + """ + + def __init__(self, paths: Union[str, 'os.PathLike[str]', Sequence[Union[str, 'os.PathLike[str]']]]): + """ + :param paths: the files to load overrides from, as a single path or a list of paths. + Each path is a string or a path object such as :class:`pathlib.Path`. The order is + significant: it determines which file wins under the duplicate keys handling when the + same key appears in more than one file. Relative paths are resolved against the + current working directory when the source is built. + :raises TypeError: when a path is neither a string nor a path object that names a string + path + """ + # bytes and bytearray are single values too, so that they get the error below instead of + # being read as a sequence of integers. + if isinstance(paths, (str, bytes, bytearray, os.PathLike)): + paths = [paths] + self.__paths: List[str] = [] + for path in paths: + try: + name: object = os.fspath(path) + except TypeError: + name = None + if not isinstance(name, str): + raise TypeError("file paths for the file-based override source must be strings or path objects; got %r" % (path,)) + self.__paths.append(name) + self.__duplicate_keys_handling = DuplicateKeysHandling.FAIL + self.__change_detection = ChangeDetection.POLLING + self.__poll_interval = self.DEFAULT_POLL_INTERVAL + + def duplicate_keys_handling(self, handling: Union[DuplicateKeysHandling, str]) -> 'FileOverrideSourceBuilder': + """ + Specifies how to handle the same key appearing in more than one file. The default is + :attr:`DuplicateKeysHandling.FAIL`, which treats the reload as failed and keeps the + previously loaded overrides. :attr:`DuplicateKeysHandling.IGNORE` keeps the entry from + the first configured file that defines the key and discards the others. + + :param handling: the handling, as the enum or its string value (``"fail"`` or ``"ignore"``) + """ + self.__duplicate_keys_handling = DuplicateKeysHandling(handling) + return self + + def change_detection(self, mode: Union[ChangeDetection, str]) -> 'FileOverrideSourceBuilder': + """ + Selects how the source detects file changes. The default is + :attr:`ChangeDetection.POLLING`. The two modes are alternatives, so setting one replaces + the other. :attr:`ChangeDetection.WATCHING` requires the ``watchdog`` package. + + Watching follows edits to the configured files and to the targets of configured symbolic + links, including the update of a mounted Kubernetes ConfigMap. It does not follow a + deployment that points a link further along the path, or a configured directory, at + another location, such as a ``current`` link to a release directory, or that replaces a + configured directory. Use polling for such deployments. + + :param mode: the mode, as the enum or its string value (``"polling"`` or ``"watching"``) + """ + self.__change_detection = ChangeDetection(mode) + return self + + def poll_interval(self, seconds: float) -> 'FileOverrideSourceBuilder': + """ + Sets the interval between examinations of the files in polling mode. Watching mode + ignores it. The default is :attr:`DEFAULT_POLL_INTERVAL`. An interval below + :attr:`MINIMUM_POLL_INTERVAL` is raised to the minimum. + + :param seconds: the interval in seconds + :raises TypeError: when the interval is not a number + :raises ValueError: when the interval is not a finite number, or is longer than the + runtime can wait + """ + if isinstance(seconds, bool) or not isinstance(seconds, (int, float)): + raise TypeError("the poll interval must be a number of seconds") + if not math.isfinite(seconds): + raise ValueError("the poll interval must be a finite number of seconds") + if seconds > threading.TIMEOUT_MAX: + raise ValueError("the poll interval must not exceed %s seconds" % threading.TIMEOUT_MAX) + self.__poll_interval = seconds + return self + + def build(self, config: DataSourceBuilderConfig) -> OverrideSource: # pylint: disable=unused-argument + """ + Builds the override source. This is called internally by the SDK. It raises + ``ValueError`` when no file paths were specified, when watching mode was selected + without the ``watchdog`` package, or when a YAML file was specified without the + ``pyyaml`` package. + """ + if len(self.__paths) == 0: + raise ValueError("no file paths were specified for the file-based override source") + if self.__change_detection == ChangeDetection.WATCHING and not have_watchdog: + raise ValueError("the file-based override source cannot watch files for changes because the watchdog package is not installed; install it or use polling") + if not have_yaml: + for path in self.__paths: + if is_yaml_path(path): + raise ValueError("the file-based override source cannot read the YAML file %s because the pyyaml package is not installed; install it or use JSON" % path) + + poll_interval = self.__poll_interval + if self.__change_detection == ChangeDetection.POLLING and poll_interval < self.MINIMUM_POLL_INTERVAL: + log.warning("Poll interval %s is below the minimum for the file-based override source; using %s", poll_interval, self.MINIMUM_POLL_INTERVAL) + poll_interval = self.MINIMUM_POLL_INTERVAL + + # A path listed more than once would make every entry of that file a duplicate of itself, + # so the repeats are dropped and the first occurrence keeps its place in the order. Paths + # are compared the way the platform compares file names: without regard to case on + # Windows, exactly elsewhere. + all_paths = abs_file_paths(self.__paths) + paths: List[str] = [] + seen = set() + for path in all_paths: + key = os.path.normcase(path) + if key not in seen: + seen.add(key) + paths.append(path) + if len(paths) != len(all_paths): + log.warning("The file-based override source lists the same file more than once; the repeats are ignored") + + return _FileOverrideSource(paths, self.__duplicate_keys_handling, self.__change_detection, poll_interval) + + +__all__ = ['ChangeDetection', 'DuplicateKeysHandling', 'FileOverrideSourceBuilder'] diff --git a/ldclient/interfaces.py b/ldclient/interfaces.py index de403354..f1b3314c 100644 --- a/ldclient/interfaces.py +++ b/ldclient/interfaces.py @@ -2132,6 +2132,85 @@ def stop(self): raise NotImplementedError +class OverrideSink(Protocol): # pylint: disable=too-few-public-methods + """ + OverrideSink receives the contents of the SDK's flag and segment override layer. The SDK + implements it and passes it to an :class:`OverrideSource` when the source starts. Override + sources call it, they do not implement it. + + Flag overrides are currently experimental and subject to change. + """ + + @abstractmethod + def set_overrides(self, flags: Mapping[str, Any], segments: Mapping[str, Any]) -> None: + """ + Replaces the entire override layer with the given flag and segment definitions. Each call + is a full snapshot: entries absent from the call are removed from the layer, and empty + mappings clear the layer. The new contents are visible to evaluations when the call + returns. + + The mappings are keyed by flag key and segment key. Each value is a full definition, + either a :class:`ldclient.impl.model.FeatureFlag` or :class:`ldclient.impl.model.Segment` + model object or its JSON dictionary form, whose own key is the key it has in the mapping. + A dictionary that is not a valid definition, or a definition whose key differs from its + key in the mapping, raises ``ValueError``, and the layer is left unchanged. A snapshot + that contains a flag or segment marked as deleted is rejected with ``ValueError``, and + the overrides in effect stay; to remove an override, leave it out of the snapshot. The + SDK itself marks the entries as overrides. + + This method is safe to call from any thread. Calls are serialized by the SDK. + + :param flags: the flag definitions, keyed by flag key + :param segments: the segment definitions, keyed by segment key + """ + raise NotImplementedError + + +class OverrideSource(Protocol): + """ + OverrideSource supplies flag and segment overrides that take precedence over LaunchDarkly data + at evaluation time, on a per-key basis. Overrides exist for resilience during an incident. + They let an operator force one or more flags to a known state on a running client, whether + or not the client can reach LaunchDarkly. + + An override source is not a data source. It does not participate in the data system's + initializer and synchronizer pipeline. The override layer it populates has no effect on the + client's initialization status, data availability, or data source status. + + To configure an override source, use :meth:`ldclient.datasystem.ConfigBuilder.overrides`. + + Flag overrides are currently experimental and subject to change. + """ + + @abstractmethod + def start(self, sink: OverrideSink) -> None: + """ + Begins supplying overrides to the sink and returns without blocking on long-running + work. Implementations typically perform an initial load synchronously, then push a full + replacement snapshot to the sink whenever their backing data changes, until + :meth:`close` is called. A failed load leaves the previously supplied layer untouched by + not calling the sink. + + This method is not intended to fail. A source that cannot load its data logs the failure + and keeps retrying, as the SDK's file-based source does. A configuration error belongs in + the builder's ``build``, where it is raised when the client is created. If this method does + raise, the client still starts: the SDK logs the exception, closes the source, removes any + snapshot the source supplied, and the client runs without overrides. + + :param sink: the sink that receives each snapshot + """ + raise NotImplementedError + + @abstractmethod + def close(self) -> None: + """ + Stops the source and releases any resources it holds. An exception raised from this + method is logged and does not stop the client from shutting down. A snapshot that the + source supplies after the client begins to close it is ignored. + """ + raise NotImplementedError + + class AsyncInitializer(Protocol): # pylint: disable=too-few-public-methods """ AsyncInitializer represents a component capable of retrieving a single data result diff --git a/ldclient/testing/file_util.py b/ldclient/testing/file_util.py new file mode 100644 index 00000000..25f5a10a --- /dev/null +++ b/ldclient/testing/file_util.py @@ -0,0 +1,26 @@ +import time + + +def rewrite_in_place(path: str, content: str, pause: float) -> None: + """ + Rewrites a file the way applications and editors often do, without replacing it atomically: + the file is truncated, and the new content is written after the pause. A read during the + pause finds the file empty. A file that does not exist is created with the content. + """ + try: + f = open(path, 'r+') + except FileNotFoundError: + with open(path, 'w') as created: + created.write(content) + return + with f: + f.truncate(0) + f.flush() + time.sleep(pause) + f.write(content) + + +def truncate(path: str) -> None: + """Empties a file in place, which is how an in-place rewrite starts.""" + with open(path, 'r+') as f: + f.truncate(0) diff --git a/ldclient/testing/impl/datasystem/test_store_external_dependencies.py b/ldclient/testing/impl/datasystem/test_store_external_dependencies.py new file mode 100644 index 00000000..87422a6a --- /dev/null +++ b/ldclient/testing/impl/datasystem/test_store_external_dependencies.py @@ -0,0 +1,137 @@ +""" +Tests for the store's use of a second dependency graph: a LaunchDarkly change to a flag or +segment that an override entry depends on notifies the override entry's flag as well. +""" +from typing import List, Optional + +from ldclient.impl.datasystem.store import Store +from ldclient.impl.dependency_tracker import KindAndKey +from ldclient.impl.listeners import Listeners +from ldclient.impl.overrides import OverrideLayer +from ldclient.interfaces import ( + Change, + ChangeSet, + ChangeType, + FlagChange, + IntentCode, + ObjectKind, + Selector +) +from ldclient.testing.builders import ( + FlagBuilder, + FlagRuleBuilder, + SegmentBuilder, + make_clause_matching_segment_key +) +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + + +def _flag(key: str, version: int, prerequisite: Optional[str] = None, segment: Optional[str] = None) -> dict: + builder = FlagBuilder(key).version(version).on(True).variations(True, False).fallthrough_variation(1) + if prerequisite is not None: + builder.prerequisite(prerequisite, 0) + if segment is not None: + builder.rules(FlagRuleBuilder().id('r').variation(0).clauses(make_clause_matching_segment_key(segment)).build()) + return builder.build().to_json_dict() + + +def _segment(key: str, version: int) -> dict: + return SegmentBuilder(key).version(version).build().to_json_dict() + + +def _put(kind: ObjectKind, item: dict) -> Change: + return Change(action=ChangeType.PUT, kind=kind, key=item['key'], version=item['version'], object=item) + + +def _full(*changes: Change) -> ChangeSet: + return ChangeSet(intent_code=IntentCode.TRANSFER_FULL, changes=list(changes), selector=Selector.no_selector()) + + +def _delta(*changes: Change) -> ChangeSet: + return ChangeSet(intent_code=IntentCode.TRANSFER_CHANGES, changes=list(changes), selector=Selector.no_selector()) + + +class Fixture: + def __init__(self, layer: Optional[OverrideLayer] = None): + self.notified: List[str] = [] + listeners = Listeners() + listeners.add(lambda change: self.notified.append(change.key)) + self.store = Store(listeners, Listeners()) + if layer is not None: + self.store.set_external_dependencies(layer) + + def take_notified(self) -> List[str]: + result = sorted(self.notified) + del self.notified[:] + return result + + +def test_delta_to_a_prerequisite_notifies_the_override_flag_that_depends_on_it(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, prerequisite='ld-flag')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1))), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag', 'override-flag'] + + +def test_full_data_set_replacement_notifies_the_override_flags_that_depend_on_changed_items(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, segment='ld-segment')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.SEGMENT, _segment('ld-segment', 1)), _put(ObjectKind.FLAG, _flag('unrelated', 1))), False) + f.take_notified() + + f.store.apply(_full(_put(ObjectKind.SEGMENT, _segment('ld-segment', 2)), _put(ObjectKind.FLAG, _flag('unrelated', 1))), False) + # The segment is not a flag, so only the dependent override flag is notified. + assert f.take_notified() == ['override-flag'] + + +def test_dependency_chain_crosses_between_the_store_and_the_layer(): + # LaunchDarkly flag -> override flag -> LaunchDarkly segment: the chain enters the layer + # and comes back out, and a segment change must reach the LaunchDarkly flag at the top. + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, segment='ld-segment')}, {}) + f = Fixture(layer) + f.store.apply(_full( + _put(ObjectKind.SEGMENT, _segment('ld-segment', 1)), + _put(ObjectKind.FLAG, _flag('ld-flag', 1, prerequisite='override-flag')), + ), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.SEGMENT, _segment('ld-segment', 2))), False) + assert f.take_notified() == ['ld-flag', 'override-flag'] + + +def test_layer_replacement_changes_which_flags_a_store_change_notifies(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, prerequisite='ld-flag')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1))), False) + f.take_notified() + + layer.set_all({}, {}) + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag'] + + +def test_store_without_external_dependencies_notifies_only_its_own_dependents(): + f = Fixture() + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1)), _put(ObjectKind.FLAG, _flag('top', 1, prerequisite='ld-flag'))), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag', 'top'] + + +def test_external_dependencies_do_not_hide_the_store_s_own_dependents(): + layer = OverrideLayer() + layer.set_all({'override-flag': _flag('override-flag', 1, prerequisite='ld-flag')}, {}) + f = Fixture(layer) + f.store.apply(_full(_put(ObjectKind.FLAG, _flag('ld-flag', 1)), _put(ObjectKind.FLAG, _flag('top', 1, prerequisite='ld-flag'))), False) + f.take_notified() + + f.store.apply(_delta(_put(ObjectKind.FLAG, _flag('ld-flag', 2))), False) + assert f.take_notified() == ['ld-flag', 'override-flag', 'top'] diff --git a/ldclient/testing/impl/events/test_event_overrides.py b/ldclient/testing/impl/events/test_event_overrides.py new file mode 100644 index 00000000..4f4345a3 --- /dev/null +++ b/ldclient/testing/impl/events/test_event_overrides.py @@ -0,0 +1,188 @@ +""" +Tests for how analytics events treat override-affected evaluations: they appear in summary +counters only, under a counter that carries the override-affected marker, and they produce no +individual feature event and no debug event. +""" +import json +import time +from typing import Any, Dict, List + +import pytest + +from ldclient.async_config import AsyncConfig +from ldclient.config import Config +from ldclient.context import Context +from ldclient.impl.events.async_event_processor import ( + DefaultAsyncEventProcessor +) +from ldclient.impl.events.event_processor import DefaultEventProcessor +from ldclient.impl.events.event_processor_common import EventOutputFormatter +from ldclient.impl.events.event_summarizer import ( + EventSummarizer, + EventSummaryCounter +) +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.testing.builders import FlagBuilder +from ldclient.testing.impl.events.test_async_event_processor import MockAioHttp +from ldclient.testing.stub_util import MockHttp + +context = Context.builder('userkey').name('Red').build() +timestamp = 10000 + + +def tracked_flag(key: str = 'flagkey', version: int = 2): + """A flag that requests individual feature events and debug events far into the future.""" + return FlagBuilder(key).version(version).track_events(True).debug_events_until_date(int(time.time() * 1000) + 100000).build() + + +def evaluation(flag, value: str = 'value', variation: int = 1, override_affected: bool = False, track_events: bool = True) -> EventInputEvaluation: + return EventInputEvaluation(timestamp, context, flag.key, flag, variation, value, None, 'default', None, track_events, override_affected) + + +def events_of_kind(output: List[Dict[str, Any]], kind: str) -> List[Dict[str, Any]]: + return [e for e in output if e['kind'] == kind] + + +# --------------------------------------------------------------------------- +# Summarizer and output formatter +# --------------------------------------------------------------------------- + +def test_summarizer_keeps_override_affected_counters_separate(): + flag = tracked_flag() + es = EventSummarizer() + es.summarize_event(evaluation(flag)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(evaluation(flag)) + counters = es.snapshot().flags[flag.key].counters + assert counters == { + (1, flag.version, False): EventSummaryCounter(2, 'value'), + (1, flag.version, True): EventSummaryCounter(2, 'value'), + } + + +def test_summary_output_carries_the_marker_only_on_override_affected_counters(): + flag = tracked_flag() + es = EventSummarizer() + es.summarize_event(evaluation(flag)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(EventInputEvaluation(timestamp, context, 'unknown-flag', None, None, 'default', None, 'default', None, False, False)) + output = EventOutputFormatter(Config('SDK_KEY')).make_summary_event(es.snapshot()) + + counters = output['features'][flag.key]['counters'] + assert len(counters) == 2 + by_marker = {c.get('overrideAffected'): c for c in counters} + assert by_marker[None] == {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version} + assert by_marker[True] == {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True} + assert 'overrideAffected' not in json.dumps(output['features']['unknown-flag']) + + +def test_event_input_evaluation_defaults_to_not_override_affected(): + event = EventInputEvaluation(timestamp, context, 'flag', None, None, 'default', None, 'default') + assert event.override_affected is False + assert event.to_debugging_dict()['override_affected'] is False + + +# --------------------------------------------------------------------------- +# Sync event processor +# --------------------------------------------------------------------------- + +def flush_and_get_events(ep: DefaultEventProcessor, mock_http: MockHttp) -> List[Dict[str, Any]]: + ep.flush() + ep._wait_until_inactive() + assert mock_http.request_data is not None, 'Expected to get an HTTP request but did not get one' + return json.loads(mock_http.request_data) + + +def make_processor(mock_http: MockHttp) -> DefaultEventProcessor: + return DefaultEventProcessor(Config('SDK_KEY', diagnostic_opt_out=True), mock_http) + + +def test_override_affected_evaluation_produces_no_feature_or_debug_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + output = flush_and_get_events(ep, mock_http) + + assert [e['kind'] for e in output] == ['index', 'summary'] + counters = output[1]['features'][flag.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}] + + +def test_ordinary_evaluation_still_produces_feature_and_debug_events(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag)) + output = flush_and_get_events(ep, mock_http) + + assert sorted(e['kind'] for e in output) == ['debug', 'feature', 'index', 'summary'] + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}] + + +def test_mixed_evaluations_of_one_flag_split_into_two_counters_and_one_feature_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag)) + output = flush_and_get_events(ep, mock_http) + + assert len(events_of_kind(output, 'feature')) == 1 + assert len(events_of_kind(output, 'debug')) == 1 + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert sorted(counters, key=lambda c: c['count']) == [ + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}, + {'count': 2, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}, + ] + + +def test_override_affected_prerequisite_record_produces_no_feature_event(): + parent = tracked_flag('parent', 1) + prereq = tracked_flag('prereq', 7) + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(EventInputEvaluation(timestamp, context, prereq.key, prereq, 1, 'value', None, None, parent, True, True)) + output = flush_and_get_events(ep, mock_http) + + assert [e['kind'] for e in output] == ['index', 'summary'] + counters = output[1]['features'][prereq.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': prereq.version, 'overrideAffected': True}] + + +def test_override_affected_evaluation_still_produces_an_index_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + output = flush_and_get_events(ep, mock_http) + assert output[0]['kind'] == 'index' + assert output[0]['context'] == context.to_dict() + + +# --------------------------------------------------------------------------- +# Async event processor (shares the dispatch logic) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_async_processor_suppresses_individual_events_for_override_affected_evaluations(): + flag = tracked_flag() + mock_http = MockAioHttp() + ep = DefaultAsyncEventProcessor(AsyncConfig('SDK_KEY', diagnostic_opt_out=True), mock_http) + try: + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag)) + assert await ep.flush_and_wait(5) is True + output = json.loads(mock_http.request_data) + finally: + await ep.stop() + + assert sorted(e['kind'] for e in output) == ['debug', 'feature', 'index', 'summary'] + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert sorted(counters, key=lambda c: 'overrideAffected' in c) == [ + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}, + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}, + ] diff --git a/ldclient/testing/impl/events/test_event_summarizer.py b/ldclient/testing/impl/events/test_event_summarizer.py index d7cf4ad1..3b2a2f82 100644 --- a/ldclient/testing/impl/events/test_event_summarizer.py +++ b/ldclient/testing/impl/events/test_event_summarizer.py @@ -41,8 +41,8 @@ def test_summarize_event_increments_counters(): data = es.snapshot() expected = { - 'flag1': EventSummaryFlag({'user'}, 'default1', {(1, flag1.version): EventSummaryCounter(2, 'value1'), (2, flag1.version): EventSummaryCounter(1, 'value2')}), - 'flag2': EventSummaryFlag({'user'}, 'default2', {(1, flag2.version): EventSummaryCounter(1, 'value99')}), - 'badkey': EventSummaryFlag({'user'}, 'default3', {(None, None): EventSummaryCounter(1, 'default3')}), + 'flag1': EventSummaryFlag({'user'}, 'default1', {(1, flag1.version, False): EventSummaryCounter(2, 'value1'), (2, flag1.version, False): EventSummaryCounter(1, 'value2')}), + 'flag2': EventSummaryFlag({'user'}, 'default2', {(1, flag2.version, False): EventSummaryCounter(1, 'value99')}), + 'badkey': EventSummaryFlag({'user'}, 'default3', {(None, None, False): EventSummaryCounter(1, 'default3')}), } assert data.flags == expected diff --git a/ldclient/testing/impl/test_async_evaluator_overrides.py b/ldclient/testing/impl/test_async_evaluator_overrides.py new file mode 100644 index 00000000..e193d3bf --- /dev/null +++ b/ldclient/testing/impl/test_async_evaluator_overrides.py @@ -0,0 +1,154 @@ +""" +Tests for the override marking performed by the async evaluator. These mirror the key +scenarios of the sync evaluator override tests. +""" +import pytest + +from ldclient import Context +from ldclient.impl.events.types import EventFactory +from ldclient.testing.builders import * +from ldclient.testing.impl.test_async_evaluator import AsyncEvaluatorBuilder +from ldclient.testing.impl.test_evaluator_overrides import ( + assert_override_affected, + assert_reason_override_affected, + make_override_test_flag, + prereq_records, + require_prereq_record +) + +override_test_context = Context.create('userkey') +event_factory_with_reasons = EventFactory(True, lambda: 0) + + +@pytest.mark.asyncio +async def test_override_flag_marks_evaluation(): + flag = make_override_test_flag('feature').with_override_marker() + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_plain_flag_is_not_marked(): + flag = make_override_test_flag('feature') + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert_override_affected(False, result) + + +@pytest.mark.asyncio +async def test_malformed_override_flag_error_result_is_marked(): + flag = FlagBuilder('feature').on(True).fallthrough_variation(99).variations('off', 'on').build().with_override_marker() + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_prerequisite_cycle_through_an_override_flag_is_marked(): + prereq = make_override_test_flag('prereq', 'feature').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = AsyncEvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + assert prereq_records(result) == [] + + +@pytest.mark.asyncio +async def test_prerequisite_cycle_below_an_override_flag_keeps_the_flag_marking(): + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_override_prerequisite_marks_prerequisite_record_and_top_level(): + prereq = make_override_test_flag('prereq').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(True, require_prereq_record(result, 'prereq').reason) + + +@pytest.mark.asyncio +async def test_override_flag_does_not_mark_unaffected_prerequisite_record(): + prereq = make_override_test_flag('prereq') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +@pytest.mark.asyncio +async def test_unaffected_sibling_prerequisite_record_stays_unmarked(): + s1 = SegmentBuilder('s1').included(override_test_context.key).build() + d = make_override_test_flag('d').with_override_marker() + b = make_override_test_flag('b', 'd') + c = FlagBuilder('c').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(s1.key)).build() + ).build() + a = make_override_test_flag('a', 'b', 'c') + evaluator = AsyncEvaluatorBuilder().with_flag(b).with_flag(c).with_flag(d).with_segment(s1).build() + result = await evaluator.evaluate(a, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert len(prereq_records(result)) == 3 + assert_reason_override_affected(True, require_prereq_record(result, 'd').reason) + assert_reason_override_affected(True, require_prereq_record(result, 'b').reason) + assert_reason_override_affected(False, require_prereq_record(result, 'c').reason) + + +@pytest.mark.asyncio +async def test_override_segment_read_without_matching_marks_evaluation(): + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = AsyncEvaluatorBuilder().with_segment(segment).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_override_segment_referenced_by_segment_rule_marks_evaluation(): + nested = SegmentBuilder('nested-segment').included(override_test_context.key).build().with_override_marker() + outer = SegmentBuilder('outer-segment').rules(SegmentRuleBuilder().clauses(make_clause_matching_segment_key(nested.key)).build()).build() + flag = make_boolean_flag_matching_segment(outer) + evaluator = AsyncEvaluatorBuilder().with_segment(outer).with_segment(nested).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.value is True + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_missing_definitions_do_not_mark_evaluation(): + unrelated = make_override_test_flag('unrelated').with_override_marker() + flag = make_override_test_flag('feature', 'missing') + evaluator = AsyncEvaluatorBuilder().with_flag(unrelated).with_unknown_flag('missing').build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': 'missing'} + assert_override_affected(False, result) + + segment_flag = make_boolean_flag_with_clauses(make_clause_matching_segment_key('missing')) + evaluator = AsyncEvaluatorBuilder().with_unknown_segment('missing').build() + result = await evaluator.evaluate(segment_flag, override_test_context, event_factory_with_reasons) + assert_override_affected(False, result) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("flag_override,prereq_override", [(False, False), (True, False), (False, True), (True, True)]) +async def test_result_override_affected_matches_reason_indicator(flag_override: bool, prereq_override: bool): + prereq = make_override_test_flag('prereq') + if prereq_override: + prereq = prereq.with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + if flag_override: + flag = flag.with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(flag_override or prereq_override, result) + assert_reason_override_affected(prereq_override, require_prereq_record(result, 'prereq').reason) diff --git a/ldclient/testing/impl/test_evaluator_overrides.py b/ldclient/testing/impl/test_evaluator_overrides.py new file mode 100644 index 00000000..309c3a6b --- /dev/null +++ b/ldclient/testing/impl/test_evaluator_overrides.py @@ -0,0 +1,325 @@ +""" +Tests for the override marking performed by the evaluator. An evaluation is override-affected +when any flag or segment definition it reads carries the override marker. The marking +propagates upward from prerequisites to the flags that depend on them and never sideways or +downward. +""" +from typing import List, Optional + +import pytest + +from ldclient import Context +from ldclient.impl.evaluator_common import EvalResult, is_override_definition +from ldclient.impl.events.types import EventFactory, EventInputEvaluation +from ldclient.testing.builders import * +from ldclient.testing.impl.evaluator_util import EvaluatorBuilder + +override_test_context = Context.create('userkey') + +# Events carry their reasons, so a prerequisite record's marking is visible on its reason. +event_factory_with_reasons = EventFactory(True, lambda: 0) + + +def assert_override_affected(expected: bool, result: EvalResult) -> None: + """Checks the reason indicator and the result scalar together. Both report the same marking.""" + assert result.override_affected is expected, "result.override_affected" + assert_reason_override_affected(expected, result.detail.reason) + + +def assert_reason_override_affected(expected: bool, reason: Optional[dict]) -> None: + assert reason is not None + if expected: + assert reason.get('overrideAffected') is True, "reason indicator: %s" % reason + else: + assert 'overrideAffected' not in reason, "reason indicator: %s" % reason + + +def make_override_test_flag(key: str, *prereq_keys: str) -> FeatureFlag: + """A flag that is on and serves variation 1 ("on") by fallthrough, with prerequisites that must each serve variation 1.""" + builder = FlagBuilder(key).on(True).fallthrough_variation(1).off_variation(0).variations('off', 'on') + for prereq_key in prereq_keys: + builder.prerequisite(prereq_key, 1) + return builder.build() + + +def prereq_records(result: EvalResult) -> List[EventInputEvaluation]: + return result.events or [] + + +def require_prereq_record(result: EvalResult, prereq_key: str) -> EventInputEvaluation: + for event in prereq_records(result): + if event.flag is not None and event.flag.key == prereq_key: + return event + raise AssertionError("no record for prerequisite %s" % prereq_key) + + +def test_override_flag_marks_evaluation_when_off(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'OFF' + assert result.detail.value == 'off' + assert_override_affected(True, result) + + +def test_override_flag_marks_evaluation_on_fallthrough(): + flag = make_override_test_flag('feature').with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + +def test_override_flag_marks_evaluation_on_rule_match(): + rule = FlagRuleBuilder().id('rule-id').variation(1).clauses(make_clause_matching_context(override_test_context)).build() + flag = FlagBuilder('feature').on(True).fallthrough_variation(0).variations('off', 'on').rules(rule).build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.reason['ruleId'] == 'rule-id' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + +def test_plain_flag_alone_is_not_marked(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'OFF'} + assert_override_affected(False, result) + + +def test_plain_flag_with_prerequisite_and_segment_is_not_marked(): + segment = SegmentBuilder('segment').included(override_test_context.key).build() + prereq = FlagBuilder('prereq').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(segment.key)).build() + ).build() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert result.detail.value == 'on' + assert_override_affected(False, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +def test_malformed_override_flag_error_result_is_marked(): + flag = FlagBuilder('feature').on(True).fallthrough_variation(99).variations('off', 'on').build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'ERROR' + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert result.detail.value is None + assert_override_affected(True, result) + + +def test_prerequisite_cycle_through_an_override_flag_is_marked(): + # feature -> prereq -> feature; only prereq is an override + prereq = make_override_test_flag('prereq', 'feature').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'ERROR' + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + assert prereq_records(result) == [] + + +def test_prerequisite_cycle_below_an_override_flag_keeps_the_flag_marking(): + # feature -> prereq -> feature; only feature is an override. The failure happens inside the + # plain prerequisite's subtree, and the top-level marking from the flag's own read survives it. + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +def test_prerequisite_cycle_through_plain_flags_is_not_marked(): + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(False, result) + + +def test_override_prerequisite_marks_prerequisite_record_and_top_level(): + prereq = make_override_test_flag('prereq').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + record = require_prereq_record(result, 'prereq') + assert record.reason is not None + assert record.reason['kind'] == 'FALLTHROUGH' + assert_reason_override_affected(True, record.reason) + + +def test_override_flag_does_not_mark_unaffected_prerequisite_record(): + # The marking propagates upward only. The top-level flag's own marker does not leak into + # the record of a prerequisite whose subtree read no override definition. + prereq = make_override_test_flag('prereq') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +def test_override_prerequisite_at_depth_two_marks_all_affected_scopes(): + prereq2 = make_override_test_flag('prereq2').with_override_marker() + prereq1 = make_override_test_flag('prereq1', 'prereq2') + flag = make_override_test_flag('feature', 'prereq1') + evaluator = EvaluatorBuilder().with_flag(prereq1).with_flag(prereq2).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + # The nested record is produced first, during the evaluation of prereq1. + records = prereq_records(result) + assert [r.flag.key for r in records] == ['prereq2', 'prereq1'] + assert_reason_override_affected(True, records[0].reason) + assert_reason_override_affected(True, records[1].reason) + + +def test_unaffected_sibling_prerequisite_record_stays_unmarked(): + # Flag a has prerequisites b and c. Only d, a prerequisite of b, is an override. The marking + # reaches a, b, and d. It does not reach the sibling c, and the plain segments s1 and s2 + # mark nothing. + s1 = SegmentBuilder('s1').included(override_test_context.key).build() + s2 = SegmentBuilder('s2').included(override_test_context.key).build() + d = make_override_test_flag('d').with_override_marker() + b = make_override_test_flag('b', 'd') + c = FlagBuilder('c').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(s1.key)).build() + ).build() + a = FlagBuilder('a').on(True).fallthrough_variation(0).off_variation(0).prerequisite('b', 1).prerequisite('c', 1).rules( + FlagRuleBuilder().id('rule-s2').variation(1).clauses(make_clause_matching_segment_key(s2.key)).build() + ).variations('off', 'on').build() + + evaluator = EvaluatorBuilder().with_flag(b).with_flag(c).with_flag(d).with_segment(s1).with_segment(s2).build() + result = evaluator.evaluate(a, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + assert len(prereq_records(result)) == 3 + assert_reason_override_affected(True, require_prereq_record(result, 'd').reason) + assert_reason_override_affected(True, require_prereq_record(result, 'b').reason) + assert_reason_override_affected(False, require_prereq_record(result, 'c').reason) + + +def test_override_segment_referenced_by_flag_rule_marks_evaluation(): + segment = SegmentBuilder('segment').included(override_test_context.key).build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_read_without_matching_marks_evaluation(): + # A read is enough to mark the evaluation. The segment does not need to match. + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert result.detail.value is False + assert_override_affected(True, result) + + +def test_override_segment_read_through_negated_clause_marks_evaluation(): + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_with_clauses(negate_clause(make_clause_matching_segment_key(segment.key))) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_referenced_by_segment_rule_marks_evaluation(): + # The outer segment is plain. A rule of the outer segment reads a nested override segment. + nested = SegmentBuilder('nested-segment').included(override_test_context.key).build().with_override_marker() + outer = SegmentBuilder('outer-segment').rules(SegmentRuleBuilder().clauses(make_clause_matching_segment_key(nested.key)).build()).build() + flag = make_boolean_flag_matching_segment(outer) + evaluator = EvaluatorBuilder().with_segment(outer).with_segment(nested).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_referenced_by_prerequisite_marks_prerequisite_record_and_top_level(): + segment = SegmentBuilder('segment').included(override_test_context.key).build().with_override_marker() + prereq = FlagBuilder('prereq').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(segment.key)).build() + ).build() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + assert_reason_override_affected(True, require_prereq_record(result, 'prereq').reason) + + +def test_missing_prerequisite_does_not_mark_evaluation(): + # A definition that cannot be resolved contributes nothing, because nothing was read. The + # store holds an unrelated override definition to show that only reads count. + unrelated = make_override_test_flag('unrelated').with_override_marker() + flag = make_override_test_flag('feature', 'missing') + evaluator = EvaluatorBuilder().with_flag(unrelated).with_unknown_flag('missing').build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': 'missing'} + assert result.detail.value == 'off' + assert_override_affected(False, result) + assert prereq_records(result) == [] + + +def test_missing_segment_does_not_mark_evaluation(): + unrelated = SegmentBuilder('unrelated').included(override_test_context.key).build().with_override_marker() + flag = make_boolean_flag_with_clauses(make_clause_matching_segment_key('missing')) + evaluator = EvaluatorBuilder().with_segment(unrelated).with_unknown_segment('missing').build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert result.detail.value is False + assert_override_affected(False, result) + + +@pytest.mark.parametrize("flag_override,prereq_override", [(False, False), (True, False), (False, True), (True, True)]) +def test_result_override_affected_matches_reason_indicator(flag_override: bool, prereq_override: bool): + prereq = make_override_test_flag('prereq') + if prereq_override: + prereq = prereq.with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + if flag_override: + flag = flag.with_override_marker() + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(flag_override or prereq_override, result) + assert_reason_override_affected(prereq_override, require_prereq_record(result, 'prereq').reason) + + +def test_marking_does_not_disturb_other_reason_properties(): + segment = SegmentBuilder('segment').unbounded(True).generation(1).build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).with_no_big_segments_for_key(override_test_context.key).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH', 'bigSegmentsStatus': 'HEALTHY', 'overrideAffected': True} + + +def test_reason_indicator_is_absent_when_events_do_not_carry_reasons(): + # The marking does not depend on the event factory. The reason on the result is marked either way. + flag = make_override_test_flag('feature').with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, EventFactory(False, lambda: 0)) + assert_override_affected(True, result) + + +def test_is_override_definition_reads_the_marker_and_never_fails(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build() + assert is_override_definition(flag.with_override_marker()) is True + assert is_override_definition(flag) is False + # An item without the marker attribute, such as a raw dict, reads as unmarked. + assert is_override_definition(flag.to_json_dict()) is False + assert is_override_definition(None) is False diff --git a/ldclient/testing/impl/test_model_override_marker.py b/ldclient/testing/impl/test_model_override_marker.py new file mode 100644 index 00000000..203e0fc5 --- /dev/null +++ b/ldclient/testing/impl/test_model_override_marker.py @@ -0,0 +1,76 @@ +""" +Tests for the override marker carried on the flag and segment models. The marker lives on the +model type and never in the JSON representation. +""" +import json + +import pytest + +from ldclient.impl.model import FeatureFlag, ModelEncoder, Segment +from ldclient.testing.builders import FlagBuilder, SegmentBuilder +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + +flag_data = {'key': 'flag1', 'version': 2, 'on': True, 'variations': [True, False], 'fallthrough': {'variation': 0}, 'salt': 'x'} +segment_data = {'key': 'seg1', 'version': 3, 'included': ['user1'], 'salt': 'y'} + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marker_defaults_to_false(kind, data): + entity = kind.decode(dict(data)) + assert entity.is_override is False + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marked_copy_is_marked_and_source_is_not(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.is_override is True + assert entity.is_override is False + assert type(marked) is type(entity) + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marker_is_never_serialized(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.to_json_dict() == data + assert kind.encode(marked) == data + assert json.loads(ModelEncoder().encode(marked)) == data + assert 'is_override' not in marked.to_json_dict() + assert 'override' not in json.dumps(marked.to_json_dict()).lower() + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marked_copy_shares_the_definition(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.key == entity.key + assert marked.version == entity.version + assert marked.to_json_dict() is entity.to_json_dict() + assert marked == entity + + +def test_marked_flag_keeps_its_parsed_properties(): + flag = FlagBuilder('flag1').version(7).on(True).variations('a', 'b').fallthrough_variation(1).prerequisite('p', 0).track_events(True).build() + marked = flag.with_override_marker() + assert marked.on is True + assert marked.variations == ['a', 'b'] + assert marked.fallthrough.variation == 1 + assert [p.key for p in marked.prerequisites] == ['p'] + assert marked.track_events is True + assert marked.version == 7 + + +def test_marked_segment_keeps_its_parsed_properties(): + segment = SegmentBuilder('seg1').version(4).included('a').excluded('b').build() + marked = segment.with_override_marker() + assert marked.included == {'a'} + assert marked.excluded == {'b'} + assert marked.version == 4 + + +def test_marking_a_marked_copy_is_a_marked_copy(): + flag = FeatureFlag(dict(flag_data)).with_override_marker() + again = flag.with_override_marker() + assert again.is_override is True + assert again is not flag diff --git a/ldclient/testing/impl/test_overrides.py b/ldclient/testing/impl/test_overrides.py new file mode 100644 index 00000000..1681d8e4 --- /dev/null +++ b/ldclient/testing/impl/test_overrides.py @@ -0,0 +1,553 @@ +""" +Tests for the override layer, the overlay at the store read boundary, and the sink that applies +snapshots and notifies flag change listeners. +""" +import logging +from typing import Any, Dict, List, Optional + +import pytest + +from ldclient.impl.datasystem.store import InMemoryFeatureStore +from ldclient.impl.dependency_tracker import KindAndKey +from ldclient.impl.overrides import ( + AsyncOverrideStoreView, + OverrideLayer, + OverrideSinkImpl, + OverrideStoreView +) +from ldclient.testing.builders import ( + FlagBuilder, + FlagRuleBuilder, + SegmentBuilder, + make_clause_matching_segment_key +) +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + + +class FakeBaseStore: + """A minimal read-only store with controllable contents, initialization state, and failures.""" + + def __init__(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None, initialized: bool = True): + self.flags = dict(flags or {}) + self.segments = dict(segments or {}) + self._initialized = initialized + self.fail_all = False + + def _items(self, kind): + return self.flags if kind == FEATURES else self.segments + + def get(self, kind, key, callback=lambda x: x): + return callback(self._items(kind).get(key)) + + def all(self, kind, callback=lambda x: x): + if self.fail_all: + raise RuntimeError("store failure") + return callback(dict(self._items(kind))) + + @property + def initialized(self) -> bool: + return self._initialized + + +class FakeAsyncBaseStore: + def __init__(self, base: FakeBaseStore): + self._base = base + + async def get(self, kind, key): + return self._base.get(kind, key) + + async def all(self, kind): + return self._base.all(kind) + + +def flag(key: str, version: int = 1): + return FlagBuilder(key).version(version).on(True).variations(False, True).fallthrough_variation(1).off_variation(0).build() + + +def segment(key: str, version: int = 1): + return SegmentBuilder(key).version(version).build() + + +# --------------------------------------------------------------------------- +# Layer +# --------------------------------------------------------------------------- + +def test_layer_marks_copies_without_mutating_source(): + layer = OverrideLayer() + f = flag('flag1', 2) + s = segment('segment1', 3) + layer.set_all({'flag1': f}, {'segment1': s}) + + assert f.is_override is False + assert s.is_override is False + stored_flag = layer.get(FEATURES, 'flag1') + assert stored_flag is not None + assert stored_flag.is_override is True + assert stored_flag.version == 2 + stored_segment = layer.get(SEGMENTS, 'segment1') + assert stored_segment is not None + assert stored_segment.is_override is True + + +def test_layer_decodes_dictionaries_and_marks_them(): + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1').to_json_dict()}, {'segment1': segment('segment1').to_json_dict()}) + stored_flag = layer.get(FEATURES, 'flag1') + assert stored_flag is not None + assert stored_flag.is_override is True + assert stored_flag.on is True + stored_segment = layer.get(SEGMENTS, 'segment1') + assert stored_segment is not None + assert stored_segment.is_override is True + + +def test_layer_rejects_invalid_definitions_and_keeps_its_contents(): + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1')}, {}) + with pytest.raises(ValueError): + layer.set_all({'flag2': {'key': 'flag2', 'version': 'not a number'}}, {}) + assert layer.get(FEATURES, 'flag1') is not None + assert layer.get(FEATURES, 'flag2') is None + with pytest.raises(ValueError): + layer.set_all({}, {'seg': {'key': 'seg', 'version': 1, 'included': 'not a list'}}) + assert layer.get(FEATURES, 'flag1') is not None + + +@pytest.mark.parametrize( + 'flags,segments', + [ + ({'a': flag('b')}, {}), + ({'a': flag('b').to_json_dict()}, {}), + ({}, {'s': segment('t')}), + ({}, {'s': segment('t').to_json_dict()}), + ], + ids=['flag model', 'flag dict', 'segment model', 'segment dict'], +) +def test_layer_rejects_a_definition_whose_key_differs_from_its_mapping_key(flags, segments): + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1')}, {}) + with pytest.raises(ValueError, match='the key of each definition must be its key in the snapshot'): + layer.set_all(flags, segments) + assert list(layer.all(FEATURES).keys()) == ['flag1'] + assert layer.all(SEGMENTS) == {} + + +def snapshot_with_deleted_item(kind, as_model: bool): + """ + A snapshot that changes the entries flag1 and segment1 to version 2 and adds an item of the + given kind, named deleted-key, that is marked as deleted, as a model object or in its JSON + dictionary form. Because the snapshot changes the other entries too, applying any part of it + would show. + """ + data = {'key': 'deleted-key', 'version': 2, 'deleted': True} + deleted = {'deleted-key': kind.decode(data) if as_model else data} + flags = {'flag1': flag('flag1', 2), **(deleted if kind == FEATURES else {})} + segments = {'segment1': segment('segment1', 2), **(deleted if kind == SEGMENTS else {})} + return flags, segments + + +def deleted_item_message(kind) -> str: + return '%s "deleted-key" is marked as deleted; an override is removed by leaving it out of the snapshot' % ('flag' if kind == FEATURES else 'segment') + + +@pytest.mark.parametrize('as_model', [False, True], ids=['dict', 'model']) +@pytest.mark.parametrize('kind', [FEATURES, SEGMENTS], ids=['flag', 'segment']) +def test_layer_rejects_a_snapshot_that_holds_a_deleted_item_and_keeps_its_contents(kind, as_model): + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1')}, {'segment1': segment('segment1')}) + flags, segments = snapshot_with_deleted_item(kind, as_model) + + with pytest.raises(ValueError) as e: + layer.set_all(flags, segments) + + assert str(e.value) == deleted_item_message(kind) + assert layer.get(FEATURES, 'flag1').version == 1 + assert layer.get(SEGMENTS, 'segment1').version == 1 + assert layer.get(kind, 'deleted-key') is None + + +def test_layer_accepts_definitions_marked_as_not_deleted(): + layer = OverrideLayer() + layer.set_all({'flag1': dict(flag('flag1').to_json_dict(), deleted=False)}, {'segment1': dict(segment('segment1').to_json_dict(), deleted=False)}) + stored_flag = layer.get(FEATURES, 'flag1') + assert stored_flag is not None + assert stored_flag.on is True + assert layer.get(SEGMENTS, 'segment1') is not None + + +def test_layer_replacement_semantics(): + layer = OverrideLayer() + assert layer.is_empty is True + assert layer.get(FEATURES, 'flag1') is None + + layer.set_all({'flag1': flag('flag1')}, {}) + assert layer.is_empty is False + assert layer.get(FEATURES, 'flag1') is not None + + # A replacement is a full snapshot: entries absent from it are removed. + layer.set_all({'flag2': flag('flag2')}, {}) + assert layer.get(FEATURES, 'flag1') is None + assert layer.get(FEATURES, 'flag2') is not None + + layer.set_all({}, {}) + assert layer.is_empty is True + assert layer.get(FEATURES, 'flag2') is None + + +def test_layer_set_all_returns_previous_and_current_contents(): + layer = OverrideLayer() + previous, current = layer.set_all({'flag1': flag('flag1')}, {'seg': segment('seg')}) + assert previous == {FEATURES: {}, SEGMENTS: {}} + assert list(current[FEATURES].keys()) == ['flag1'] + assert list(current[SEGMENTS].keys()) == ['seg'] + previous, current = layer.set_all({}, {}) + assert list(previous[FEATURES].keys()) == ['flag1'] + assert current == {FEATURES: {}, SEGMENTS: {}} + + +def test_layer_all_returns_entries_of_a_kind(): + layer = OverrideLayer() + layer.set_all({'a': flag('a'), 'b': flag('b')}, {'s': segment('s')}) + assert sorted(layer.all(FEATURES).keys()) == ['a', 'b'] + assert list(layer.all(SEGMENTS).keys()) == ['s'] + assert all(item.is_override for item in layer.all(FEATURES).values()) + + +# --------------------------------------------------------------------------- +# Overlay +# --------------------------------------------------------------------------- + +def test_layer_add_affected_items_reaches_the_entries_that_depend_on_an_item(): + top = FlagBuilder('top').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('mid', 0).build() + mid = FlagBuilder('mid').version(1).on(True).variations(True, False).fallthrough_variation(1).rules( + FlagRuleBuilder().id('r').variation(0).clauses(make_clause_matching_segment_key('seg')).build() + ).build() + layer = OverrideLayer() + layer.set_all({'top': top, 'mid': mid, 'other': flag('other')}, {}) + + # The segment is not an override entry; the chain still starts from it. + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=SEGMENTS, key='seg')) + assert affected == {KindAndKey(SEGMENTS, 'seg'), KindAndKey(FEATURES, 'mid'), KindAndKey(FEATURES, 'top')} + + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=FEATURES, key='other')) + assert affected == {KindAndKey(FEATURES, 'other')} + + +def test_layer_dependency_graph_follows_each_replacement(): + top = FlagBuilder('top').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('leaf', 0).build() + layer = OverrideLayer() + layer.set_all({'top': top}, {}) + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=FEATURES, key='leaf')) + assert affected == {KindAndKey(FEATURES, 'leaf'), KindAndKey(FEATURES, 'top')} + + layer.set_all({}, {}) + affected = set() + layer.add_affected_items(affected, KindAndKey(kind=FEATURES, key='leaf')) + assert affected == {KindAndKey(FEATURES, 'leaf')} + + +def test_overlay_get_precedence(): + base = FakeBaseStore(flags={'both': flag('both', 1), 'base-only': flag('base-only', 1)}) + layer = OverrideLayer() + layer.set_all({'both': flag('both', 99), 'override-only': flag('override-only', 1)}, {}) + overlay = OverrideStoreView(base, layer) + + item = overlay.get(FEATURES, 'both') + assert item.version == 99 + assert item.is_override is True + + item = overlay.get(FEATURES, 'base-only') + assert item.is_override is False + + item = overlay.get(FEATURES, 'override-only') + assert item.is_override is True + + assert overlay.get(FEATURES, 'nowhere') is None + assert overlay.get(FEATURES, 'both', lambda x: x.version) == 99 + + +def test_overlay_get_serves_overrides_from_uninitialized_base(): + base = FakeBaseStore(initialized=False) + layer = OverrideLayer() + layer.set_all({'flag1': flag('flag1')}, {}) + overlay = OverrideStoreView(base, layer) + item = overlay.get(FEATURES, 'flag1') + assert item.is_override is True + assert overlay.initialized is False + + +def test_overlay_get_all_union(): + memory = InMemoryFeatureStore() + memory.set_basis({ + FEATURES: { + 'both': flag('both', 1).to_json_dict(), + 'base-only': flag('base-only', 1).to_json_dict(), + 'tombstone': {'key': 'tombstone', 'version': 5, 'deleted': True}, + }, + SEGMENTS: {}, + }) + layer = OverrideLayer() + layer.set_all({'both': flag('both', 99), 'tombstone': flag('tombstone', 1), 'override-only': flag('override-only', 1)}, {}) + overlay = OverrideStoreView(memory, layer) + + items = overlay.all(FEATURES) + assert sorted(items.keys()) == ['base-only', 'both', 'override-only', 'tombstone'] + assert items['both'].version == 99 + assert items['both'].is_override is True + assert items['base-only'].is_override is False + assert items['tombstone'].is_override is True, "an override wins over a deleted item" + assert items['override-only'].is_override is True + assert overlay.all(FEATURES, lambda x: len(x)) == 4 + + +def test_overlay_get_all_with_empty_layer_is_passthrough(): + base = FakeBaseStore(flags={'flag1': flag('flag1')}) + overlay = OverrideStoreView(base, OverrideLayer()) + assert list(overlay.all(FEATURES).keys()) == ['flag1'] + base.fail_all = True + with pytest.raises(RuntimeError): + overlay.all(FEATURES) + + +def test_overlay_get_all_propagates_a_base_failure_when_the_layer_holds_entries(): + # The layer's entries alone would look like a complete result to a caller such as the + # all-flags state, so the failure propagates as it does without the overlay. A per-key read + # still serves the override. + base = FakeBaseStore() + base.fail_all = True + layer = OverrideLayer() + layer.set_all({'override-1': flag('override-1', 1)}, {}) + overlay = OverrideStoreView(base, layer) + with pytest.raises(RuntimeError): + overlay.all(FEATURES) + assert overlay.get(FEATURES, 'override-1').is_override is True + + +def test_overlay_get_all_keeps_a_missing_base_result_missing(): + # A store returns no result when it cannot read. The overlay does not turn that into the + # layer's entries alone. + class NoResultStore(FakeBaseStore): + def all(self, kind, callback=lambda x: x): + return callback(None) + + layer = OverrideLayer() + layer.set_all({'override-1': flag('override-1', 1)}, {}) + assert OverrideStoreView(NoResultStore(), layer).all(FEATURES) is None + + +def test_overlay_initialized_follows_the_base(): + base = FakeBaseStore(initialized=True) + overlay = OverrideStoreView(base, OverrideLayer()) + assert overlay.initialized is True + base._initialized = False + assert overlay.initialized is False + + +@pytest.mark.asyncio +async def test_async_overlay_get_and_all(): + base = FakeBaseStore(flags={'both': flag('both', 1), 'base-only': flag('base-only', 1)}, initialized=False) + layer = OverrideLayer() + layer.set_all({'both': flag('both', 99), 'override-only': flag('override-only', 1)}, {'seg': segment('seg')}) + overlay = AsyncOverrideStoreView(FakeAsyncBaseStore(base), layer) + + both = await overlay.get(FEATURES, 'both') + assert both.version == 99 and both.is_override is True + base_only = await overlay.get(FEATURES, 'base-only') + assert base_only.is_override is False + assert (await overlay.get(FEATURES, 'nowhere')) is None + assert (await overlay.get(SEGMENTS, 'seg')).is_override is True + + items = await overlay.all(FEATURES) + assert sorted(items.keys()) == ['base-only', 'both', 'override-only'] + assert items['both'].version == 99 + + +@pytest.mark.asyncio +async def test_async_overlay_all_when_base_fails(): + base = FakeBaseStore() + base.fail_all = True + layer = OverrideLayer() + overlay = AsyncOverrideStoreView(FakeAsyncBaseStore(base), layer) + with pytest.raises(RuntimeError): + await overlay.all(FEATURES) + # The failure propagates whatever the layer holds. + layer.set_all({'override-1': flag('override-1')}, {}) + with pytest.raises(RuntimeError): + await overlay.all(FEATURES) + assert (await overlay.get(FEATURES, 'override-1')).is_override is True + + +# --------------------------------------------------------------------------- +# Sink +# --------------------------------------------------------------------------- + +class SinkFixture: + def __init__(self, base: FakeBaseStore): + self.base = base + self.layer = OverrideLayer() + self.notified: List[str] = [] + self.listening = True + self.sink = OverrideSinkImpl(self.layer, base, lambda key: self.notified.append(key), lambda: self.listening) + + def take_notified(self) -> List[str]: + result = sorted(self.notified) + del self.notified[:] + return result + + +def test_sink_notifies_on_add_change_remove(): + base = FakeBaseStore(flags={'flag1': flag('flag1', 1)}) + f = SinkFixture(base) + + # Adding an override is a change even though flag1 also exists in base data. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': flag('flag2', 1)}, {}) + assert f.take_notified() == ['flag1', 'flag2'] + + # An identical replacement (rebuilt from scratch, new objects) changes nothing. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': flag('flag2', 1)}, {}) + assert f.take_notified() == [] + + # Changing one entry notifies only that entry. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': flag('flag2', 2)}, {}) + assert f.take_notified() == ['flag2'] + + # A change in content at the same version is a change. + f.sink.set_overrides({'flag1': flag('flag1', 1), 'flag2': FlagBuilder('flag2').version(2).on(False).build()}, {}) + assert f.take_notified() == ['flag2'] + + # Removing overrides notifies them: flag1 reverts to base data, flag2 to not-found. + f.sink.set_overrides({}, {}) + assert f.take_notified() == ['flag1', 'flag2'] + + +def test_sink_segment_override_fans_out_to_dependent_flags(): + dependent = FlagBuilder('dependent').version(1).on(True).variations(True, False).fallthrough_variation(1).rules( + FlagRuleBuilder().id('r').variation(0).clauses(make_clause_matching_segment_key('segment1')).build() + ).build() + base = FakeBaseStore( + flags={'dependent': dependent, 'unrelated': flag('unrelated')}, + segments={'segment1': segment('segment1', 1)}, + ) + f = SinkFixture(base) + f.sink.set_overrides({}, {'segment1': segment('segment1', 99)}) + # The segment itself is not a flag, so only the dependent flag is notified. + assert f.take_notified() == ['dependent'] + + +def test_sink_prerequisite_override_fans_out_through_the_chain(): + top = FlagBuilder('top').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('mid', 0).build() + mid = FlagBuilder('mid').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('leaf', 0).build() + base = FakeBaseStore(flags={'top': top, 'mid': mid, 'leaf': flag('leaf'), 'other': flag('other')}) + f = SinkFixture(base) + f.sink.set_overrides({'leaf': flag('leaf', 5)}, {}) + assert f.take_notified() == ['leaf', 'mid', 'top'] + + +def test_sink_removing_a_segment_override_notifies_flags_that_reference_it(): + # The override of "dependent" references segment "s". Removing the override of "s" alone + # changes what "dependent" evaluates to, so "dependent" is notified through the dependency + # fan-out even though its own override did not change. + dependent_override = FlagBuilder('dependent').version(2).on(True).variations(True, False).fallthrough_variation(1).rules( + FlagRuleBuilder().variation(0).clauses(make_clause_matching_segment_key('s')).build() + ).build() + base = FakeBaseStore(flags={'dependent': flag('dependent')}, segments={'s': segment('s')}) + f = SinkFixture(base) + f.sink.set_overrides({'dependent': dependent_override}, {'s': segment('s', 2)}) + f.take_notified() + f.sink.set_overrides({'dependent': dependent_override}, {}) + assert f.take_notified() == ['dependent'] + + +def test_sink_skips_change_computation_when_nothing_listens(): + base = FakeBaseStore(flags={'flag1': flag('flag1')}) + f = SinkFixture(base) + f.listening = False + f.sink.set_overrides({'flag1': flag('flag1', 2)}, {}) + assert f.take_notified() == [] + assert f.layer.get(FEATURES, 'flag1').version == 2 + + +def test_sink_close_keeps_the_layer_and_ignores_later_snapshots(): + f = SinkFixture(FakeBaseStore()) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + assert f.take_notified() == ['flag1'] + + f.sink.close() + assert f.layer.get(FEATURES, 'flag1') is not None + assert f.take_notified() == [] + + # A snapshot after closing changes nothing. + f.sink.set_overrides({'flag2': flag('flag2')}, {}) + assert f.layer.get(FEATURES, 'flag1') is not None + assert f.layer.get(FEATURES, 'flag2') is None + assert f.take_notified() == [] + + +def test_sink_close_and_clear_clears_the_layer_and_ignores_later_snapshots(): + f = SinkFixture(FakeBaseStore()) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + assert f.take_notified() == ['flag1'] + + # Clearing removes what the source supplied, and the removal is a change. + f.sink.close_and_clear() + assert f.layer.is_empty is True + assert f.take_notified() == ['flag1'] + + # A snapshot after closing changes nothing. + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + assert f.layer.is_empty is True + assert f.take_notified() == [] + + +def test_sink_notifies_directly_changed_keys_when_base_read_fails(): + base = FakeBaseStore(flags={'flag1': flag('flag1')}) + base.fail_all = True + f = SinkFixture(base) + f.sink.set_overrides({'flag1': flag('flag1', 2)}, {}) + assert f.take_notified() == ['flag1'] + + +def test_sink_invalid_snapshot_raises_and_changes_nothing(): + base = FakeBaseStore() + f = SinkFixture(base) + f.sink.set_overrides({'flag1': flag('flag1')}, {}) + f.take_notified() + with pytest.raises(ValueError): + f.sink.set_overrides({'flag1': {'key': 'flag1', 'version': 'bad'}}, {}) + assert f.layer.get(FEATURES, 'flag1').version == 1 + assert f.take_notified() == [] + + +@pytest.mark.parametrize('listening', [True, False], ids=['listening', 'not-listening']) +@pytest.mark.parametrize('as_model', [False, True], ids=['dict', 'model']) +@pytest.mark.parametrize('kind', [FEATURES, SEGMENTS], ids=['flag', 'segment']) +def test_sink_rejects_a_snapshot_that_holds_a_deleted_item_logs_it_and_notifies_nothing(kind, as_model, listening, caplog): + dependent = FlagBuilder('dependent').version(1).on(True).variations(True, False).fallthrough_variation(1).rules( + FlagRuleBuilder().variation(0).clauses(make_clause_matching_segment_key('segment1')).build() + ).build() + f = SinkFixture(FakeBaseStore(flags={'dependent': dependent})) + f.sink.set_overrides({'flag1': flag('flag1')}, {'segment1': segment('segment1')}) + f.take_notified() + f.listening = listening + caplog.clear() + flags, segments = snapshot_with_deleted_item(kind, as_model) + + with pytest.raises(ValueError) as e: + f.sink.set_overrides(flags, segments) + + assert str(e.value) == deleted_item_message(kind) + assert f.layer.get(FEATURES, 'flag1').version == 1 + assert f.layer.get(SEGMENTS, 'segment1').version == 1 + assert f.layer.get(kind, 'deleted-key') is None + assert f.take_notified() == [] + errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] + assert errors == ['Rejected an override snapshot; the overrides in effect stay: ' + deleted_item_message(kind)] + + # The sink still accepts the next snapshot. + f.listening = True + f.sink.set_overrides({'flag1': flag('flag1', 3)}, {}) + assert f.layer.get(FEATURES, 'flag1').version == 3 + assert f.take_notified() == ['dependent', 'flag1'] diff --git a/ldclient/testing/integrations/test_file_data_sources_pinned_behavior.py b/ldclient/testing/integrations/test_file_data_sources_pinned_behavior.py new file mode 100644 index 00000000..6213d69c --- /dev/null +++ b/ldclient/testing/integrations/test_file_data_sources_pinned_behavior.py @@ -0,0 +1,483 @@ +""" +Pins the behavior of the two existing file data sources, the FDv1 update processor and the FDv2 +initializer and synchronizer. The file-based override source is built on separate code and +behaves differently in several of these respects. These tests keep the existing sources +observably unchanged: the flagValues expansion and its evaluation reason, the version fallback, +the duplicate key and load failure messages, the FDv2 status and error kinds, the acceptance of +entries marked as deleted, and the polling and watching rules. +""" +import logging +import os +import threading +import time +from typing import Any, Callable, List, Optional + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.datasystem import custom +from ldclient.feature_store import InMemoryFeatureStore +from ldclient.impl.datasource.status import DataSourceUpdateSinkImpl +from ldclient.impl.integrations.files import file_data_sourcev2 +from ldclient.impl.integrations.files.file_data_source import _FileDataSource +from ldclient.impl.integrations.files.file_data_sourcev2 import ( + _FileDataSourceV2, + _PollingAutoUpdaterV2, + _WatchdogAutoUpdaterV2 +) +from ldclient.impl.listeners import Listeners +from ldclient.integrations import Files +from ldclient.interfaces import ( + DataSourceErrorKind, + DataSourceState, + ObjectKind, + Selector +) +from ldclient.testing.mock_components import MockSelectorStore +from ldclient.testing.test_util import SpyListener +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + +have_watchdog = file_data_sourcev2.have_watchdog +watchdog_required = pytest.mark.skipif(not have_watchdog, reason="watchdog is not installed") + +user = Context.create('user') + +DOCUMENT = ''' +{ + "flags": { + "flag1": { + "key": "flag1", + "on": true, + "fallthrough": {"variation": 2}, + "variations": ["fall", "off", "on"] + }, + "flag-versioned": { + "key": "flag-versioned", + "version": 7, + "on": false, + "offVariation": 0, + "variations": ["x"] + } + }, + "flagValues": { + "flag2": "value2" + }, + "segments": { + "seg1": { + "key": "seg1", + "included": ["user1"] + } + } +} +''' + +# The expansion of a flagValues entry: an on flag whose fallthrough serves its single variation. +EXPANDED_FLAG2 = {'key': 'flag2', 'version': 1, 'on': True, 'fallthrough': {'variation': 0}, 'variations': ['value2']} + + +def write_file(path: str, content: str) -> None: + with open(path, 'w') as f: + f.write(content) + + +def make_v1_source(path: str, store: InMemoryFeatureStore, listeners: Optional[Listeners] = None, **kwargs) -> _FileDataSource: + config = Config('SDK_KEY') + if listeners is not None: + config._data_source_update_sink = DataSourceUpdateSinkImpl(store, listeners, Listeners()) + factory = Files.new_data_source(paths=[path], **kwargs) + assert factory is not None + source = factory(config, store, threading.Event()) + assert isinstance(source, _FileDataSource) + return source + + +def make_v2_source(path: str, **kwargs) -> _FileDataSourceV2: + source = Files.new_data_source_v2(paths=[path], **kwargs).build(Config('SDK_KEY')) + assert isinstance(source, _FileDataSourceV2) + return source + + +def v2_changes_by_key(change_set) -> dict: + return {change.key: change for change in change_set.changes} + + +# --------------------------------------------------------------------------- +# flagValues expansion and its evaluation reason +# --------------------------------------------------------------------------- + +def test_v1_expands_flag_values_to_an_on_flag_with_fallthrough(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, DOCUMENT) + store = InMemoryFeatureStore() + source = make_v1_source(path, store) + source.start() + try: + assert store.get(FEATURES, 'flag2').to_json_dict() == EXPANDED_FLAG2 + finally: + source.stop() + + +def test_v1_flag_values_evaluate_with_a_fallthrough_reason(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, DOCUMENT) + config = Config('SDK_KEY', update_processor_class=Files.new_data_source(paths=[path]), send_events=False) + with LDClient(config) as client: + detail = client.variation_detail('flag2', user, 'default') + assert detail.value == 'value2' + assert detail.variation_index == 0 + assert detail.reason == {'kind': 'FALLTHROUGH'} + + +def test_v2_expands_flag_values_to_an_on_flag_with_fallthrough(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, DOCUMENT) + source = make_v2_source(path) + result = source.fetch(MockSelectorStore(Selector.no_selector())) + changes = v2_changes_by_key(result.value.change_set) + assert changes['flag2'].object == EXPANDED_FLAG2 + assert changes['flag2'].version == 1 + + +def test_v2_flag_values_evaluate_with_a_fallthrough_reason(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, DOCUMENT) + datasystem = custom().initializers([Files.new_data_source_v2(paths=[path])]).build() + config = Config('SDK_KEY', datasystem_config=datasystem, send_events=False) + with LDClient(config) as client: + detail = client.variation_detail('flag2', user, 'default') + assert detail.value == 'value2' + assert detail.variation_index == 0 + assert detail.reason == {'kind': 'FALLTHROUGH'} + + +# --------------------------------------------------------------------------- +# Version fallback +# --------------------------------------------------------------------------- + +def test_v1_stamps_version_1_on_entries_without_one_and_keeps_explicit_versions(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, DOCUMENT) + store = InMemoryFeatureStore() + source = make_v1_source(path, store) + source.start() + try: + assert store.get(FEATURES, 'flag1').version == 1 + assert store.get(SEGMENTS, 'seg1').version == 1 + assert store.get(FEATURES, 'flag-versioned').version == 7 + finally: + source.stop() + + +def test_v2_stamps_version_1_on_entries_without_one_and_keeps_explicit_versions(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, DOCUMENT) + result = make_v2_source(path).fetch(MockSelectorStore(Selector.no_selector())) + changes = v2_changes_by_key(result.value.change_set) + assert changes['flag1'].version == 1 + assert changes['flag1'].object['version'] == 1 + assert changes['seg1'].kind == ObjectKind.SEGMENT + assert changes['seg1'].version == 1 + assert changes['seg1'].object['version'] == 1 + assert changes['flag-versioned'].version == 7 + + +# --------------------------------------------------------------------------- +# Failure messages, statuses, and error kinds +# --------------------------------------------------------------------------- + +def test_v1_logs_the_load_failure_with_the_path_and_reports_invalid_data(tmp_path, caplog): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, '{"flagValues":{') + store = InMemoryFeatureStore() + listeners = Listeners() + spy = SpyListener() + listeners.add(spy) + source = make_v1_source(path, store, listeners) + with caplog.at_level(logging.ERROR): + source.start() + try: + assert store.initialized is False + errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] + assert any(m.startswith('Unable to load flag data from "%s": ' % path) for m in errors), errors + assert len(spy.statuses) == 1 + assert spy.statuses[0].error.kind == DataSourceErrorKind.INVALID_DATA + finally: + source.stop() + + +def test_v1_missing_file_fails_the_load_with_the_same_message(tmp_path, caplog): + path = os.path.join(str(tmp_path), 'missing.json') + store = InMemoryFeatureStore() + source = make_v1_source(path, store) + with caplog.at_level(logging.ERROR): + source.start() + try: + assert source.initialized() is False + errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] + assert any(m.startswith('Unable to load flag data from "%s": ' % path) for m in errors), errors + finally: + source.stop() + + +def test_v1_duplicate_key_message(tmp_path, caplog): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"flag1": "a"}}') + write_file(second, '{"flagValues": {"flag1": "b"}}') + store = InMemoryFeatureStore() + config = Config('SDK_KEY') + source = Files.new_data_source(paths=[first, second])(config, store, threading.Event()) + with caplog.at_level(logging.ERROR): + source.start() + try: + assert store.initialized is False + errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] + assert any('In features, key "flag1" was used more than once' in m for m in errors), errors + finally: + source.stop() + + +def test_v2_fetch_failure_message_names_the_path(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, '{"flagValues":{') + result = make_v2_source(path).fetch(MockSelectorStore(Selector.no_selector())) + assert result.error.startswith('Unable to load flag data from "%s": ' % path) + + +def test_v2_sync_ends_with_off_when_the_initial_load_fails(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, '{"flagValues":{') + source = make_v2_source(path, force_polling=True, poll_interval=0.1) + try: + updates = list(source.sync(MockSelectorStore(Selector.no_selector()))) + assert len(updates) == 1 + assert updates[0].state == DataSourceState.OFF + assert updates[0].change_set is None + assert updates[0].error is not None + assert updates[0].error.kind == DataSourceErrorKind.INVALID_DATA + assert updates[0].error.message.startswith('Unable to load flag data from "%s": ' % path) + finally: + source.stop() + + +def test_v2_sync_reports_invalid_data_when_a_file_becomes_malformed(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source = make_v2_source(path, force_polling=True, poll_interval=0.1) + updates: List[Any] = [] + received = threading.Event() + + def collect(): + for update in source.sync(MockSelectorStore(Selector.no_selector())): + updates.append(update) + received.set() + if len(updates) >= 2: + break + + thread = threading.Thread(target=collect, daemon=True) + thread.start() + try: + assert received.wait(5) + assert updates[0].state == DataSourceState.VALID + received.clear() + time.sleep(0.2) + write_file(path, '{"flagValues"') + assert received.wait(5) + assert updates[1].state == DataSourceState.INTERRUPTED + assert updates[1].error.kind == DataSourceErrorKind.INVALID_DATA + assert updates[1].error.message.startswith('Unable to load flag data from "%s": ' % path) + finally: + source.stop() + thread.join(5) + + +# --------------------------------------------------------------------------- +# Entries marked as deleted. The override source rejects a file that holds one. These sources +# accept it, and rejecting it here is a breaking change left to a later major version. +# --------------------------------------------------------------------------- + +DOCUMENT_WITH_DELETED_ENTRIES = ''' +{ + "flags": { + "deleted-flag": {"key": "deleted-flag", "version": 3, "deleted": true}, + "flag1": {"key": "flag1", "on": false, "offVariation": 0, "variations": ["x"]} + }, + "segments": { + "deleted-segment": {"key": "deleted-segment", "deleted": true} + } +} +''' + + +def test_v1_loads_a_file_with_deleted_entries_and_stores_them_as_deleted(tmp_path, caplog): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, DOCUMENT_WITH_DELETED_ENTRIES) + store = InMemoryFeatureStore() + source = make_v1_source(path, store) + with caplog.at_level(logging.ERROR): + source.start() + try: + assert store.initialized is True + assert [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] == [] + assert store.get(FEATURES, 'flag1') is not None + assert store.get(FEATURES, 'deleted-flag') is None + assert store.get(SEGMENTS, 'deleted-segment') is None + assert list(store.all(FEATURES, lambda x: x).keys()) == ['flag1'] + finally: + source.stop() + + +def test_v2_supplies_deleted_entries_as_they_are_in_the_file(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, DOCUMENT_WITH_DELETED_ENTRIES) + result = make_v2_source(path).fetch(MockSelectorStore(Selector.no_selector())) + assert result.value is not None + changes = v2_changes_by_key(result.value.change_set) + assert set(changes.keys()) == {'deleted-flag', 'flag1', 'deleted-segment'} + assert changes['deleted-flag'].object == {'key': 'deleted-flag', 'version': 3, 'deleted': True} + assert changes['deleted-flag'].version == 3 + assert changes['deleted-segment'].kind == ObjectKind.SEGMENT + assert changes['deleted-segment'].object == {'key': 'deleted-segment', 'version': 1, 'deleted': True} + + +# --------------------------------------------------------------------------- +# Polling rules: modification time only, a missing file is not a change +# --------------------------------------------------------------------------- + +class Counter: + def __init__(self): + self.count = 0 + + def __call__(self): + self.count += 1 + + +def set_mtime(path: str, seconds: float) -> None: + os.utime(path, (seconds, seconds)) + + +@pytest.fixture(params=['v1', 'v2']) +def make_poller(request): + pollers = [] + + def factory(paths: List[str], on_change: Callable[[], None]): + # The interval is long, so only the direct _poll calls below examine the files. + poller: Any + if request.param == 'v1': + poller = _FileDataSource.PollingAutoUpdater(paths, on_change, 1000) + else: + poller = _PollingAutoUpdaterV2(paths, on_change, 1000) + pollers.append(poller) + return poller + + yield factory + for poller in pollers: + poller.stop() + + +def test_poller_reloads_when_the_modification_time_changes(tmp_path, make_poller): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'aaa') + set_mtime(path, 1000000) + counter = Counter() + poller = make_poller([path], counter) + write_file(path, 'bbb') + set_mtime(path, 2000000) + poller._poll() + assert counter.count == 1 + + +def test_poller_ignores_a_size_change_with_the_same_modification_time(tmp_path, make_poller): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'aaa') + set_mtime(path, 1000000) + counter = Counter() + poller = make_poller([path], counter) + write_file(path, 'aaaa') + set_mtime(path, 1000000) + poller._poll() + assert counter.count == 0 + + +def test_poller_ignores_a_file_that_disappears(tmp_path, make_poller): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'aaa') + counter = Counter() + poller = make_poller([path], counter) + os.remove(path) + poller._poll() + assert counter.count == 0 + + +def test_poller_reloads_when_a_missing_file_appears(tmp_path, make_poller): + path = os.path.join(str(tmp_path), 'data.json') + counter = Counter() + poller = make_poller([path], counter) + write_file(path, 'aaa') + poller._poll() + assert counter.count == 1 + + +# --------------------------------------------------------------------------- +# Watching rules: any notification on the file's path reloads, a move destination does not +# --------------------------------------------------------------------------- + +def handlers_of(observer) -> list: + return [handler for handlers in observer._handlers.values() for handler in handlers] + + +@pytest.fixture(params=['v1', 'v2']) +def make_watcher(request): + watchers = [] + + def factory(paths: List[str], on_change: Callable[[], None]): + watcher: Any + if request.param == 'v1': + watcher = _FileDataSource.WatchdogAutoUpdater(paths, on_change) + else: + watcher = _WatchdogAutoUpdaterV2(paths, on_change) + watchers.append(watcher) + return watcher + + yield factory + for watcher in watchers: + watcher.stop() + + +@watchdog_required +def test_watcher_reloads_on_any_notification_for_the_file_path(tmp_path, make_watcher): + import watchdog.events + + path = os.path.realpath(os.path.join(str(tmp_path), 'data.json')) + write_file(path, 'aaa') + counter = Counter() + watcher = make_watcher([path], counter) + handlers = handlers_of(watcher._observer) + assert len(handlers) == 1 + handler = handlers[0] + events = [watchdog.events.FileModifiedEvent(path), watchdog.events.FileOpenedEvent(path)] + # Older watchdog versions report no event for a read-only close. + closed_no_write = getattr(watchdog.events, 'FileClosedNoWriteEvent', None) + if closed_no_write is not None: + events.append(closed_no_write(path)) + for event in events: + handler.on_any_event(event) + assert counter.count == len(events) + handler.on_any_event(watchdog.events.FileModifiedEvent(os.path.join(os.path.dirname(path), 'other.json'))) + assert counter.count == len(events) + + +@watchdog_required +def test_watcher_ignores_a_move_whose_destination_is_the_file_path(tmp_path, make_watcher): + import watchdog.events + + path = os.path.realpath(os.path.join(str(tmp_path), 'data.json')) + temp = os.path.realpath(os.path.join(str(tmp_path), 'data.json.tmp')) + write_file(path, 'aaa') + counter = Counter() + watcher = make_watcher([path], counter) + handler = handlers_of(watcher._observer)[0] + handler.on_any_event(watchdog.events.FileMovedEvent(temp, path)) + assert counter.count == 0 + handler.on_any_event(watchdog.events.FileMovedEvent(path, temp)) + assert counter.count == 1 diff --git a/ldclient/testing/integrations/test_file_override_source.py b/ldclient/testing/integrations/test_file_override_source.py new file mode 100644 index 00000000..5504f178 --- /dev/null +++ b/ldclient/testing/integrations/test_file_override_source.py @@ -0,0 +1,899 @@ +""" +Tests for the file-based override source: its builder, initial load, multi-file merging, +absent files, change detection in both modes, failure retention, and its Info log. +""" +import logging +import ntpath +import os +import threading +import time +from queue import Empty, Queue +from typing import Any, Dict, List, Optional, Set, Tuple + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.config import Config as SDKConfig +from ldclient.datasystem import custom +from ldclient.evaluation import EvaluationDetail +from ldclient.impl.integrations.files import filedata +from ldclient.impl.integrations.overrides import file_override_source +from ldclient.impl.integrations.overrides.file_override_source import ( + _FileOverrideSource +) +from ldclient.integrations.overrides import ( + ChangeDetection, + DuplicateKeysHandling, + FileOverrideSourceBuilder +) +from ldclient.testing.file_util import rewrite_in_place, truncate +from ldclient.testing.mock_components import HangingSynchronizer +from ldclient.testing.stub_util import MockEventProcessor + +TEST_TIMEOUT = 10.0 +QUIET_PERIOD = 0.3 + +watchdog_required = pytest.mark.skipif(not filedata.have_watchdog, reason="watchdog is not installed") +yaml_required = pytest.mark.skipif(not filedata.have_yaml, reason="pyyaml is not installed") + +Snapshot = Tuple[Dict[str, Any], Dict[str, Any]] + + +class CapturingSink: + """Records every override snapshot it receives.""" + + def __init__(self): + self.snapshots: Queue = Queue() + + def set_overrides(self, flags, segments) -> None: + self.snapshots.put((dict(flags), dict(segments))) + + def require_snapshot(self, timeout: float = TEST_TIMEOUT) -> Snapshot: + try: + return self.snapshots.get(timeout=timeout) + except Empty: + pytest.fail("timed out waiting for an override snapshot") + + def require_no_snapshot(self, duration: float = QUIET_PERIOD) -> None: + try: + snapshot = self.snapshots.get(timeout=duration) + except Empty: + return + pytest.fail("received an unexpected override snapshot: %r" % (snapshot,)) + + +def write_file(path: str, content: str) -> None: + with open(path, 'w') as f: + f.write(content) + + +def flag_values(snapshot: Snapshot) -> Dict[str, Any]: + """The single variation of each flag in a snapshot, keyed by flag key.""" + return {key: flag.variations[0] for key, flag in snapshot[0].items()} + + +@pytest.fixture +def sources(): + started: List[_FileOverrideSource] = [] + yield started + for source in started: + source.close() + + +def build_source(sources, paths, configure=None) -> Tuple[_FileOverrideSource, CapturingSink]: + builder = FileOverrideSourceBuilder(paths) + if configure is not None: + configure(builder) + source = builder.build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + sink = CapturingSink() + source.start(sink) + sources.append(source) + return source, sink + + +def info_lines(caplog) -> List[str]: + return [r.getMessage() for r in caplog.records if r.levelno == logging.INFO] + + +def require_info_line(caplog, expected: str) -> None: + """Waits for an Info line equal to the expected text. The source logs after it hands the snapshot to the sink.""" + deadline = time.time() + TEST_TIMEOUT + while True: + if expected in info_lines(caplog): + return + if time.time() > deadline: + pytest.fail("timed out waiting for the Info line %r; Info output: %r" % (expected, info_lines(caplog))) + time.sleep(0.01) + + +# --------------------------------------------------------------------------- +# Builder +# --------------------------------------------------------------------------- + +def test_builder_requires_paths(): + with pytest.raises(ValueError): + FileOverrideSourceBuilder([]).build(SDKConfig('SDK_KEY')) + + +def test_builder_accepts_a_single_path_string(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + source = FileOverrideSourceBuilder(path).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [path] + + +def test_builder_accepts_a_single_path_object(tmp_path): + path = tmp_path / 'overrides.json' + source = FileOverrideSourceBuilder(path).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [str(path)] + + +def test_builder_accepts_a_list_of_path_objects(tmp_path): + paths = [tmp_path / 'a.json', tmp_path / 'b.json'] + source = FileOverrideSourceBuilder(paths).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [str(path) for path in paths] + + +class LabeledPath: + """A path object whose string form is not its path, as with :class:`os.DirEntry`.""" + + def __init__(self, path): + self._path = path + + def __fspath__(self): + return self._path + + def __str__(self): + return '' + + +@pytest.mark.parametrize('as_list', [False, True], ids=['single', 'list']) +def test_builder_uses_the_file_system_path_of_a_path_object(tmp_path, as_list): + path = os.path.join(str(tmp_path), 'overrides.json') + paths = [LabeledPath(path)] if as_list else LabeledPath(path) + source = FileOverrideSourceBuilder(paths).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [path] + + +@pytest.mark.parametrize( + 'paths', + [ + b'/overrides.json', + bytearray(b'/overrides.json'), + [b'/overrides.json'], + LabeledPath(b'/overrides.json'), + [LabeledPath(b'/overrides.json')], + [42], + [None], + ], + ids=['bytes', 'bytearray', 'bytes in a list', 'path object naming bytes', 'path object naming bytes in a list', 'int', 'none'], +) +def test_builder_rejects_a_path_that_is_not_a_string_or_path_object(paths): + with pytest.raises(TypeError, match='file paths for the file-based override source must be strings or path objects'): + FileOverrideSourceBuilder(paths) + + +def test_builder_names_a_rejected_bytes_path_whole(): + with pytest.raises(TypeError) as excinfo: + FileOverrideSourceBuilder(b'/overrides.json') + assert "b'/overrides.json'" in str(excinfo.value) + + +def test_builder_resolves_relative_paths(): + source = FileOverrideSourceBuilder(['relative/overrides.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('relative/overrides.json')] + + +def test_builder_drops_a_repeated_path_and_warns(caplog): + # Under the default fail handling the same file listed twice would make every one of its keys + # a duplicate of itself. The repeat is ignored and the first occurrence keeps its place. + with caplog.at_level(logging.WARNING): + source = FileOverrideSourceBuilder(['/a.json', '/b.json', '/a.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/b.json')] + assert any('more than once' in r.getMessage() for r in caplog.records) + + +def test_builder_treats_a_path_repeated_in_different_case_as_a_repeat_only_where_file_names_ignore_case(): + source = FileOverrideSourceBuilder(['/a.json', '/A.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + if os.name == 'nt': + assert source._paths == [os.path.abspath('/a.json')] + else: + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/A.json')] + + +def test_builder_drops_a_path_repeated_in_different_case_on_windows(monkeypatch): + # Windows file names ignore case, so two spellings of one path name the same file. The path + # comparison of Windows is used here on any platform. + monkeypatch.setattr(os.path, 'normcase', ntpath.normcase) + source = FileOverrideSourceBuilder(['/a.json', '/b.json', '/A.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/b.json')] + + +def test_builder_polls_by_default(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + source = FileOverrideSourceBuilder([path]).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._change_detection == ChangeDetection.POLLING + assert source._poll_interval == FileOverrideSourceBuilder.DEFAULT_POLL_INTERVAL + assert source._duplicate_keys_handling == DuplicateKeysHandling.FAIL + + +def test_builder_rejects_unknown_change_detection(): + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).change_detection('notify') + assert 'notify' in str(excinfo.value) + + +def test_builder_rejects_unknown_duplicate_keys_handling(): + with pytest.raises(ValueError): + FileOverrideSourceBuilder(['a']).duplicate_keys_handling('merge') + + +def test_builder_accepts_string_values(): + builder = FileOverrideSourceBuilder(['a']).change_detection('polling').duplicate_keys_handling('ignore') + source = builder.build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._change_detection == ChangeDetection.POLLING + assert source._duplicate_keys_handling == DuplicateKeysHandling.IGNORE + + +def test_builder_clamps_poll_interval_to_the_minimum(caplog): + with caplog.at_level(logging.WARNING): + source = FileOverrideSourceBuilder(['a']).poll_interval(0.001).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._poll_interval == FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL + assert any('below the minimum' in r.getMessage() for r in caplog.records) + + source = FileOverrideSourceBuilder(['a']).poll_interval(2.5).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._poll_interval == 2.5 + + +@pytest.mark.parametrize('seconds', ['1', None, True], ids=['string', 'none', 'bool']) +def test_builder_rejects_a_poll_interval_that_is_not_a_number(seconds): + with pytest.raises(TypeError): + FileOverrideSourceBuilder(['a']).poll_interval(seconds) + + +def test_builder_rejects_a_poll_interval_longer_than_the_runtime_can_wait(): + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).poll_interval(1e20) + assert 'must not exceed' in str(excinfo.value) + + +@pytest.mark.parametrize('seconds', [float('nan'), float('inf'), float('-inf')], ids=['nan', 'inf', '-inf']) +def test_builder_rejects_a_poll_interval_that_is_not_finite(seconds): + with pytest.raises(ValueError): + FileOverrideSourceBuilder(['a']).poll_interval(seconds) + + +def test_builder_rejects_watching_without_watchdog(monkeypatch): + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_watchdog', False) + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).change_detection(ChangeDetection.WATCHING).build(SDKConfig('SDK_KEY')) + assert 'watchdog' in str(excinfo.value) + + +@pytest.mark.parametrize('name', ['overrides.yaml', 'overrides.YML']) +def test_builder_rejects_a_yaml_file_without_pyyaml(monkeypatch, name): + # A YAML file can never load without a YAML parser, so it is a configuration error rather + # than a failure that every reload would report. + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_yaml', False) + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['/overrides.json', '/' + name]).build(SDKConfig('SDK_KEY')) + assert 'pyyaml' in str(excinfo.value) + assert name in str(excinfo.value) + + +def test_builder_accepts_json_files_without_pyyaml(monkeypatch): + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_yaml', False) + source = FileOverrideSourceBuilder(['/overrides.json', '/overrides']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + + +# --------------------------------------------------------------------------- +# Loading +# --------------------------------------------------------------------------- + +def test_source_loads_initial_data_synchronously(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true, "flag2": "x"}, "segments": {"seg": {"key": "seg"}}}') + source, sink = build_source(sources, [path]) + # The snapshot was delivered before start returned. + snapshot = sink.snapshots.get_nowait() + assert flag_values(snapshot) == {'flag1': True, 'flag2': 'x'} + assert list(snapshot[1].keys()) == ['seg'] + assert snapshot[0]['flag1'].is_override is False, "the source supplies unmarked definitions; the SDK marks them" + + +def test_source_loads_a_file_given_as_a_path_object(tmp_path, sources): + path = tmp_path / 'overrides.json' + write_file(str(path), '{"flagValues": {"flag1": true}}') + _, sink = build_source(sources, path) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + +@yaml_required +def test_source_loads_yaml(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.yaml') + write_file(path, 'flagValues:\n yaml-flag: "override-value"\n') + _, sink = build_source(sources, [path]) + assert flag_values(sink.require_snapshot()) == {'yaml-flag': 'override-value'} + + +def test_source_merges_files_in_configured_order(tmp_path, sources): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"shared": "first", "only-first": 1}}') + write_file(second, '{"flagValues": {"shared": "second", "only-second": 2}}') + _, sink = build_source(sources, [first, second], lambda b: b.duplicate_keys_handling(DuplicateKeysHandling.IGNORE)) + assert flag_values(sink.require_snapshot()) == {'shared': 'first', 'only-first': 1, 'only-second': 2} + + +def test_source_duplicate_keys_fail_by_default(tmp_path, sources, caplog): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"shared": "first"}}') + write_file(second, '{"flagValues": {"shared": "second"}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [first, second]) + # The load failed, so no snapshot was supplied and the failure was logged. + sink.require_no_snapshot() + assert any("is specified by multiple files" in r.getMessage() for r in caplog.records) + + +def test_source_starts_with_missing_file(tmp_path, sources): + path = os.path.join(str(tmp_path), 'not-yet.json') + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + # A missing file contributes no overrides. The initial snapshot is empty. + assert sink.require_snapshot() == ({}, {}) + # Once the file appears, the change detection picks it up. + write_file(path, '{"flagValues": {"flag1": true}}') + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + +def test_source_missing_file_contributes_no_entries(tmp_path, sources): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"from-first": true}}') + + # Step 1: one configured file exists and one does not. The existing file applies. + _, sink = build_source(sources, [first, second], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'from-first': True} + + # Step 2: the second file appears. Both apply. + write_file(second, '{"flagValues": {"from-second": true}}') + assert flag_values(sink.require_snapshot()) == {'from-first': True, 'from-second': True} + + # Step 3: the second file is deleted. Its overrides are removed. + os.remove(second) + assert flag_values(sink.require_snapshot()) == {'from-first': True} + + # Step 4: the last file is deleted. The layer is cleared. + os.remove(first) + assert sink.require_snapshot() == ({}, {}) + + +def test_source_logs_overrides_in_effect_on_each_change(tmp_path, sources, caplog): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"flag1": true, "flag2": false}, "segments": {"seg": {"key": "seg"}}}') + + with caplog.at_level(logging.INFO): + # Step 1: at startup, one file supplies entries and the other is absent. + _, sink = build_source(sources, [first, second], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides in effect: 2 flags, 1 segment (%s: 2 flags, 1 segment; %s: absent)" % (first, second)) + + # Step 2: the absent file appears with one entry. + write_file(second, '{"flagValues": {"flag3": true}}') + sink.require_snapshot() + require_info_line(caplog, "Flag overrides in effect: 3 flags, 1 segment (%s: 2 flags, 1 segment; %s: 1 flag)" % (first, second)) + + # Step 3: both files are deleted. Nothing is in effect. + os.remove(first) + os.remove(second) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: absent; %s: absent)" % (first, second)) + + +def test_source_logs_none_in_effect_at_startup_without_files(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + with caplog.at_level(logging.INFO): + _, sink = build_source(sources, [path]) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: absent)" % path) + + +def test_source_logs_a_present_file_with_no_entries(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + with caplog.at_level(logging.INFO): + _, sink = build_source(sources, [path]) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: no entries)" % path) + + +def wait_for_log(caplog, text: str) -> None: + deadline = time.time() + TEST_TIMEOUT + while not any(text in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "timed out waiting for a log line containing %r" % text + time.sleep(0.01) + + +def test_source_keeps_the_overrides_when_a_json_file_is_emptied(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + # An empty JSON file is not a document. The overrides stay in effect and, once the file + # has been read again, the failure is logged. + truncate(path) + wait_for_log(caplog, 'no content') + sink.require_no_snapshot() + + # A document with no entries removes the overrides. + write_file(path, '{}') + assert sink.require_snapshot() == ({}, {}) + + +@yaml_required +@watchdog_required +def test_source_removes_the_overrides_of_an_emptied_yaml_file_after_the_retry_delay(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.yaml') + write_file(path, 'flagValues:\n flag1: true\n') + _, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + # An empty YAML file is a document with no entries. The watch notices the truncation at + # once, and the file is read again after the retry delay before it is applied, in case it + # is in the middle of an in-place rewrite. + emptied = time.monotonic() + truncate(path) + assert sink.require_snapshot() == ({}, {}) + assert time.monotonic() - emptied >= filedata.DEFAULT_RETRY_DELAY + + +# --------------------------------------------------------------------------- +# Change detection +# --------------------------------------------------------------------------- + +@watchdog_required +def test_watching_mode_is_quiet_when_file_is_absent(tmp_path, sources): + path = os.path.join(str(tmp_path), 'not-yet.json') + _, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert sink.require_snapshot() == ({}, {}) + sink.require_no_snapshot() + write_file(path, '{"flagValues": {"flag1": true}}') + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + +@watchdog_required +def test_watching_mode_reloads_on_change(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert isinstance(source._change_detector, filedata.Watcher) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +def test_polling_mode_reloads_on_change(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.POLLING).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert isinstance(source._change_detector, filedata.Poller) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +@pytest.mark.parametrize("mode", [ChangeDetection.POLLING, pytest.param(ChangeDetection.WATCHING, marks=watchdog_required)]) +def test_source_retains_last_good_data_across_malformed_edit(tmp_path, sources, caplog, mode): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + + # A malformed edit produces no snapshot: the previously applied overrides stay in + # effect because the sink is never called. The failure is logged. + write_file(path, '{"flagValues"') + sink.require_no_snapshot(1.5) + deadline = time.time() + TEST_TIMEOUT + while not any('Unable to load flag data' in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "the load failure was not logged" + time.sleep(0.01) + + # Fixing the file recovers, through the change notification or the failure retry. + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +def test_source_retries_a_failed_load_without_a_change_signal(tmp_path, sources, caplog): + # The fixed content has the same size as the malformed content and the file keeps its + # modification time, so the poller sees no change. Only the automatic retry can recover. + # The initial content has a different size from the malformed content, so the poller sees + # the malformed write even where file times are coarse, as on Windows, and the two writes + # can share a modification time. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true, "flag2": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + + malformed = '{"flagValues": {"flag1": false}' + fixed = '{"flagValues": {"flag1":false}}' + assert len(malformed) == len(fixed) + write_file(path, malformed) + deadline = time.time() + TEST_TIMEOUT + while not any('Unable to load flag data' in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "the load failure was not logged" + time.sleep(0.01) + observed = os.stat(path) + write_file(path, fixed) + os.utime(path, ns=(observed.st_atime_ns, observed.st_mtime_ns)) + assert os.stat(path).st_size == observed.st_size + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +# --------------------------------------------------------------------------- +# Entries marked as deleted +# --------------------------------------------------------------------------- + +def document_with_entry_marked(file_format: str, kind_name: str, deleted: str) -> str: + """ + A document with the flag value flag1 set to "changed" and an entry "d" of the given kind + whose deleted property is given as it is written in the file. + """ + member = 'flags' if kind_name == 'flag' else 'segments' + if file_format == 'json': + return '{"flagValues": {"flag1": "changed"}, "%s": {"d": {"key": "d", "deleted": %s}}}' % (member, deleted) + return 'flagValues:\n flag1: changed\n%s:\n d:\n key: d\n deleted: %s\n' % (member, deleted) + + +def deleted_entry_failure(kind_name: str, path: str) -> str: + return 'Unable to load flag data: %s "d" in %s is marked as deleted; remove the entry to remove its override' % (kind_name, path) + + +def error_lines(caplog) -> List[str]: + return [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] + + +@pytest.mark.parametrize('kind_name', ['flag', 'segment']) +@pytest.mark.parametrize('file_format', ['json', pytest.param('yaml', marks=yaml_required)]) +def test_source_initial_load_fails_for_a_file_with_an_entry_marked_as_deleted(tmp_path, sources, caplog, file_format, kind_name): + # The whole load fails, so the other file's overrides are not applied either. + other = os.path.join(str(tmp_path), 'other.json') + write_file(other, '{"flagValues": {"other": true}}') + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, document_with_entry_marked(file_format, kind_name, 'true')) + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [other, path]) + sink.require_no_snapshot() + assert error_lines(caplog) == [deleted_entry_failure(kind_name, path)] + + +@pytest.mark.parametrize('kind_name', ['flag', 'segment']) +def test_source_reload_fails_for_an_entry_marked_as_deleted_keeps_the_overrides_and_retries(tmp_path, sources, caplog, kind_name): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": "original"}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': 'original'} + + # The edit also changes flag1, and none of it is applied: the sink is not called. + deleted = document_with_entry_marked('json', kind_name, 'true ') + fixed = document_with_entry_marked('json', kind_name, 'false') + assert len(deleted) == len(fixed) + write_file(path, deleted) + deadline = time.time() + TEST_TIMEOUT + while deleted_entry_failure(kind_name, path) not in error_lines(caplog): + assert time.time() < deadline, "the load failure was not logged" + time.sleep(0.01) + sink.require_no_snapshot() + + # The fix keeps the file's size and modification time, so the poller sees no change and + # only the retry of the failed load picks it up. An entry marked as not deleted loads. + observed = os.stat(path) + write_file(path, fixed) + os.utime(path, ns=(observed.st_atime_ns, observed.st_mtime_ns)) + flags, segments = sink.require_snapshot() + assert flags['flag1'].variations == ['changed'] + assert 'd' in (flags if kind_name == 'flag' else segments) + + +@pytest.mark.parametrize('file_format', ['json', pytest.param('yaml', marks=yaml_required)]) +def test_source_loads_entries_marked_as_not_deleted(tmp_path, sources, file_format): + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + if file_format == 'json': + write_file(path, '{"flags": {"f": {"key": "f", "on": true, "variations": ["x"], "fallthrough": {"variation": 0}, "deleted": false}}, ' + '"segments": {"s": {"key": "s", "deleted": false}}}') + else: + write_file(path, 'flags:\n f:\n key: f\n "on": true\n variations: [x]\n fallthrough: {variation: 0}\n deleted: false\n' + 'segments:\n s:\n key: s\n deleted: false\n') + _, sink = build_source(sources, [path]) + snapshot = sink.require_snapshot() + assert flag_values(snapshot) == {'f': 'x'} + assert list(snapshot[1].keys()) == ['s'] + + +@pytest.mark.parametrize("mode", [ChangeDetection.POLLING, pytest.param(ChangeDetection.WATCHING, marks=watchdog_required)]) +def test_source_detects_an_edit_made_during_the_initial_load(tmp_path, sources, monkeypatch, mode): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": "initial"}}') + original_reload_now = filedata.Reloader.reload_now + + def load_then_edit(reloader: filedata.Reloader) -> None: + # The edit lands after the files were read and before start returns. + original_reload_now(reloader) + write_file(path, '{"flagValues": {"flag1": "edited-after-the-first-read"}}') + + monkeypatch.setattr(filedata.Reloader, 'reload_now', load_then_edit) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + + # The initial snapshot holds what the first read saw. The edit is detected and applied + # without any further change to the file. + assert flag_values(sink.require_snapshot()) == {'flag1': 'initial'} + assert flag_values(sink.require_snapshot()) == {'flag1': 'edited-after-the-first-read'} + + +# --------------------------------------------------------------------------- +# In-place rewrites +# --------------------------------------------------------------------------- + +IN_PLACE_REWRITE_CASES = [ + pytest.param('json', ChangeDetection.POLLING, id='json-polling'), + pytest.param('yaml', ChangeDetection.POLLING, marks=yaml_required, id='yaml-polling'), + pytest.param('json', ChangeDetection.WATCHING, marks=watchdog_required, id='json-watching'), + pytest.param('yaml', ChangeDetection.WATCHING, marks=[watchdog_required, yaml_required], id='yaml-watching'), +] + + +def override_document(file_format: str, value: str) -> str: + if file_format == 'json': + return '{"flagValues": {"flag1": "%s"}}' % value + return 'flagValues:\n flag1: "%s"\n' % value + + +def configure_rewrite_timing(monkeypatch, mode: ChangeDetection) -> float: + """ + Returns the pause of a slow in-place rewrite and sets a retry delay longer than it, so that + the rewrite completes before the retry, even on a slow machine. A watch notices the + truncation at once. A poll notices it at the next examination, so in polling mode the pause + is longer than the poll interval, which makes a poll find the file empty. + """ + if mode == ChangeDetection.WATCHING: + pause, retry_delay = 0.3, 1.5 + else: + pause, retry_delay = FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL + 0.6, 3.0 + monkeypatch.setattr(file_override_source, 'DEFAULT_RETRY_DELAY', retry_delay) + return pause + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_source_never_supplies_a_snapshot_without_the_override_during_an_in_place_rewrite(tmp_path, sources, monkeypatch, file_format, mode): + pause = configure_rewrite_timing(monkeypatch, mode) + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': 'a'} + + # The file is empty for the whole pause, and a load finds it empty. The next snapshot holds + # the rewritten override, and the retry that the empty file armed supplies nothing more. + rewrite_in_place(path, override_document(file_format, 'b'), pause) + assert flag_values(sink.require_snapshot()) == {'flag1': 'b'} + sink.require_no_snapshot(file_override_source.DEFAULT_RETRY_DELAY) + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_source_never_supplies_a_snapshot_without_the_override_during_successive_in_place_rewrites(tmp_path, sources, file_format, mode): + # The source's own timing: the default retry delay and a one-second poll interval. An + # operator rewrites the file several times in a row, in place. A watch sees each write and + # the next truncation together; a poll can see a truncation without the content written + # before it. No snapshot may lack the override. + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': 'a'} + + values = ['b', 'c', 'd', 'e'] + for value in values: + rewrite_in_place(path, override_document(file_format, value), 0.3) + if mode == ChangeDetection.POLLING: + # The rewrites start about a second apart. + time.sleep(0.7) + + supplied: List[Dict[str, Any]] = [] + while len(supplied) == 0 or supplied[-1] != {'flag1': values[-1]}: + supplied.append(flag_values(sink.require_snapshot())) + assert all(list(snapshot.keys()) == ['flag1'] for snapshot in supplied), supplied + sink.require_no_snapshot(filedata.DEFAULT_RETRY_DELAY + 0.5) + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_client_serves_the_override_throughout_an_in_place_rewrite(tmp_path, monkeypatch, file_format, mode): + # An operator changes an override by rewriting its file in place. A client that evaluates the + # flag throughout serves the old value and then the new one, never the default. + pause = configure_rewrite_timing(monkeypatch, mode) + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + source = FileOverrideSourceBuilder([path]).change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + user = Context.create('user-key') + + with LDClient(config, start_wait=0) as client: + served: List[Any] = [] + stop = threading.Event() + + def sample() -> None: + while not stop.is_set(): + value = client.variation('flag1', user, 'default') + if len(served) == 0 or served[-1] != value: + served.append(value) + time.sleep(0.001) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + try: + rewrite_in_place(path, override_document(file_format, 'b'), pause) + deadline = time.time() + TEST_TIMEOUT + while served[-1] != 'b': + assert time.time() < deadline, "the rewritten override was not served; served %r" % served + time.sleep(0.01) + time.sleep(0.5) + finally: + stop.set() + sampler.join(TEST_TIMEOUT) + assert served == ['a', 'b'] + + +def reloader_threads() -> Set[threading.Thread]: + return {t for t in threading.enumerate() if t.name == 'ldclient.filedata.reloader'} + + +def test_source_close_stops_the_reloader_when_the_change_detector_fails_to_close(tmp_path, sources, monkeypatch): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + before = reloader_threads() + source, sink = build_source(sources, [path]) + sink.require_snapshot() + started = reloader_threads() - before + assert len(started) == 1 + + detector: Any = source._change_detector + original_close = detector.close + + def failing_close() -> None: + original_close() + raise RuntimeError("detector close failure") + + monkeypatch.setattr(detector, 'close', failing_close) + + # The failure propagates, and the reloader is stopped all the same: its worker ends. + with pytest.raises(RuntimeError): + source.close() + worker = started.pop() + worker.join(TEST_TIMEOUT) + assert not worker.is_alive() + + +def test_source_does_not_supply_snapshots_after_close(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + source.close() + write_file(path, '{"flagValues": {"flag1": false}}') + sink.require_no_snapshot(2.5) + + +def test_source_close_is_idempotent(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + source, _ = build_source(sources, [path]) + source.close() + source.close() + + +def test_source_start_after_close_does_nothing(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source = FileOverrideSourceBuilder([path]).build(SDKConfig('SDK_KEY')) + source.close() + sink = CapturingSink() + source.start(sink) + sink.require_no_snapshot(0.2) + + +# --------------------------------------------------------------------------- +# Through the client +# --------------------------------------------------------------------------- + +def test_file_overrides_end_to_end(tmp_path): + # An operator writes, edits, and empties an override file. Evaluations follow without any + # client restart, even though the client never obtains data from LaunchDarkly. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + user = Context.create('user-key') + source = FileOverrideSourceBuilder([path]).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + + def eventually(predicate, message: str) -> None: + deadline = time.time() + TEST_TIMEOUT + while not predicate(): + assert time.time() < deadline, message + time.sleep(0.05) + + with LDClient(config, start_wait=0) as client: + # Not initialized and no override present: the default is served. + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is False + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + + # An operator adds an override. The running client picks it up. + write_file(path, '{"flagValues": {"overridden-flag": true}}') + eventually(lambda: client.variation('overridden-flag', user, False) is True, "the override was not picked up") + + # The override changes value. + write_file(path, '{"flagValues": {"overridden-flag": false}}') + + def changed() -> bool: + detail = client.variation_detail('overridden-flag', user, True) + return detail.value is False and detail.reason.get('overrideAffected') is True + + eventually(changed, "the changed override was not picked up") + + # The override is removed. The not-initialized short-circuit returns. + write_file(path, '{}') + eventually(lambda: client.variation_detail('overridden-flag', user, False).reason.get('errorKind') == 'CLIENT_NOT_READY', "the removed override was not picked up") + + +def test_client_side_all_flags_state_includes_a_full_definition_with_client_side_but_not_a_value_entry(tmp_path): + # A flagValues entry is not available to client-side SDKs. A full definition that sets + # clientSide is, so that is how an operator overrides a flag that client-side SDKs use. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"value-flag": true}, ' + '"flags": {"full-flag": {"key": "full-flag", "version": 1, "on": true, "variations": [true], ' + '"fallthrough": {"variation": 0}, "clientSide": true}}}') + source = FileOverrideSourceBuilder([path]) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + + with LDClient(config, start_wait=0) as client: + user = Context.create('user-key') + assert set(client.all_flags_state(user).to_values_map().keys()) == {'value-flag', 'full-flag'} + state = client.all_flags_state(user, client_side_only=True) + assert state.valid + assert state.to_values_map() == {'full-flag': True} + + +def test_client_serves_no_override_from_a_file_with_an_entry_marked_as_deleted(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"value-flag": true}, "flags": {"deleted-flag": {"key": "deleted-flag", "deleted": true}}}') + source = FileOverrideSourceBuilder([path]) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + + with LDClient(config, start_wait=0) as client: + user = Context.create('user-key') + not_ready = {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + assert client.variation('deleted-flag', user, 'default') == 'default' + assert client.variation_detail('deleted-flag', user, 'default') == EvaluationDetail('default', None, not_ready) + assert client.variation_detail('value-flag', user, False) == EvaluationDetail(False, None, not_ready) diff --git a/ldclient/testing/integrations/test_filedata.py b/ldclient/testing/integrations/test_filedata.py new file mode 100644 index 00000000..2716c637 --- /dev/null +++ b/ldclient/testing/integrations/test_filedata.py @@ -0,0 +1,2873 @@ +import logging +import os +import shutil +import sys +import threading +import time +from queue import Empty, Queue +from typing import Any, Dict, List, Optional, Set, Tuple +from unittest import mock + +import pytest + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None # type: ignore + +from ldclient.impl.integrations.files import filedata +from ldclient.impl.integrations.files.filedata import ( + DeletedEntryError, + Document, + DuplicateKeyError, + DuplicateKeysHandling, + FileReadError, + FileSummary, + MergeResult, + Poller, + Reloader, + Watcher, + abs_file_paths, + have_watchdog, + have_yaml, + is_yaml_path, + load_files, + make_flag_with_value, + merge, + parse_document, + read_file +) +from ldclient.impl.model import FeatureFlag, Segment +from ldclient.testing.file_util import rewrite_in_place, truncate + +TEST_TIMEOUT = 5.0 + +# Timings for the reloader and poller tests. They are generous multiples of the configured +# delays so the tests stay deterministic on a loaded machine. +SHORT_DELAY = 0.05 +QUIET_PERIOD = 0.3 + +# The pause of the fixture's in-place rewrites, between truncating the file and writing it. +REWRITE_PAUSE = 0.03 + + +def write_file(path: str, content: str) -> None: + with open(path, 'w') as f: + f.write(content) + + +def take(queue: Queue, timeout: float = TEST_TIMEOUT) -> Any: + try: + return queue.get(timeout=timeout) + except Empty: + pytest.fail("timed out waiting for a callback") + + +def require_quiet(queue: Queue, duration: float = QUIET_PERIOD) -> None: + try: + item = queue.get(timeout=duration) + except Empty: + return + pytest.fail("received an unexpected callback: %r" % (item,)) + + +# --------------------------------------------------------------------------- +# Parsing +# --------------------------------------------------------------------------- + +def test_parse_json_document(): + document = parse_document(b'{"flags": {"flag1": {"key": "flag1", "version": 3, "on": true}}, "flagValues": {"flag2": "value2"}, "segments": {"seg1": {"key": "seg1", "version": 2, "included": ["user1"]}}}') + assert list(document.flags.keys()) == ['flag1'] + assert isinstance(document.flags['flag1'], FeatureFlag) + assert document.flags['flag1'].version == 3 + assert document.flags['flag1'].on is True + assert document.flag_values == {'flag2': 'value2'} + assert isinstance(document.segments['seg1'], Segment) + assert document.segments['seg1'].included == {'user1'} + + +@pytest.mark.parametrize('path', ['/data.yaml', '/data.yml', '/data.YAML', '/data.Yml', 'relative/data.yaml', '/x/.yaml', '/x/.YML']) +def test_a_yaml_extension_selects_yaml(path): + assert is_yaml_path(path) + + +@pytest.mark.parametrize('path', ['/data.json', '/data', '/data.txt', '/data.yaml.json', '/yaml', '/data.yaml/file', '/data.yamlx']) +def test_any_other_file_is_json(path): + assert not is_yaml_path(path) + + +def test_parse_chooses_the_parser_by_the_extension_not_the_content(): + # A tab inside a string is valid YAML but not valid JSON. Installing pyyaml does not make a + # JSON file YAML: the extension decides, whatever the content looks like. + content = b' \n {"flagValues": {"flag1": "a\tb"}}' + with pytest.raises(ValueError): + parse_document(content) + with pytest.raises(ValueError): + parse_document(b'flagValues:\n flag1: true\n') + if have_yaml: + assert parse_document(content, is_yaml=True).flag_values == {'flag1': 'a\tb'} + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_parse_yaml_document(): + document = parse_document(b'---\nflags:\n flag1:\n key: flag1\n "on": true\nflagValues:\n flag2: value2\nsegments:\n seg1:\n key: seg1\n', is_yaml=True) + assert document.flags['flag1'].on is True + assert document.flag_values == {'flag2': 'value2'} + assert list(document.segments.keys()) == ['seg1'] + + +def test_parse_yaml_without_pyyaml_fails(monkeypatch): + monkeypatch.setattr(filedata, 'have_yaml', False) + with pytest.raises(ValueError) as excinfo: + parse_document(b'flagValues:\n flag1: true\n', is_yaml=True) + assert 'pyyaml' in str(excinfo.value) + + +@pytest.mark.parametrize('content', [b'', b' \n\t\r\n', b'\xef\xbb\xbf', b'\xef\xbb\xbf \n'], ids=['empty', 'white-space', 'byte-order-mark', 'byte-order-mark-and-white-space']) +def test_parse_empty_json_is_not_a_document(content): + # A file that is rewritten in place is empty for a moment. As JSON it is invalid, so the load + # fails and the last good data stays. + with pytest.raises(ValueError) as excinfo: + parse_document(content) + assert str(excinfo.value) == 'the file has no content, which is not a JSON document' + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +@pytest.mark.parametrize('content', [b'', b' \n\n', b'---\n', b'# no entries yet\n', b'\xef\xbb\xbf'], ids=['empty', 'white-space', 'document-start', 'comment', 'byte-order-mark']) +def test_parse_empty_yaml_has_no_entries(content): + assert parse_document(content, is_yaml=True) == Document() + + +def test_parse_json_object_with_no_members_has_no_entries(): + assert parse_document(b'{}') == Document() + assert parse_document(b' {} \n') == Document() + + +def test_parse_ignores_a_byte_order_mark(): + assert parse_document(b'\xef\xbb\xbf{"flagValues": {"flag1": true}}').flag_values == {'flag1': True} + if have_yaml: + assert parse_document(b'\xef\xbb\xbfflagValues:\n flag1: true\n', is_yaml=True).flag_values == {'flag1': True} + + +def test_parse_fills_in_missing_key_and_version(): + document = parse_document(b'{"flags": {"flag1": {"on": true}}, "segments": {"seg1": {}}}') + assert document.flags['flag1'].key == 'flag1' + assert document.flags['flag1'].version == 1 + assert document.segments['seg1'].key == 'seg1' + assert document.segments['seg1'].version == 1 + + +def test_parse_replaces_a_different_key_with_the_map_key(): + document = parse_document(b'{"flags": {"flag1": {"key": "other-flag", "on": true}}, "segments": {"seg1": {"key": "other-segment"}}}') + assert list(document.flags.keys()) == ['flag1'] + assert document.flags['flag1'].key == 'flag1' + assert list(document.segments.keys()) == ['seg1'] + assert document.segments['seg1'].key == 'seg1' + + +def test_merge_treats_entries_with_the_same_key_property_under_different_map_keys_as_different_flags(): + document = parse_document(b'{"flags": {"flag1": {"key": "shared", "on": true}, "flag2": {"key": "shared", "on": false}}}') + result = merge([document], DuplicateKeysHandling.FAIL) + assert sorted(result.flags.keys()) == ['flag1', 'flag2'] + assert result.flags['flag1'].key == 'flag1' + assert result.flags['flag2'].key == 'flag2' + + +@pytest.mark.parametrize('content', [b'null', b' null\n', b'["a", "b"]', b'1', b'"text"', b'true']) +def test_parse_rejects_a_json_document_that_is_not_an_object(content): + with pytest.raises(ValueError) as excinfo: + parse_document(content) + assert str(excinfo.value) == 'the document must be an object' + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_parse_reads_yaml_null_as_no_entries(): + assert parse_document(b'null\n', is_yaml=True) == Document() + assert parse_document(b'~\n', is_yaml=True) == Document() + + +def test_parse_rejects_documents_that_are_not_objects(): + if have_yaml: + with pytest.raises(ValueError): + parse_document(b'- a\n- b\n', is_yaml=True) + with pytest.raises(ValueError): + parse_document(b'["a", "b"]') + with pytest.raises(ValueError): + parse_document(b'{"flags": ["not", "an", "object"]}') + with pytest.raises(ValueError): + parse_document(b'{"flags": {"flag1": "not an object"}}') + with pytest.raises(ValueError): + parse_document(b'{"segments": {"seg1": 3}}') + with pytest.raises(ValueError): + parse_document(b'{"flagValues": 3}') + + +def test_parse_rejects_malformed_json(): + with pytest.raises(ValueError): + parse_document(b'{"flagValues"') + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_parse_rejects_malformed_yaml(): + with pytest.raises(yaml.YAMLError): + parse_document(b'flagValues: [', is_yaml=True) + + +def test_parse_validates_definition_property_types(): + with pytest.raises(ValueError): + parse_document(b'{"flags": {"flag1": {"key": "flag1", "version": "not a number"}}}') + with pytest.raises(ValueError): + parse_document(b'{"segments": {"seg1": {"key": "seg1", "version": 1, "included": "not a list"}}}') + + +@pytest.mark.parametrize('content', [ + '{"flagValues": {"flag1": true}}'.encode('utf-16'), + '{"flagValues": {"flag1": true}}'.encode('utf-32'), + b'{"flagValues": {"flag1": "\xff"}}', +], ids=['utf-16', 'utf-32', 'invalid-utf-8']) +@pytest.mark.parametrize('is_yaml', [False, pytest.param(True, marks=pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed"))], ids=['json', 'yaml']) +def test_parse_rejects_content_that_is_not_utf_8(content, is_yaml): + with pytest.raises(ValueError): + parse_document(content, is_yaml=is_yaml) + + +def test_reloader_reports_a_file_that_is_not_utf_8_and_keeps_working(make_fixture): + # A file in another encoding fails to load like any other content that cannot be parsed, and + # the reloader keeps loading the files when they change. + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.reload_now() + f.require_applied() + with open(f.path, 'wb') as out: + out.write('{"flagValues": {"flag1": false}}'.encode('utf-16')) + f.reloader.trigger() + assert isinstance(f.require_errored(), FileReadError) + write_file(f.path, '{"flagValues": {"flag1": false}}') + f.reloader.trigger() + assert f.require_applied().flags['flag1'].variations == [False] + + +def test_read_file_json_and_errors(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, '{"flagValues": {"flag1": true}}') + assert read_file(path).flag_values == {'flag1': True} + + with pytest.raises(FileReadError) as excinfo: + read_file(os.path.join(str(tmp_path), 'missing.json')) + assert excinfo.value.path == os.path.join(str(tmp_path), 'missing.json') + assert 'unable to read file' in str(excinfo.value) + + write_file(path, '{"flagValues"') + with pytest.raises(FileReadError) as excinfo: + read_file(path) + assert 'error parsing file' in str(excinfo.value) + assert path in str(excinfo.value) + + write_file(path, '') + with pytest.raises(FileReadError) as excinfo: + read_file(path) + assert 'no content' in str(excinfo.value) + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_read_file_yaml(tmp_path): + path = os.path.join(str(tmp_path), 'data.yaml') + write_file(path, 'flagValues:\n flag1: true\n') + assert read_file(path).flag_values == {'flag1': True} + write_file(path, '') + assert read_file(path) == Document() + + +def test_abs_file_paths(): + paths = abs_file_paths(['relative/data.json', '/absolute/data.json']) + assert paths[0] == os.path.abspath('relative/data.json') + assert paths[1] == os.path.abspath('/absolute/data.json') + + +def test_make_flag_with_value_is_on_and_serves_the_value_by_fallthrough(): + flag = make_flag_with_value('flag1', 'value1') + assert flag.key == 'flag1' + assert flag.version == 1 + assert flag.on is True + assert flag.off_variation is None + assert flag.fallthrough.variation == 0 + assert flag.variations == ['value1'] + assert flag.to_json_dict() == {'key': 'flag1', 'version': 1, 'on': True, 'fallthrough': {'variation': 0}, 'variations': ['value1']} + + +# --------------------------------------------------------------------------- +# Merging +# --------------------------------------------------------------------------- + +def doc(json_text: str) -> Document: + return parse_document(json_text.encode('utf-8')) + + +def test_merge_combines_documents(): + result = merge([ + doc('{"flags": {"flag1": {"key": "flag1", "version": 1}}, "segments": {"seg1": {"key": "seg1", "version": 1}}}'), + doc('{"flagValues": {"flag2": "value2"}}'), + ], DuplicateKeysHandling.FAIL) + assert list(result.flags.keys()) == ['flag1', 'flag2'] + assert list(result.segments.keys()) == ['seg1'] + assert result.flags['flag2'].variations == ['value2'] + assert result.flags['flag2'].on is True + assert result.flags['flag2'].fallthrough.variation == 0 + + +def test_merge_duplicate_keys_fail(): + documents = [doc('{"flagValues": {"flag1": "a"}}'), doc('{"flags": {"flag1": {"key": "flag1", "version": 1}}}')] + with pytest.raises(DuplicateKeyError) as excinfo: + merge(documents, DuplicateKeysHandling.FAIL) + assert "flag 'flag1' is specified by multiple files" in str(excinfo.value) + + segment_documents = [doc('{"segments": {"seg1": {"key": "seg1"}}}'), doc('{"segments": {"seg1": {"key": "seg1"}}}')] + with pytest.raises(DuplicateKeyError) as excinfo: + merge(segment_documents, DuplicateKeysHandling.FAIL) + assert "segment 'seg1' is specified by multiple files" in str(excinfo.value) + + +def test_merge_duplicate_keys_within_one_document_between_flags_and_flag_values_fail(): + documents = [doc('{"flags": {"flag1": {"key": "flag1", "version": 1}}, "flagValues": {"flag1": "a"}}')] + with pytest.raises(DuplicateKeyError): + merge(documents, DuplicateKeysHandling.FAIL) + + +def test_merge_duplicate_keys_ignore_keeps_first(): + result = merge([ + doc('{"flagValues": {"flag1": "first"}}'), + doc('{"flagValues": {"flag1": "second", "flag2": "other"}}'), + ], DuplicateKeysHandling.IGNORE) + assert result.flags['flag1'].variations == ['first'] + assert list(result.flags.keys()) == ['flag1', 'flag2'] + + +def test_merge_preserves_document_order(): + result = merge([ + doc('{"flagValues": {"b": 1, "a": 2}}'), + doc('{"flagValues": {"c": 3}}'), + doc('{"flags": {"d": {"key": "d", "version": 1}}}'), + ], DuplicateKeysHandling.FAIL) + assert list(result.flags.keys()) == ['b', 'a', 'c', 'd'] + + +def test_merge_counts_entries_kept_from_each_document(): + result = merge([ + doc('{"flagValues": {"flag1": true, "flag2": false}, "segments": {"seg": {"key": "seg"}}}'), + doc('{"flagValues": {"flag2": true, "flag3": true}}'), + doc('{}'), + ], DuplicateKeysHandling.IGNORE) + assert [(d.flags, d.segments) for d in result.documents] == [(2, 1), (1, 0), (0, 0)] + + +# --------------------------------------------------------------------------- +# Loading files +# --------------------------------------------------------------------------- + +def test_load_files_fails_on_missing_path_by_default(tmp_path): + present = os.path.join(str(tmp_path), 'present.json') + missing = os.path.join(str(tmp_path), 'missing.json') + write_file(present, '{"flagValues": {"flag1": true}}') + with pytest.raises(FileReadError) as excinfo: + load_files([present, missing], DuplicateKeysHandling.FAIL) + assert excinfo.value.path == missing + + +def test_load_files_skips_missing_paths_when_configured(tmp_path): + present = os.path.join(str(tmp_path), 'present.json') + missing = os.path.join(str(tmp_path), 'missing.json') + write_file(present, '{"flagValues": {"flag1": true}}') + result = load_files([present, missing], DuplicateKeysHandling.FAIL, skip_missing_paths=True) + assert list(result.flags.keys()) == ['flag1'] + assert result.files == [FileSummary(path=present, present=True, flags=1), FileSummary(path=missing, present=False)] + + +def test_load_files_reports_parse_error_with_path(tmp_path): + path = os.path.join(str(tmp_path), 'bad.json') + write_file(path, '{"flagValues"') + with pytest.raises(FileReadError) as excinfo: + load_files([path], DuplicateKeysHandling.FAIL) + assert excinfo.value.path == path + + +@pytest.mark.parametrize('member, kind_name', [('flags', 'flag'), ('segments', 'segment')]) +def test_load_files_fails_for_an_entry_marked_as_deleted_and_names_the_file_and_the_key(tmp_path, member, kind_name): + good = os.path.join(str(tmp_path), 'good.json') + path = os.path.join(str(tmp_path), 'deleted.json') + write_file(good, '{"flagValues": {"flag1": true}}') + write_file(path, '{"%s": {"entry": {"key": "entry", "deleted": true}}}' % member) + expected = '%s "entry" in %s is marked as deleted; remove the entry to remove its override' % (kind_name, path) + + with pytest.raises(DeletedEntryError) as excinfo: + load_files([good, path], DuplicateKeysHandling.FAIL) + # It is a per-file failure, like a file that cannot be parsed. + assert isinstance(excinfo.value, FileReadError) + assert excinfo.value.path == path + assert str(excinfo.value) == expected + + with pytest.raises(DeletedEntryError) as excinfo: + read_file(path) + assert str(excinfo.value) == expected + + +def test_load_files_accepts_entries_marked_as_not_deleted(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, '{"flags": {"flag1": {"key": "flag1", "on": true, "deleted": false}}, "segments": {"seg1": {"key": "seg1", "deleted": false}}}') + result = load_files([path], DuplicateKeysHandling.FAIL) + assert list(result.flags.keys()) == ['flag1'] + assert result.flags['flag1'].on is True + assert list(result.segments.keys()) == ['seg1'] + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_load_files_parses_each_file_in_the_format_of_its_extension(tmp_path): + json_path = os.path.join(str(tmp_path), 'first.json') + yaml_path = os.path.join(str(tmp_path), 'second.YML') + write_file(json_path, '{"flagValues": {"flag1": "json"}}') + write_file(yaml_path, 'flagValues:\n flag2: yaml\n') + result = load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert {key: flag.variations for key, flag in result.flags.items()} == {'flag1': ['json'], 'flag2': ['yaml']} + + # The same YAML content in a file without a YAML extension is not JSON. + write_file(json_path, 'flagValues:\n flag1: json\n') + with pytest.raises(FileReadError) as excinfo: + load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert excinfo.value.path == json_path + + +@pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") +def test_load_files_reads_an_empty_yaml_file_as_no_entries_and_fails_on_an_empty_json_file(tmp_path): + json_path = os.path.join(str(tmp_path), 'data.json') + yaml_path = os.path.join(str(tmp_path), 'data.yaml') + write_file(json_path, '{"flagValues": {"flag1": true}}') + write_file(yaml_path, '') + result = load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert list(result.flags.keys()) == ['flag1'] + assert result.files[1] == FileSummary(path=yaml_path, present=True) + + write_file(json_path, '') + with pytest.raises(FileReadError) as excinfo: + load_files([json_path, yaml_path], DuplicateKeysHandling.FAIL) + assert excinfo.value.path == json_path + assert 'no content' in str(excinfo.value) + + +def test_load_files_reports_duplicate_keys(tmp_path): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"flag1": "first"}}') + write_file(second, '{"flagValues": {"flag1": "second"}}') + with pytest.raises(DuplicateKeyError): + load_files([first, second], DuplicateKeysHandling.FAIL) + result = load_files([first, second], DuplicateKeysHandling.IGNORE) + assert result.flags['flag1'].variations == ['first'] + assert result.files[0].flags == 1 + assert result.files[1].flags == 0 + + +# --------------------------------------------------------------------------- +# Reloader +# --------------------------------------------------------------------------- + +class ReloaderFixture: + def __init__(self, tmp_path, initial_content: str, name: str = 'data.json', **kwargs): + self.path = os.path.join(str(tmp_path), name) + self.applied: Queue = Queue() + self.errored: Queue = Queue() + write_file(self.path, initial_content) + options: Dict[str, Any] = dict( + paths=[self.path], + duplicate_keys_handling=DuplicateKeysHandling.FAIL, + apply=self.applied.put, + on_error=self.errored.put, + ) + options.update(kwargs) + self.reloader = Reloader(**options) + + def write(self, content: str) -> None: + # Applications rewrite data files in place, which is not atomic. The fixture does too, + # deliberately slowly: the file is truncated and written a little later, so a load can + # find it empty, as it would with an application's writes, instead of only by chance. + rewrite_in_place(self.path, content, REWRITE_PAUSE) + + def require_applied(self, timeout: float = TEST_TIMEOUT) -> MergeResult: + return take(self.applied, timeout) + + def require_errored(self, timeout: float = TEST_TIMEOUT) -> Exception: + return take(self.errored, timeout) + + def require_quiet(self, duration: float = QUIET_PERIOD) -> None: + deadline = time.time() + duration + while time.time() < deadline: + if not self.applied.empty(): + pytest.fail("unexpected apply call") + if not self.errored.empty(): + pytest.fail("unexpected on_error call") + time.sleep(0.01) + + def drain(self, duration: float = QUIET_PERIOD) -> Tuple[List[MergeResult], List[Exception]]: + """Collects every apply and on_error call that arrives within the duration.""" + applied: List[MergeResult] = [] + errored: List[Exception] = [] + deadline = time.time() + duration + while time.time() < deadline: + while not self.applied.empty(): + applied.append(self.applied.get_nowait()) + while not self.errored.empty(): + errored.append(self.errored.get_nowait()) + time.sleep(0.01) + return applied, errored + + def close(self) -> None: + self.reloader.close() + + +@pytest.fixture +def make_fixture(tmp_path): + fixtures: List[ReloaderFixture] = [] + + def factory(initial_content: str, **kwargs) -> ReloaderFixture: + fixture = ReloaderFixture(tmp_path, initial_content, **kwargs) + fixtures.append(fixture) + return fixture + + yield factory + for fixture in fixtures: + fixture.close() + + +def test_reloader_initial_load(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.reload_now() + result = f.require_applied() + assert list(result.flags.keys()) == ['flag1'] + assert result.files == [FileSummary(path=f.path, present=True, flags=1)] + + +def test_reloader_fails_on_missing_path_by_default(make_fixture, tmp_path): + missing = os.path.join(str(tmp_path), 'missing.json') + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader = Reloader([f.path, missing], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + err = f.require_errored() + assert isinstance(err, FileReadError) + assert err.path == missing + f.require_quiet(0.1) + + +def test_reloader_skips_missing_paths_when_configured(make_fixture, tmp_path): + second = os.path.join(str(tmp_path), 'second.json') + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader = Reloader([f.path, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, skip_unchanged=True) + + # Step 1: one file exists and one does not. The reload succeeds with the existing file. + f.reloader.reload_now() + result = f.require_applied() + assert list(result.flags.keys()) == ['flag1'] + assert result.files == [FileSummary(path=f.path, present=True, flags=1), FileSummary(path=second, present=False)] + + # Step 2: the missing file appears. Its data is merged in. + write_file(second, '{"flagValues": {"flag2": true}}') + f.reloader.reload_now() + result = f.require_applied() + assert list(result.flags.keys()) == ['flag1', 'flag2'] + + # Step 3: the file is deleted. Its data is gone and the reload still succeeds. + os.remove(second) + f.reloader.reload_now() + result = f.require_applied() + assert list(result.flags.keys()) == ['flag1'] + f.require_quiet(0.1) + + +def test_reloader_reports_failure_and_applies_nothing(make_fixture): + f = make_fixture('{"flagValues"') + f.reloader.reload_now() + err = f.require_errored() + assert isinstance(err, FileReadError) + f.require_quiet(0.1) + + +def test_reloader_reports_merge_failure(make_fixture, tmp_path): + second = os.path.join(str(tmp_path), 'second.json') + write_file(second, '{"flagValues": {"flag1": "dup"}}') + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader = Reloader([f.path, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + err = f.require_errored() + assert isinstance(err, DuplicateKeyError) + + +def test_reloader_debounce_coalesces_triggers(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}', debounce_delay=0.2) + for _ in range(20): + f.reloader.trigger() + f.require_applied() + f.require_quiet() + + +def test_reloader_without_debounce_reloads_on_each_trigger(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.trigger() + f.require_applied() + f.reloader.trigger() + f.require_applied() + + +def test_reloader_debounce_window_is_extended_by_each_trigger(make_fixture): + # The debounce is a settle window: each trigger moves the deadline out again. A stream + # of triggers spaced closer together than the window must produce no reload while the + # stream continues, and exactly one reload after it stops. + window = 0.25 + f = make_fixture('{"flagValues": {"flag1": true}}', debounce_delay=window) + stop = time.time() + 5 * window + while time.time() < stop: + f.reloader.trigger() + time.sleep(window / 5) + assert f.applied.empty(), "a reload ran while triggers were still arriving" + f.require_applied() + f.require_quiet(2 * window) + + +def test_reloader_reports_identical_failure_only_once(make_fixture): + f = make_fixture('{"flagValues"', retry_delay=SHORT_DELAY) + f.reloader.reload_now() + f.require_errored() + # The automatic retries keep failing in the same way. They do not report again. + f.require_quiet() + + # A different failure is reported. The write is in place, so a retry can run while the file + # is empty. That retry reads the file again instead of reporting it, and a retry that finds + # it still empty reports that it has no content. The final content is loaded on demand, and + # every report is checked: each one is the new failure or the empty file, the new failure is + # among them, and once it is the current failure, the retries that repeat it stay silent. + f.write('{"flags": {"flag1": "not an object"}}') + f.reloader.reload_now() + applied, errored = f.drain() + assert applied == [] + assert any('must be an object' in str(e) for e in errored), errored + assert all('must be an object' in str(e) or 'no content' in str(e) for e in errored), errored + f.require_quiet() + + +def reloader_threads() -> Set[threading.Thread]: + return {t for t in threading.enumerate() if t.name == 'ldclient.filedata.reloader'} + + +def test_reloader_unused_spawns_no_thread(make_fixture): + before = reloader_threads() + f = make_fixture('{"flagValues": {"flag1": true}}') + assert reloader_threads() == before + f.reloader.reload_now() + assert len(reloader_threads() - before) == 1 + f.require_applied() + + +def test_reloader_close_does_not_wait_for_in_flight_reload(make_fixture): + entered = threading.Event() + release = threading.Event() + + def blocking_apply(result: MergeResult) -> None: + entered.set() + # The callback stays parked for far longer than close is given, so a close that waits + # for the in-flight reload is detected as a failure rather than hidden by this timeout. + release.wait(TEST_TIMEOUT * 6) + + f = make_fixture('{"flagValues": {"flag1": true}}', apply=blocking_apply) + f.reloader.trigger() + assert entered.wait(TEST_TIMEOUT), "the reload did not start" + + closed = threading.Event() + + def do_close(): + f.reloader.close() + closed.set() + + threading.Thread(target=do_close, daemon=True).start() + assert closed.wait(2.0), "close blocked on an in-flight reload" + release.set() + + +def test_reloader_worker_thread_exits_after_close(make_fixture): + before = reloader_threads() + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.reload_now() + f.require_applied() + started = reloader_threads() - before + assert len(started) == 1 + f.reloader.close() + worker = started.pop() + worker.join(TEST_TIMEOUT) + assert not worker.is_alive() + + +def test_reloader_retries_after_failure_without_further_triggers(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}', retry_delay=SHORT_DELAY) + f.reloader.reload_now() + f.require_applied() + + f.write('{"flagValues"') + f.reloader.trigger() + f.require_errored() + + # Fix the file without triggering. Only the automatic retry can observe the fix. + f.write('{"flagValues": {"flag1": false}}') + f.require_applied() + + +def test_reloader_retries_after_failed_initial_load(make_fixture): + f = make_fixture('{"flagValues"', retry_delay=SHORT_DELAY) + f.reloader.reload_now() + f.require_errored() + f.write('{"flagValues": {"flag1": true}}') + f.require_applied() + + +def test_reloader_stops_retrying_after_success(make_fixture): + # The retry delay is well over the pause of the fixture's in-place writes, so that a retry is + # unlikely to read the file in the middle of one on a slow machine. + f = make_fixture('{"flagValues"', retry_delay=0.2, skip_unchanged=True) + f.reloader.reload_now() + f.require_errored() + + f.write('{"flagValues": {"flag1": true}}') + f.require_applied() + + # After the successful reload there are no further attempts: a changed file is not + # picked up without a trigger. + f.write('{"flagValues": {"flag1": false}}') + f.require_quiet() + + +def test_reloader_does_not_retry_when_retry_delay_is_zero(make_fixture): + f = make_fixture('{"flagValues"') + f.reloader.reload_now() + f.require_errored() + f.write('{"flagValues": {"flag1": true}}') + f.require_quiet() + + +def test_reloader_skip_unchanged(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}', skip_unchanged=True) + f.reloader.reload_now() + f.require_applied() + + f.reloader.trigger() + f.require_quiet() + + f.write('{"flagValues": {"flag1": false}}') + f.reloader.trigger() + f.require_applied() + + +def test_reloader_recovery_applies_even_when_content_unchanged(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}', skip_unchanged=True) + f.reloader.reload_now() + f.require_applied() + + # A reload fails. Consumers hear about it and may move to an interrupted state. + os.remove(f.path) + f.reloader.trigger() + f.require_errored() + + # The file comes back with byte-identical content. The success is applied despite + # skip_unchanged, because only an application tells the consumer the interruption is over. + f.write('{"flagValues": {"flag1": true}}') + f.reloader.trigger() + f.require_applied() + + # Once recovered, identical content skips again. + f.reloader.trigger() + f.require_quiet() + + +def test_reloader_applies_every_reload_when_skip_unchanged_is_off(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.reload_now() + f.require_applied() + f.reloader.trigger() + f.require_applied() + + +def test_reloader_merges_multiple_files_in_order(make_fixture, tmp_path): + second = os.path.join(str(tmp_path), 'second.json') + write_file(second, '{"flagValues": {"flag1": "second"}}') + f = make_fixture('{"flagValues": {"flag1": "first"}}') + f.reloader = Reloader([f.path, second], DuplicateKeysHandling.IGNORE, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + result = f.require_applied() + assert result.flags['flag1'].variations == ['first'] + + +def test_reloader_does_nothing_after_close(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.reload_now() + f.require_applied() + f.reloader.close() + f.reloader.close() + f.reloader.trigger() + f.reloader.reload_now() + f.require_quiet(0.1) + + +def test_reloader_serializes_reload_now_against_worker_reloads(make_fixture): + # Concurrent reload_now calls and triggers must not interleave: every application is a + # complete merged result, and the callback never runs on two threads at once. + in_apply = threading.Lock() + overlaps: List[bool] = [] + applied_keys: List[List[str]] = [] + + def apply(result: MergeResult) -> None: + acquired = in_apply.acquire(blocking=False) + overlaps.append(not acquired) + try: + applied_keys.append(list(result.flags.keys())) + time.sleep(0.002) + finally: + if acquired: + in_apply.release() + + f = make_fixture('{"flagValues": {"flag1": true}}', apply=apply) + threads = [threading.Thread(target=lambda: [f.reloader.reload_now() for _ in range(20)]) for _ in range(4)] + for t in threads: + t.start() + for _ in range(20): + f.reloader.trigger() + for t in threads: + t.join(TEST_TIMEOUT) + f.reloader.close() + assert not any(overlaps) + assert len(applied_keys) > 0 + assert all(keys == ['flag1'] for keys in applied_keys) + + +def test_reloader_reports_an_apply_failure_and_retries_it(make_fixture): + # The consumer rejects every result until it is released. Each call records whether the + # result was accepted. + accept = threading.Event() + calls: Queue = Queue() + + def apply(result: MergeResult) -> None: + accepted = accept.is_set() + calls.put(accepted) + if not accepted: + raise RuntimeError("consumer failure") + + f = make_fixture('{"flagValues": {"flag1": true}}', apply=apply, retry_delay=SHORT_DELAY, skip_unchanged=True) + + # The synchronous load returns normally. The rejection is reported like a load failure. + f.reloader.reload_now() + assert take(calls) is False + assert isinstance(f.require_errored(), RuntimeError) + + # The automatic retry offers the unchanged content again until the consumer accepts it. + accept.set() + while take(calls) is False: + pass + + # Once the consumer has accepted the result, the retries stop. + with pytest.raises(Empty): + calls.get(timeout=QUIET_PERIOD) + + +def test_reloader_retries_when_the_error_callback_raises(make_fixture): + def on_error(err: Exception) -> None: + raise RuntimeError("reporter failure") + + f = make_fixture('{"flagValues"', on_error=on_error, retry_delay=SHORT_DELAY) + + # The failing report does not escape the synchronous load. + f.reloader.reload_now() + + # The automatic retry still runs and picks up the corrected file. + f.write('{"flagValues": {"flag1": true}}') + f.require_applied() + + +# --------------------------------------------------------------------------- +# Reloader: files that are found empty +# --------------------------------------------------------------------------- + +yaml_required = pytest.mark.skipif(not have_yaml, reason="pyyaml is not installed") + +# The retry delay of the tests of a file that is found empty. The retry comes long enough after a +# trigger for a test to finish rewriting the file before it, even on a slow machine. +EMPTY_FILE_RETRY_DELAY = 1.5 + +# Shorter than EMPTY_FILE_RETRY_DELAY, so a load that happens within it after a trigger is not +# the retry. +BEFORE_THE_RETRY = 1.0 + +DEFERRAL_LOG = 'waiting for it to be written' + +YAML_FLAG1 = 'flagValues:\n flag1: true\n' +YAML_FLAG2 = 'flagValues:\n flag2: true\n' + + +def flag_keys(result: MergeResult) -> List[str]: + return list(result.flags.keys()) + + +def deferrals_logged(caplog) -> int: + return len([r for r in caplog.records if r.levelno == logging.DEBUG and DEFERRAL_LOG in r.getMessage()]) + + +def wait_for_deferrals(caplog, count: int) -> None: + deadline = time.monotonic() + TEST_TIMEOUT + while deferrals_logged(caplog) < count: + assert time.monotonic() < deadline, "the empty file was not held back" + time.sleep(0.01) + + +def change_modification_time(path: str) -> None: + """ + Makes sure that the file's modification time differs from what it was, as a rewrite changes + it, even on a file system whose timestamps are too coarse to tell two quick writes apart. + """ + modified = os.stat(path).st_mtime_ns + 1000000 + os.utime(path, ns=(modified, modified)) + + +class FakeClock: + """A monotonic clock that only moves when a test advances it.""" + + def __init__(self): + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +# The retry delay of the tests that time a hold on a fake clock. The timers use the real clock, +# so no retry fires during such a test, and each load is started with reload_now. +HOUR = 3600.0 + + +def make_held_fixture(make_fixture, content: str, name: str = 'data.yaml', **kwargs) -> Tuple[ReloaderFixture, FakeClock]: + clock = FakeClock() + f = make_fixture(content, name=name, retry_delay=HOUR, clock=clock, **kwargs) + f.reloader.reload_now() + f.require_applied() + return f, clock + + +def make_two_file_fixture(make_fixture, tmp_path, first: str, second: str, **kwargs) -> Tuple[ReloaderFixture, FakeClock, str]: + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + write_file(other, second) + f = make_fixture(first, name='data.yaml') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock, **kwargs) + f.reloader.reload_now() + f.require_applied() + return f, clock, other + + +def load(f: ReloaderFixture) -> Optional[MergeResult]: + """Loads the files at once, and returns what was applied, or None when nothing was.""" + f.reloader.reload_now() + try: + return f.applied.get_nowait() + except Empty: + return None + + +def require_held_back(f: ReloaderFixture) -> None: + assert load(f) is None, "the load applied the file instead of holding it back" + assert f.errored.empty(), "the load reported %r" % f.errored.get_nowait() + + +@yaml_required +def test_reloader_holds_back_a_file_until_it_has_stayed_empty_for_exactly_the_retry_delay(make_fixture, caplog): + # A writer that rewrites the file in place truncates it and then writes the new content. A + # load that finds the file empty while its content is in effect keeps the data. In YAML an + # empty file is valid, so applying it would briefly remove every entry. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + assert deferrals_logged(caplog) == 1 + + clock.advance(0.95 * HOUR) + require_held_back(f) + + clock.advance(0.05 * HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_restarts_the_delay_when_the_held_back_file_is_written(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + + # Rewritten and truncated again between two loads: its state changed, so it is a new rewrite. + clock.advance(0.6 * HOUR) + write_file(f.path, YAML_FLAG2) + truncate(f.path) + change_modification_time(f.path) + require_held_back(f) + + clock.advance(0.6 * HOUR) + require_held_back(f) + + clock.advance(0.4 * HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_observes_a_held_back_file_through_a_symbolic_link(make_fixture, tmp_path): + # The configured path is a link to the file that is rewritten, as with a mounted ConfigMap. + # Writing the file changes its state as seen through the link, so the delay starts over. + target = os.path.join(str(tmp_path), 'target.yaml') + link = os.path.join(str(tmp_path), 'link.yaml') + write_file(target, YAML_FLAG1) + try: + os.symlink(target, link) + except (OSError, NotImplementedError): + pytest.skip("symbolic links are not available") + clock = FakeClock() + f = make_fixture(YAML_FLAG1, name='unused.yaml') + f.reloader = Reloader([link], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(target) + require_held_back(f) + clock.advance(0.6 * HOUR) + write_file(target, YAML_FLAG2) + truncate(target) + change_modification_time(target) + require_held_back(f) + clock.advance(0.6 * HOUR) + require_held_back(f) + + +@yaml_required +def test_reloader_forgets_an_emptying_once_the_file_is_read_with_content(make_fixture): + # A file that was held back, then applied with content, and emptied again much later starts a + # new delay, even if it ends up with the same modification time as at the first emptying, as + # when a tool restores the times of the files it writes. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + first = os.stat(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + clock.advance(10 * HOUR) + truncate(f.path) + os.utime(f.path, ns=(first.st_atime_ns, first.st_mtime_ns)) + require_held_back(f) + + +@yaml_required +def test_reloader_holds_back_a_file_emptied_again_after_a_read_with_content_whose_state_looks_the_same(make_fixture, tmp_path): + # On a file system that records modification times coarsely, a file emptied a second time can + # look exactly as it did the first time. The read of its content in between still starts its + # delay over. The other file is kept written while empty, so that no load applies. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + first_emptying = os.stat(f.path) + truncate(other) + require_held_back(f) + + clock.advance(50 / 60 * HOUR) + write_file(f.path, YAML_FLAG1) + change_modification_time(other) + require_held_back(f) + + clock.advance(20 / 60 * HOUR) + truncate(f.path) + os.utime(f.path, ns=(first_emptying.st_atime_ns, first_emptying.st_mtime_ns)) + change_modification_time(other) + require_held_back(f) + + # The other file gets its content back while this one is still within its new delay. + clock.advance(10 / 60 * HOUR) + write_file(other, YAML_FLAG2) + require_held_back(f) + + +@yaml_required +def test_reloader_holds_back_a_file_rewritten_with_the_same_content_again_later(make_fixture): + # A tool rewrites the file in place with the same content now and then. The load after each + # rewrite is skipped as unchanged, which confirms the data in effect like an application, so + # a later rewrite is held back on its own and not cut short by the first one's bound. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1, skip_unchanged=True) + truncate(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG1) + assert load(f) is None + + clock.advance(6 * HOUR) + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_holds_back_a_file_emptied_again_after_unparsable_content_whose_state_looks_the_same(make_fixture): + # The file is emptied and held back, then saved with content that cannot be parsed, then + # emptied again with the modification time of its first emptying, as a coarse file system can + # make it look. Reading content ended its empty state, so it starts a new delay. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + first_emptying = os.stat(f.path) + require_held_back(f) + write_file(f.path, 'flagValues: [') + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + + clock.advance(70 / 60 * HOUR) + truncate(f.path) + os.utime(f.path, ns=(first_emptying.st_atime_ns, first_emptying.st_mtime_ns)) + require_held_back(f) + + +@yaml_required +def test_reloader_applies_the_written_content_of_a_held_back_file_at_once(make_fixture): + # Two quick rewrites in place: the second truncation follows the first write before any load + # reads the new content, as when both fall in one debounce window. The file is empty at every + # load, and its emptiness is not applied; the content that is finally written is. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + truncate(f.path) + change_modification_time(f.path) + clock.advance(0.2 * HOUR) + require_held_back(f) + + write_file(f.path, 'flagValues:\n flag3: true\n') + result = load(f) + assert result is not None and flag_keys(result) == ['flag3'] + + +@yaml_required +@pytest.mark.parametrize('content', [' \n\n', '# no entries yet\n', '---\n', 'null\n'], ids=['white-space', 'comment', 'document-marker', 'null']) +def test_reloader_holds_back_a_yaml_file_that_has_no_entries(make_fixture, content): + # A read can see only the start of a YAML file that is being written: white space, a comment + # or a document marker, which the parser reads as nothing. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + write_file(f.path, content) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_applies_a_yaml_object_with_no_members_at_once(make_fixture, caplog): + # {} is a document with no entries, written on purpose to remove them, not part of a file. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + write_file(f.path, '{}\n') + result = load(f) + assert result is not None and result.flags == {} + assert deferrals_logged(caplog) == 0 + + +@yaml_required +def test_reloader_holds_back_files_emptied_together_for_one_retry_delay(make_fixture, tmp_path, caplog): + # Every file is read and classified before any is held back, so two files that are emptied + # together start their delays in the same load and are applied together. + caplog.set_level(logging.DEBUG) + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + truncate(other) + require_held_back(f) + assert deferrals_logged(caplog) == 2 + + clock.advance(HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_holds_back_an_emptied_file_while_another_file_changes(make_fixture, tmp_path): + # A change to one file starts a load while another is in the middle of a rewrite. The load + # applies all files or none, so the change waits too. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + + write_file(other, 'flagValues:\n flag2: false\n') + clock.advance(0.3 * HOUR) + require_held_back(f) + + write_file(f.path, YAML_FLAG1) + result = load(f) + assert result is not None and flag_keys(result) == ['flag1', 'flag2'] + assert result.flags['flag2'].variations == [False] + + +def test_reloader_fails_on_a_file_that_cannot_be_parsed_while_another_is_held_back(make_fixture, tmp_path): + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.json') + write_file(other, '{"flagValues": {"flag2": true}}') + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + # One file is mid-rewrite and the other is broken. The broken file is reported at once. + truncate(f.path) + write_file(other, '{"flagValues"') + assert load(f) is None + err = f.errored.get_nowait() + assert isinstance(err, FileReadError) + assert err.path == other + + +@yaml_required +def test_reloader_stops_holding_back_a_file_that_keeps_being_emptied_after_five_retry_delays(make_fixture): + # Every load finds the file empty with a new modification time, so its delay keeps starting + # over. The run of loads that hold it back ends after five retry delays. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + for _ in range(9): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + require_held_back(f) + + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + result = load(f) + assert result is not None and result.flags == {} + + +@yaml_required +def test_reloader_stops_holding_back_files_rewritten_in_turn_after_five_retry_delays(make_fixture, tmp_path): + # Two files are rewritten in turn, so every load finds one of them empty, and each is read + # with content in between. The run of loads that hold files back still ends after five retry + # delays. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + paths = [f.path, other] + contents = [YAML_FLAG1, YAML_FLAG2] + truncate(f.path) + require_held_back(f) + for i in range(1, 10): + clock.advance(0.5 * HOUR) + write_file(paths[(i - 1) % 2], contents[(i - 1) % 2]) + truncate(paths[i % 2]) + require_held_back(f) + + clock.advance(0.5 * HOUR) + write_file(paths[1], contents[1]) + truncate(paths[0]) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_measures_a_file_from_its_own_read_when_an_earlier_file_reads_slowly(make_fixture, tmp_path, monkeypatch): + # The read of the earlier file stalls while the later file is being rewritten in place. The + # later file's delay counts from when it was read, not from when the load began. + clock = FakeClock() + slow = os.path.join(str(tmp_path), 'slow.yaml') + write_file(slow, YAML_FLAG2) + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([slow, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + original = filedata._read_file + stall = [True] + + def read_file(path: str, skip_missing_paths: bool): + read = original(path, skip_missing_paths) + if path == slow and stall[0]: + truncate(f.path) + clock.advance(0.9 * HOUR) + return read + + monkeypatch.setattr(filedata, '_read_file', read_file) + require_held_back(f) + stall[0] = False + clock.advance(0.2 * HOUR) + require_held_back(f) + + write_file(f.path, 'flagValues:\n flag3: true\n') + result = load(f) + assert result is not None and flag_keys(result) == ['flag2', 'flag3'] + + +@yaml_required +def test_reloader_measures_a_file_from_its_own_read_when_a_later_file_reads_slowly(make_fixture, tmp_path, monkeypatch): + # The file is found empty before the read of a later file stalls. Its delay counts from its + # own read, so it is over one retry delay after that read, and the retry the hold asks for + # comes then. + clock = FakeClock() + slow = os.path.join(str(tmp_path), 'slow.yaml') + write_file(slow, YAML_FLAG2) + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, slow], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + original = filedata._read_file + stall = [True] + + def read_file(path: str, skip_missing_paths: bool): + read = original(path, skip_missing_paths) + if path == slow and stall[0]: + clock.advance(0.9 * HOUR) + return read + + monkeypatch.setattr(filedata, '_read_file', read_file) + truncate(f.path) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.1 * HOUR + 0.1 * HOUR) + stall[0] = False + clock.advance(0.1 * HOUR) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_a_failing_other_file_does_not_restart_the_bound(make_fixture, tmp_path): + # The file keeps being written while every load finds it empty, and the other file fails to + # parse every other load. The failures do not start the file's bound over. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + for i in range(1, 10): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(other, 'flagValues: [' if i % 2 == 1 else YAML_FLAG2) + assert load(f) is None + + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(other, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_another_files_bound_does_not_cut_a_rewrite_short(make_fixture, tmp_path): + # One file keeps being written while every load finds it empty, until its bound is over. + # The other file is a quarter of a retry delay into an ordinary rewrite then, and stays held + # back until it is written. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + for _ in range(9): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + require_held_back(f) + + clock.advance(0.25 * HOUR) + change_modification_time(f.path) + truncate(other) + require_held_back(f) + + clock.advance(0.25 * HOUR) + change_modification_time(f.path) + require_held_back(f) + + write_file(other, 'flagValues:\n flag2: false\n') + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + assert result.flags['flag2'].variations == [False] + + +@yaml_required +def test_reloader_applies_files_emptied_in_staggered_turns_after_the_bound(make_fixture, tmp_path): + # Two files stay empty and are truncated again in turn, so that whenever one of them has + # stayed empty for the whole delay, the other is still within its delay and holds the load + # back. A file that settles so does not get a fresh bound while the other is held back. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + applied = False + for i in range(1, 15): + clock.advance(54 / 60 * HOUR) + if i == 1: + truncate(other) + change_modification_time(other if i % 2 == 1 else f.path) + applied = load(f) is not None + if applied: + break + # The retry that the hold arms, a little after the other file's delay is over. + clock.advance(12 / 60 * HOUR) + applied = load(f) is not None + clock.advance(-12 / 60 * HOUR) + if applied: + break + assert applied, "the files were held back without end" + + +@yaml_required +def test_reloader_bound_of_a_file_held_back_during_a_failure_starts_over_once_it_is_read_again(make_fixture, tmp_path): + # While the other file fails to load, this file is rewritten once and its gap is held back. + # Much later, the other file is fixed while this one is in the middle of another rewrite, + # which is held back with a bound of its own. + clock = FakeClock() + broken = os.path.join(str(tmp_path), 'other.json') + write_file(broken, '{"flagValues": {"flag3": false}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, broken], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + write_file(broken, '{bad') + truncate(f.path) + assert load(f) is None + write_file(f.path, YAML_FLAG2) + assert load(f) is None + clock.advance(6 * HOUR) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + assert f.errored.empty(), "an identical failure was reported again" + + write_file(broken, '{"flagValues": {"flag3": false}}') + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_hold_asks_for_the_retry_from_the_end_of_the_load(make_fixture, tmp_path, monkeypatch): + # The read of the file after the held one takes longer than what is left of the held file's + # delay, so the retry is due right away, after the margin. + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + truncate(f.path) + require_held_back(f) + + original = filedata._read_file + stall = [True] + + def read_file(path: str, skip_missing_paths: bool): + read = original(path, skip_missing_paths) + if path == other and stall[0]: + stall[0] = False + clock.advance(6 / 60 * HOUR) + return read + + monkeypatch.setattr(filedata, '_read_file', read_file) + clock.advance(57 / 60 * HOUR) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.1 * HOUR) + + +@pytest.mark.parametrize('name, good, broken, fixed', [ + pytest.param('data.yaml', YAML_FLAG1, 'flagValues: [', 'flagValues:\n flag1: false\n', marks=yaml_required, id='yaml'), + pytest.param('data.json', '{"flagValues": {"flag1": true}}', '{"flagValues"', '{"flagValues": {"flag1": false}}', id='json'), +]) +def test_reloader_holds_back_the_fix_of_a_file_saved_in_place_with_an_error(make_fixture, name, good, broken, fixed): + # An operator saves a typo in place, and the fix in place much later. The truncation of the + # second save is held back like the first, so the fix is applied and the file never empty. + f, clock = make_held_fixture(make_fixture, good, name=name) + truncate(f.path) + require_held_back(f) + write_file(f.path, broken) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + for _ in range(6): + clock.advance(HOUR) + assert load(f) is None + assert f.errored.empty(), "an identical failure was reported again" + + truncate(f.path) + require_held_back(f) + write_file(f.path, fixed) + result = load(f) + assert result is not None and result.flags['flag1'].variations == [False] + + +@yaml_required +def test_reloader_file_kept_written_while_empty_is_applied_after_its_bound_while_an_earlier_file_fails(make_fixture, tmp_path): + # The file is kept written while empty. A file before it fails to parse on every other load. + # Those loads still read and hold back this file, so they do not start its bound over. + clock = FakeClock() + first = os.path.join(str(tmp_path), 'first.json') + write_file(first, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([first, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + for i in range(1, 10): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(first, '{"flagValues"' if i % 2 == 1 else '{"flagValues": {"flag3": true}}') + assert load(f) is None + + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + write_file(first, '{"flagValues": {"flag3": true}}') + result = load(f) + assert result is not None and flag_keys(result) == ['flag3'] + + +@yaml_required +def test_reloader_bound_of_a_file_after_a_failing_file_starts_over_once_it_is_read_again(make_fixture, tmp_path): + # The file is held back, then a file configured before it breaks while this one gets its + # content back. The failing loads still read this file, and they hold nothing back, so its + # bound starts over. Much later the other file is fixed while this one is rewritten in place, + # and the truncation is held back. + clock = FakeClock() + first = os.path.join(str(tmp_path), 'first.json') + write_file(first, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([first, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + write_file(first, '{"flagValues"') + write_file(f.path, YAML_FLAG2) + assert load(f) is None + for _ in range(6): + clock.advance(HOUR) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + assert f.errored.empty() + + write_file(first, '{"flagValues": {"flag3": true}}') + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_reads_a_file_after_one_that_cannot_be_read(make_fixture, tmp_path): + # A file configured before this one goes missing, and missing files are not skipped, so it + # cannot be read. The failing loads still read this file, which got its content back, and + # they hold nothing back, so its bound starts over. + clock = FakeClock() + first = os.path.join(str(tmp_path), 'first.json') + write_file(first, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([first, f.path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + os.remove(first) + write_file(f.path, YAML_FLAG2) + assert load(f) is None + assert f.errored.get_nowait().path == first + clock.advance(6 * HOUR) + assert load(f) is None + + write_file(first, '{"flagValues": {"flag3": true}}') + truncate(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_starts_the_bound_of_a_file_that_cannot_be_read_over(make_fixture, monkeypatch): + # The file is held back, then cannot be read for a long time, then is readable again and + # empty after a new write. The failing loads hold nothing back and tried to read the file, + # so its bound started over, and the new emptying is held back. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + + original = filedata._read_file + unreadable = [True] + + def read_file(path: str, skip_missing_paths: bool): + if path == f.path and unreadable[0]: + raise FileReadError(path, "unable to read file: permission denied") + return original(path, skip_missing_paths) + + monkeypatch.setattr(filedata, '_read_file', read_file) + for _ in range(6): + assert load(f) is None + clock.advance(HOUR) + assert isinstance(f.errored.get_nowait(), FileReadError) + + unreadable[0] = False + change_modification_time(f.path) + require_held_back(f) + + +@yaml_required +def test_reloader_keeps_the_delay_of_a_held_back_file_through_a_failed_read(make_fixture, monkeypatch): + # A read that fails says nothing about the content, so the file's delay goes on. Once it can + # be read again, still empty and unmodified, it has been empty for the whole delay. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + + original = filedata._read_file + unreadable = [True] + + def read_file(path: str, skip_missing_paths: bool): + if path == f.path and unreadable[0]: + raise FileReadError(path, "unable to read file: permission denied") + return original(path, skip_missing_paths) + + monkeypatch.setattr(filedata, '_read_file', read_file) + clock.advance(0.5 * HOUR) + assert load(f) is None + assert isinstance(f.errored.get_nowait(), FileReadError) + + unreadable[0] = False + clock.advance(0.5 * HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +def test_reloader_holds_back_a_json_file_that_holds_only_a_byte_order_mark(make_fixture, caplog): + # A byte order mark is not content, so a JSON file in effect that holds only one, as a read + # can see of a file being written, is held back quietly. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, '{"flagValues": {"flag1": true}}', name='data.json') + with open(f.path, 'wb') as out: + out.write(b'\xef\xbb\xbf') + require_held_back(f) + assert deferrals_logged(caplog) == 1 + + +def test_reloader_reports_a_json_file_that_is_not_an_object(make_fixture): + f, clock = make_held_fixture(make_fixture, '{"flagValues": {"flag1": true}}', name='data.json') + write_file(f.path, 'null') + assert load(f) is None + err = f.errored.get_nowait() + assert isinstance(err, FileReadError) + assert 'must be an object' in str(err) + assert 'as JSON, the format of its name' in str(err) + + +def test_reloader_reports_the_first_failure_when_an_empty_file_also_fails(make_fixture, tmp_path): + # The first file cannot be parsed, and the second is empty, which as JSON fails too. The load + # reports the first failure, and logs the other at Debug level. + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues"') + write_file(second, '') + f = make_fixture('{}') + f.reloader = Reloader([first, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + err = f.require_errored() + assert isinstance(err, FileReadError) + assert err.path == first + + +def test_reloader_logs_the_failures_after_the_first(make_fixture, tmp_path, caplog): + caplog.set_level(logging.DEBUG) + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues"') + write_file(second, '{"flags"') + f = make_fixture('{}') + f.reloader = Reloader([first, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + assert f.require_errored().path == first + assert any(r.levelno == logging.DEBUG and second in r.getMessage() for r in caplog.records) + + +def test_reloader_failing_load_reports_the_first_failure(make_fixture, tmp_path): + # Every file is read even after one fails, and the load reports the first failure. + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues"') + write_file(second, '{"flags"') + f = make_fixture('{}') + f.reloader = Reloader([first, second], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put) + f.reloader.reload_now() + err = f.require_errored() + assert isinstance(err, FileReadError) + assert err.path == first + + +@yaml_required +def test_reloader_applies_the_same_bytes_moved_to_a_file_of_another_format(make_fixture, tmp_path): + # The format comes from the extension, so the same bytes moved from x.yaml to x.json parse + # differently: YAML reads 1e3 as a string, JSON as a number. The load is not skipped as + # unchanged. + yaml_path = os.path.join(str(tmp_path), 'x.yaml') + json_path = os.path.join(str(tmp_path), 'x.json') + content = '{"flagValues": {"flag1": 1e3}}' + write_file(yaml_path, content) + f = make_fixture('{}') + f.reloader = Reloader([yaml_path, json_path], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, skip_unchanged=True) + f.reloader.reload_now() + assert f.require_applied().flags['flag1'].variations == ['1e3'] + + os.replace(yaml_path, json_path) + result = load(f) + assert result is not None and result.flags['flag1'].variations == [1000.0] + + +@yaml_required +def test_reloader_load_that_holds_a_file_back_and_then_fails_does_not_start_bounds_over(make_fixture, tmp_path): + # Two files are rewritten in turn, and a third fails to parse on every other load, after one of + # the two has been held back. Those loads hold a file back, so they do not start the bounds + # over, and the files are applied in the end. + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + last = os.path.join(str(tmp_path), 'last.json') + write_file(other, YAML_FLAG2) + write_file(last, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, other, last], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + require_held_back(f) + applied = False + for i in range(1, 13): + clock.advance(0.5 * HOUR) + if i % 2 == 1: + write_file(f.path, YAML_FLAG1) + truncate(other) + write_file(last, '{"flagValues"') + else: + write_file(other, YAML_FLAG2) + truncate(f.path) + write_file(last, '{"flagValues": {"flag3": true}}') + if load(f) is not None: + applied = True + break + assert applied, "the files were held back without end" + + +@yaml_required +def test_reloader_file_found_missing_starts_a_new_delay_when_it_is_created_empty_again(make_fixture, tmp_path): + # The file is emptied and held back, then deleted, then created empty again with the + # modification time of its first emptying, while another file fails to parse, so nothing is + # applied meanwhile. Being found missing ended its empty state, so it starts a new delay. + clock = FakeClock() + broken = os.path.join(str(tmp_path), 'other.json') + write_file(broken, '{"flagValues": {"flag3": true}}') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, broken], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + f.require_applied() + + write_file(broken, '{bad') + truncate(f.path) + first_emptying = os.stat(f.path) + assert load(f) is None + os.remove(f.path) + assert load(f) is None + + clock.advance(70 / 60 * HOUR) + write_file(f.path, '') + os.utime(f.path, ns=(first_emptying.st_atime_ns, first_emptying.st_mtime_ns)) + write_file(broken, '{"flagValues": {"flag3": true}}') + while not f.errored.empty(): + f.errored.get_nowait() + require_held_back(f) + + +@yaml_required +def test_reloader_hold_asks_for_the_retry_after_the_delay_and_a_tenth(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(1.1 * HOUR) + + +@yaml_required +def test_reloader_later_hold_asks_for_the_rest_of_the_delay(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + clock.advance(0.4 * HOUR) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.6 * HOUR + 0.1 * HOUR) + + +@yaml_required +def test_reloader_hold_asks_for_the_retry_when_the_bound_is_over(make_fixture): + # The file was written again, so its delay would end one retry delay from now, but its bound + # ends sooner. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + for _ in range(9): + clock.advance(0.5 * HOUR) + change_modification_time(f.path) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.5 * HOUR + 0.1 * HOUR) + + +@yaml_required +@pytest.mark.parametrize('due_first', ['first', 'last']) +def test_reloader_hold_asks_for_the_retry_when_the_first_held_file_is_due(make_fixture, tmp_path, due_first): + f, clock, other = make_two_file_fixture(make_fixture, tmp_path, YAML_FLAG1, YAML_FLAG2) + earlier, later = (f.path, other) if due_first == 'first' else (other, f.path) + truncate(earlier) + require_held_back(f) + clock.advance(0.5 * HOUR) + truncate(later) + require_held_back(f) + assert f.reloader._last_requested_retry_delay == pytest.approx(0.5 * HOUR + 0.1 * HOUR) + + +@yaml_required +def test_reloader_hold_after_a_change_signal_asks_for_the_same_retry(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + + def wait_for_requested_delay(expected: float) -> None: + deadline = time.monotonic() + TEST_TIMEOUT + while f.reloader._last_requested_retry_delay != pytest.approx(expected): + assert time.monotonic() < deadline, "asked for %r" % f.reloader._last_requested_retry_delay + time.sleep(0.01) + + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + wait_for_requested_delay(1.1 * HOUR) + clock.advance(0.4 * HOUR) + f.reloader.trigger() + wait_for_deferrals(caplog, 2) + wait_for_requested_delay(0.7 * HOUR) + + +def test_reloader_failure_asks_for_the_retry_delay(make_fixture): + f = make_fixture('{"flagValues"', retry_delay=HOUR, clock=FakeClock()) + f.reloader.reload_now() + f.require_errored() + assert f.reloader._last_requested_retry_delay == HOUR + + +@yaml_required +def test_reloader_holds_back_a_file_that_is_emptied_again_after_its_content_was_applied(make_fixture): + # A load that holds no file back ends the run, so a later rewrite is held back on its own. + f, clock = make_held_fixture(make_fixture, YAML_FLAG1) + truncate(f.path) + require_held_back(f) + write_file(f.path, YAML_FLAG2) + clock.advance(0.5 * HOUR) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + clock.advance(10 * HOUR) + truncate(f.path) + require_held_back(f) + clock.advance(HOUR) + result = load(f) + assert result is not None and result.flags == {} + + +def test_reloader_holds_back_a_json_file_that_was_empty_for_long_when_it_is_rewritten(make_fixture, caplog): + # A JSON file that stays empty fails to load once the delay is over, and keeps failing. The + # failures end the run of holds, so when the file is rewritten in place much later, the new + # truncation is held back like any other, without another report. + caplog.set_level(logging.DEBUG) + f, clock = make_held_fixture(make_fixture, '{"flagValues": {"flag1": true}}', name='data.json') + truncate(f.path) + require_held_back(f) + + clock.advance(HOUR) + assert load(f) is None + assert 'no content' in str(f.errored.get_nowait()) + clock.advance(HOUR) + assert load(f) is None + assert f.errored.empty() + + clock.advance(10 * HOUR) + held_before = deferrals_logged(caplog) + truncate(f.path) + change_modification_time(f.path) + require_held_back(f) + assert deferrals_logged(caplog) == held_before + 1 + + write_file(f.path, '{"flagValues": {"flag1": false}}') + result = load(f) + assert result is not None and result.flags['flag1'].variations == [False] + + +@yaml_required +def test_reloader_does_not_hold_back_a_file_that_is_empty_from_the_start(make_fixture, tmp_path, caplog): + # An empty YAML file next to the data, such as a placeholder for later entries, has never had + # content. It does not hold back the changes of the other file. + caplog.set_level(logging.DEBUG) + clock = FakeClock() + placeholder = os.path.join(str(tmp_path), 'placeholder.yaml') + write_file(placeholder, '') + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader = Reloader([f.path, placeholder], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, retry_delay=HOUR, clock=clock) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + write_file(f.path, '{"flagValues": {"flag1": false}}') + result = load(f) + assert result is not None and result.flags['flag1'].variations == [False] + assert deferrals_logged(caplog) == 0 + + +@yaml_required +def test_reloader_applies_a_file_that_is_deleted_and_then_created_empty_at_once(make_fixture): + f, clock = make_held_fixture(make_fixture, YAML_FLAG1, skip_missing_paths=True) + + # Deleting a file that had content removes its data at once. + os.remove(f.path) + result = load(f) + assert result is not None and result.flags == {} + + # A missing file contributed no content, so the empty file that replaces it is applied at once. + write_file(f.path, '') + result = load(f) + assert result is not None and result.flags == {} + assert result.files == [FileSummary(path=f.path, present=True)] + + +@yaml_required +def test_reloader_does_not_hold_back_a_file_whose_content_moved_away(make_fixture, tmp_path): + # The content moves from one file to the other. The file that no longer exists contributes no + # content, so when it is created empty, that is applied at once. + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, retry_delay=HOUR, skip_unchanged=True, clock=clock) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + write_file(other, YAML_FLAG1) + os.remove(f.path) + result = load(f) + assert result is not None and result.files == [FileSummary(path=f.path), FileSummary(path=other, present=True, flags=1)] + + write_file(f.path, '') + result = load(f) + assert result is not None and flag_keys(result) == ['flag1'] + assert result.files == [FileSummary(path=f.path, present=True), FileSummary(path=other, present=True, flags=1)] + + +@yaml_required +def test_reloader_holds_back_a_file_whose_content_was_moved_to_it_from_another(make_fixture, tmp_path): + # The file is renamed to the other configured path (mv data.yaml other.yaml). The other + # path's content is in effect now, so an in-place rewrite of it is held back. + clock = FakeClock() + other = os.path.join(str(tmp_path), 'other.yaml') + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader = Reloader([f.path, other], DuplicateKeysHandling.FAIL, apply=f.applied.put, on_error=f.errored.put, skip_missing_paths=True, retry_delay=HOUR, skip_unchanged=True, clock=clock) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + os.replace(f.path, other) + result = load(f) + assert result is not None and result.files == [FileSummary(path=f.path), FileSummary(path=other, present=True, flags=1)] + + truncate(other) + require_held_back(f) + write_file(other, YAML_FLAG2) + result = load(f) + assert result is not None and flag_keys(result) == ['flag2'] + + +@yaml_required +def test_reloader_applies_an_emptied_file_at_once_when_retries_are_off(make_fixture, caplog): + # Nothing would load the file later, so the empty file is parsed as it is. + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml') + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + result = load(f) + assert result is not None and result.flags == {} + assert deferrals_logged(caplog) == 0 + + +def test_reloader_reports_an_emptied_json_file_at_once_when_retries_are_off(make_fixture): + f = make_fixture('{"flagValues": {"flag1": true}}') + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + assert load(f) is None + assert 'no content' in str(f.errored.get_nowait()) + + +def test_reloader_hold_is_not_a_failure(make_fixture, caplog): + # A rewrite in place that writes the same content again changes nothing. The load that finds + # the file empty reports nothing, and is not a failure that a later load recovers from, so + # the unchanged content is not applied again. + caplog.set_level(logging.DEBUG) + content = '{"flagValues": {"flag1": true}}' + f, clock = make_held_fixture(make_fixture, content, name='data.json', skip_unchanged=True) + truncate(f.path) + require_held_back(f) + write_file(f.path, content) + assert load(f) is None + assert f.errored.empty() + assert not any(r.levelno >= logging.WARNING for r in caplog.records), [r.getMessage() for r in caplog.records] + + +@yaml_required +def test_reloader_initial_load_reads_an_empty_yaml_file_as_no_entries(make_fixture): + f = make_fixture('', name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + result = f.require_applied() + assert result.flags == {} + assert result.files == [FileSummary(path=f.path, present=True)] + + +def test_reloader_initial_load_fails_on_an_empty_json_file(make_fixture): + f = make_fixture('', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + assert 'no content' in str(f.require_errored()) + f.require_quiet(0.1) + + +# Real-clock tests: the retries that holds arm do fire. + +@yaml_required +def test_reloader_holds_back_an_emptied_file_until_its_write_triggers_a_load(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY, skip_unchanged=True) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag1'] + + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + f.require_quiet(0.1) + + # The write triggers a load that applies the new content. Nothing was applied in between. + write_file(f.path, YAML_FLAG2) + f.reloader.trigger() + assert flag_keys(f.require_applied(BEFORE_THE_RETRY)) == ['flag2'] + f.require_quiet(0.1) + + +@yaml_required +def test_reloader_applies_an_emptied_yaml_file_that_stays_empty_at_the_retry(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + started = time.monotonic() + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + + # The retry that the hold armed comes when the file has stayed empty and unmodified for the + # retry delay, so that first retry applies it as a document with no entries. + assert f.require_applied().flags == {} + elapsed = time.monotonic() - started + assert EMPTY_FILE_RETRY_DELAY <= elapsed < 2 * EMPTY_FILE_RETRY_DELAY + assert deferrals_logged(caplog) == 1 + + +@yaml_required +def test_reloader_a_further_hold_does_not_put_off_the_retry(make_fixture, caplog): + # A load that holds the file back again arms the retry for the time that is left of the + # delay, so the file is still applied one retry delay after it was found empty. + caplog.set_level(logging.DEBUG) + retry_delay = 3.0 + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=retry_delay) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + started = time.monotonic() + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + time.sleep(1.5) + f.reloader.trigger() + wait_for_deferrals(caplog, 2) + + # A whole retry delay from the second hold would come at 4.5 s. + assert f.require_applied().flags == {} + elapsed = time.monotonic() - started + assert retry_delay <= elapsed < retry_delay + 1.0 + + +def test_reloader_says_why_it_loads_the_files_again(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture('{"flagValues": {"flag1": true}}', retry_delay=0.2) + f.reloader.reload_now() + f.require_applied() + + # The retry that a hold arms reads the files again because a file was found empty. It finds + # the file still empty, which fails as JSON, and the retry after that failure says so. + truncate(f.path) + f.reloader.trigger() + for expected in ('Reading flag data again after a file was found empty', 'Retrying flag data load after earlier failure'): + deadline = time.monotonic() + TEST_TIMEOUT + while not any(r.getMessage() == expected for r in caplog.records): + assert time.monotonic() < deadline, "no log line %r" % expected + time.sleep(0.01) + + +def test_reloader_reports_an_emptied_json_file_only_at_the_retry_and_keeps_the_data(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture('{"flagValues": {"flag1": true}}', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + + # The load that the trigger starts holds the file back instead of reporting it. The retry + # finds it still empty and unmodified, and an empty file is not a JSON document. + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + f.require_quiet(0.1) + err = f.require_errored() + assert isinstance(err, FileReadError) + assert 'no content' in str(err) + f.require_quiet(0.1) + + +@yaml_required +def test_reloader_holds_back_a_second_rewrite_that_the_retry_of_the_first_finds_empty(make_fixture, caplog): + # The first rewrite is held back. Its content is written and the file is truncated again by + # a second rewrite before any load reads that content, as when a polled file is rewritten + # about a second apart and the poll sees only the second truncation. The retry that the + # first rewrite armed finds the file empty again, and must not apply it. + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + + # No trigger: only the retry reads the file. + write_file(f.path, YAML_FLAG2) + truncate(f.path) + change_modification_time(f.path) + wait_for_deferrals(caplog, 2) + f.require_quiet(0.2) + + write_file(f.path, 'flagValues:\n flag3: true\n') + f.reloader.trigger() + assert flag_keys(f.require_applied(BEFORE_THE_RETRY)) == ['flag3'] + + +@yaml_required +def test_reloader_retry_that_finds_a_file_emptied_holds_it_back(make_fixture, caplog): + # A retry left over from a failure can run while the file is being rewritten in place. It + # holds the file back too, rather than applying it empty. + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY) + f.reloader.reload_now() + f.require_applied() + write_file(f.path, 'flagValues: [') + f.reloader.trigger() + f.require_errored() + + # No trigger: only the retry reads the file. + truncate(f.path) + wait_for_deferrals(caplog, 1) + write_file(f.path, YAML_FLAG2) + assert flag_keys(f.require_applied()) == ['flag2'] + f.require_quiet(0.1) + + +@yaml_required +def test_reloader_retry_left_over_from_a_hold_applies_nothing_after_a_load_on_demand(make_fixture, caplog): + caplog.set_level(logging.DEBUG) + f = make_fixture(YAML_FLAG1, name='data.yaml', retry_delay=EMPTY_FILE_RETRY_DELAY, skip_unchanged=True) + f.reloader.reload_now() + f.require_applied() + truncate(f.path) + f.reloader.trigger() + wait_for_deferrals(caplog, 1) + + # A load on demand reads the rewritten file before the retry does. + write_file(f.path, YAML_FLAG2) + f.reloader.reload_now() + assert flag_keys(f.require_applied()) == ['flag2'] + + # The retry still runs. It finds the content that was just applied, so it applies and + # reports nothing. + f.require_quiet(EMPTY_FILE_RETRY_DELAY + 0.5) + + +# --------------------------------------------------------------------------- +# Poller +# --------------------------------------------------------------------------- + +POLL_INTERVAL = 0.05 + + +class PollerFixture: + def __init__(self, paths: List[str]): + self.changes: Queue = Queue() + self.poller = Poller(paths, POLL_INTERVAL, lambda: self.changes.put(True)) + self.poller.start() + + def require_change(self) -> None: + take(self.changes) + + def require_no_change(self, duration: float = QUIET_PERIOD) -> None: + require_quiet(self.changes, duration) + + def drain(self, settle: float) -> None: + """Discards the change callbacks that arrive within the settle window. + + A plain rewrite is not atomic: the truncation, the write, and the modification-time + update are separate steps, and a poll that lands between two of them legitimately + reports an extra change. A test that wants to assert silence waits for the write to + be fully observed first. + """ + deadline = time.monotonic() + settle + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return + try: + self.changes.get(timeout=remaining) + except Empty: + return + + +@pytest.fixture +def make_poller(): + pollers: List[PollerFixture] = [] + + def factory(paths: List[str]) -> PollerFixture: + fixture = PollerFixture(paths) + pollers.append(fixture) + return fixture + + yield factory + for fixture in pollers: + fixture.poller.close() + + +def set_mtime(path: str, seconds: float) -> None: + os.utime(path, (seconds, seconds)) + + +def test_poller_detects_modification(tmp_path, make_poller): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + p = make_poller([path]) + p.require_no_change(0.15) + write_file(path, 'bb') + p.require_change() + + +def test_poller_detects_same_size_rewrite_with_new_mtime(tmp_path, make_poller): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'aaa') + set_mtime(path, 1000000) + p = make_poller([path]) + write_file(path, 'bbb') + set_mtime(path, 2000000) + p.require_change() + + +def test_poller_detects_size_change_with_same_mtime(tmp_path, make_poller): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'aaa') + set_mtime(path, 1000000) + p = make_poller([path]) + write_file(path, 'aaaa') + set_mtime(path, 1000000) + p.require_change() + + +def test_poller_stops_firing_when_file_is_stable(tmp_path, make_poller): + # The poller updates its baseline after a change, so it goes quiet once the file is stable. + # It does not promise one callback per logical change: a rewrite that it observes in more + # than one step is reported in more than one step, and the reloader absorbs the excess. + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + p = make_poller([path]) + write_file(path, 'bb') + p.require_change() + p.drain(POLL_INTERVAL * 10) + p.require_no_change() + + +def test_poller_detects_file_appearing(tmp_path, make_poller): + path = os.path.join(str(tmp_path), 'data.json') + p = make_poller([path]) + p.require_no_change(0.15) + write_file(path, 'a') + p.require_change() + + +def test_poller_detects_file_disappearing(tmp_path, make_poller): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + p = make_poller([path]) + os.remove(path) + p.require_change() + + +def test_poller_watches_all_files(tmp_path, make_poller): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, 'a') + write_file(second, 'a') + p = make_poller([first, second]) + write_file(second, 'bb') + p.require_change() + write_file(first, 'bb') + p.require_change() + + +def test_poller_stops_on_close(tmp_path, make_poller): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + p = make_poller([path]) + p.poller.close() + time.sleep(POLL_INTERVAL * 3) + write_file(path, 'bb') + p.require_no_change() + + +def test_poller_close_returns_while_callback_blocks(tmp_path): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + entered = threading.Event() + release = threading.Event() + + def on_change(): + entered.set() + release.wait(TEST_TIMEOUT) + + poller = Poller([path], POLL_INTERVAL, on_change) + poller.start() + write_file(path, 'bb') + assert entered.wait(TEST_TIMEOUT) + closed = threading.Event() + + def do_close(): + poller.close() + closed.set() + + threading.Thread(target=do_close, daemon=True).start() + assert closed.wait(TEST_TIMEOUT), "close blocked on the callback" + release.set() + + +# --------------------------------------------------------------------------- +# Watcher +# --------------------------------------------------------------------------- + +watchdog_required = pytest.mark.skipif(not have_watchdog, reason="watchdog is not installed") + + +class WatcherFixture: + def __init__(self, paths: List[str]): + self.changes: Queue = Queue() + self.watcher = Watcher(paths, lambda: self.changes.put(True)) + + def require_change(self) -> None: + take(self.changes) + + def require_no_change(self, duration: float = QUIET_PERIOD) -> None: + require_quiet(self.changes, duration) + + def drain(self) -> None: + while True: + try: + self.changes.get(timeout=0.2) + except Empty: + return + + +@pytest.fixture +def make_watcher(): + watchers: List[WatcherFixture] = [] + + def factory(paths: List[str]) -> WatcherFixture: + fixture = WatcherFixture(paths) + watchers.append(fixture) + return fixture + + yield factory + for fixture in watchers: + fixture.watcher.close() + + +@watchdog_required +def test_watcher_detects_modification(tmp_path, make_watcher): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + w = make_watcher([path]) + w.require_no_change(0.15) + write_file(path, 'bb') + w.require_change() + + +@watchdog_required +def test_watcher_ignores_other_files_in_the_directory(tmp_path, make_watcher): + path = os.path.join(str(tmp_path), 'data.json') + other = os.path.join(str(tmp_path), 'other.json') + write_file(path, 'a') + w = make_watcher([path]) + write_file(other, 'bb') + w.require_no_change() + + +@watchdog_required +def test_watcher_detects_absent_file_appearing(tmp_path, make_watcher): + path = os.path.join(str(tmp_path), 'data.json') + w = make_watcher([path]) + w.require_no_change(0.15) + write_file(path, 'a') + w.require_change() + + +@watchdog_required +def test_watcher_detects_file_written_by_rename(tmp_path, make_watcher): + path = os.path.join(str(tmp_path), 'data.json') + temp = os.path.join(str(tmp_path), 'data.json.tmp') + write_file(path, 'a') + w = make_watcher([path]) + write_file(temp, 'bb') + w.drain() + os.replace(temp, path) + w.require_change() + + +@watchdog_required +def test_watcher_matches_the_destination_of_a_move_event(tmp_path, make_watcher): + # A file written by rename arrives as a move event whose destination is the watched path. + import watchdog.events + + path = os.path.join(str(tmp_path), 'data.json') + temp = os.path.join(str(tmp_path), 'data.json.tmp') + real_path = os.path.join(os.path.realpath(str(tmp_path)), 'data.json') + real_temp = os.path.join(os.path.realpath(str(tmp_path)), 'data.json.tmp') + w = make_watcher([path]) + w.watcher._handle_event(watchdog.events.FileMovedEvent(real_temp, real_path)) + w.require_change() + w.watcher._handle_event(watchdog.events.FileMovedEvent(real_path, real_temp)) + w.require_change() + w.watcher._handle_event(watchdog.events.FileMovedEvent(real_temp, real_temp + '.other')) + w.require_no_change() + assert temp not in w.watcher._watched_paths + + +@watchdog_required +def test_watcher_ignores_events_that_do_not_change_the_file(tmp_path, make_watcher): + # Opening and reading a watched file produces notifications too. A reload reads the files, + # so reacting to those would make every reload trigger the next one. + import watchdog.events + + path = os.path.join(str(tmp_path), 'data.json') + real_path = os.path.join(os.path.realpath(str(tmp_path)), 'data.json') + write_file(path, 'a') + w = make_watcher([path]) + w.watcher._handle_event(watchdog.events.FileOpenedEvent(real_path)) + # Older watchdog versions report no event for a read-only close. + closed_no_write = getattr(watchdog.events, 'FileClosedNoWriteEvent', None) + if closed_no_write is not None: + w.watcher._handle_event(closed_no_write(real_path)) + w.require_no_change() + w.watcher._handle_event(watchdog.events.FileClosedEvent(real_path)) + w.require_change() + + +@watchdog_required +def test_watcher_does_not_signal_when_the_file_is_only_read(tmp_path, make_watcher): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + w = make_watcher([path]) + for _ in range(3): + with open(path, 'rb') as f: + f.read() + w.require_no_change() + + +@watchdog_required +def test_watcher_detects_file_deletion(tmp_path, make_watcher): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + w = make_watcher([path]) + os.remove(path) + w.require_change() + + +@watchdog_required +def test_watcher_reports_a_change_within_a_fraction_of_a_second(tmp_path, make_watcher): + # watchdog 7 on Windows holds each notification for the observer's timeout, one second by + # default, before it delivers it. The watcher sets a shorter timeout. + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + w = make_watcher([path]) + w.require_no_change(0.15) + write_file(path, 'bb') + take(w.changes, timeout=0.5) + + +@watchdog_required +def test_watcher_reports_a_change_while_another_file_in_the_directory_keeps_changing(tmp_path, make_watcher): + # watchdog 7 on Windows delivers nothing while notifications for the directory keep arriving + # within the observer's timeout, so a file appended to five times a second, such as a log, + # would hold back the change to the watched file for as long as it is written. + path = os.path.join(str(tmp_path), 'data.json') + other = os.path.join(str(tmp_path), 'app.log') + write_file(path, 'a') + write_file(other, '') + w = make_watcher([path]) + stop = threading.Event() + + def append_to_other(): + while not stop.is_set(): + with open(other, 'a') as f: + f.write('line\n') + stop.wait(0.2) + + writer = threading.Thread(target=append_to_other, daemon=True) + writer.start() + try: + w.require_no_change(0.5) + write_file(path, 'bb') + take(w.changes, timeout=2.0) + finally: + stop.set() + writer.join() + + +@watchdog_required +def test_watcher_reports_a_new_modification_time(tmp_path, make_watcher): + # A touch changes only the modification time. watchdog 7 reports it as an "attrib" event, + # where earlier versions report a modification. + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + w = make_watcher([path]) + w.require_no_change(0.15) + info = os.stat(path) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns + 1_000_000_000)) + w.require_change() + + +def make_symlink(target: str, link: str, directory: bool = False) -> None: + # Windows needs to know whether a link leads to a directory when the link is created. + try: + os.symlink(target, link, target_is_directory=directory) + except (OSError, NotImplementedError) as e: + pytest.skip('symbolic links cannot be created here: %s' % e) + + +@watchdog_required +def test_watcher_detects_a_change_to_the_target_of_a_link_in_another_directory(tmp_path, make_watcher): + data = os.path.join(str(tmp_path), 'data') + config = os.path.join(str(tmp_path), 'config') + os.mkdir(data) + os.mkdir(config) + target = os.path.join(data, 'flags.json') + link = os.path.join(config, 'flags.json') + write_file(target, 'a') + make_symlink(target, link) + w = make_watcher([link]) + w.require_no_change(0.15) + write_file(target, 'bb') + w.require_change() + + +@watchdog_required +def test_watcher_detects_a_change_to_the_target_of_a_link_in_the_same_directory(tmp_path, make_watcher): + target = os.path.join(str(tmp_path), 'real.json') + link = os.path.join(str(tmp_path), 'data.json') + write_file(target, 'a') + make_symlink(target, link) + w = make_watcher([link]) + w.require_no_change(0.15) + write_file(target, 'bb') + w.require_change() + + +@watchdog_required +def test_watcher_follows_a_link_that_is_replaced_to_point_at_another_directory(tmp_path, make_watcher): + one = os.path.join(str(tmp_path), 'one') + two = os.path.join(str(tmp_path), 'two') + config = os.path.join(str(tmp_path), 'config') + for directory in (one, two, config): + os.mkdir(directory) + first = os.path.join(one, 'flags.json') + second = os.path.join(two, 'flags.json') + link = os.path.join(config, 'flags.json') + write_file(first, 'a') + write_file(second, 'a') + make_symlink(first, link) + w = make_watcher([link]) + w.require_no_change(0.15) + + # Replaced by rename, as deployment tools replace a link. + make_symlink(second, link + '.tmp') + os.replace(link + '.tmp', link) + w.require_change() + w.drain() + + write_file(second, 'bb') + w.require_change() + w.drain() + + write_file(first, 'ccc') + w.require_no_change() + + +@watchdog_required +@pytest.mark.skipif(not sys.platform.startswith('linux'), reason='the directory loss is reported through inotify') +def test_watcher_follows_a_link_through_a_directory_link_that_is_replaced_as_in_a_configmap(tmp_path, make_watcher): + # The layout that Kubernetes uses for a mounted ConfigMap: the file is a link through the + # ..data link to a timestamped directory, and an update writes a new directory, replaces + # ..data, and deletes the old directory. + config = os.path.join(str(tmp_path), 'config') + old = os.path.join(config, '..2026_a') + new = os.path.join(config, '..2026_b') + os.mkdir(config) + os.mkdir(old) + write_file(os.path.join(old, 'flags.json'), 'a') + make_symlink('..2026_a', os.path.join(config, '..data'), directory=True) + make_symlink('..data/flags.json', os.path.join(config, 'flags.json')) + w = make_watcher([os.path.join(config, 'flags.json')]) + w.require_no_change(0.15) + + os.mkdir(new) + write_file(os.path.join(new, 'flags.json'), 'bb') + make_symlink('..2026_b', os.path.join(config, '..data_tmp'), directory=True) + os.replace(os.path.join(config, '..data_tmp'), os.path.join(config, '..data')) + shutil.rmtree(old) + w.require_change() + w.drain() + + write_file(os.path.join(new, 'flags.json'), 'ccc') + w.require_change() + + +@watchdog_required +def test_watcher_follows_a_relative_link_in_a_directory_that_is_reached_through_another_link(tmp_path, make_watcher): + # A deployment layout: current is a link to a release directory, and the data file in the + # release is a relative link to a directory next to the releases. + releases = os.path.join(str(tmp_path), 'releases') + os.mkdir(releases) + os.mkdir(os.path.join(releases, '2026')) + os.mkdir(os.path.join(releases, 'shared')) + target = os.path.join(releases, 'shared', 'flags.json') + write_file(target, 'a') + make_symlink('../shared/flags.json', os.path.join(releases, '2026', 'flags.json')) + make_symlink(os.path.join('releases', '2026'), os.path.join(str(tmp_path), 'current'), directory=True) + w = make_watcher([os.path.join(str(tmp_path), 'current', 'flags.json')]) + w.require_no_change(0.15) + write_file(target, 'bb') + w.require_change() + + +@watchdog_required +def test_watcher_signals_when_a_link_resolves_to_a_target_in_a_directory_that_is_already_watched(tmp_path, make_watcher, monkeypatch): + monkeypatch.setattr(filedata, '_WATCH_RETRY_INTERVAL', SHORT_DELAY) + config = os.path.join(str(tmp_path), 'config') + other = os.path.join(str(tmp_path), 'other') + os.mkdir(config) + os.mkdir(other) + write_file(os.path.join(other, 'other.json'), 'a') + # The link leads through data, which does not exist yet, to a directory that the watcher + # already watches because another configured file is in it. + make_symlink(os.path.join(str(tmp_path), 'data', 'flags.json'), os.path.join(config, 'flags.json')) + w = make_watcher([os.path.join(config, 'flags.json'), os.path.join(other, 'other.json')]) + w.require_no_change(0.15) + + write_file(os.path.join(other, 'flags.json'), 'a') + w.drain() + make_symlink(other, os.path.join(str(tmp_path), 'data'), directory=True) + # The retry that resolves the link signals a change so the file is read. + w.require_change() + + +@watchdog_required +def test_watcher_keeps_a_target_watch_that_another_thread_set_up_at_the_same_time(tmp_path, make_watcher, monkeypatch): + data = os.path.join(str(tmp_path), 'data') + config = os.path.join(str(tmp_path), 'config') + os.mkdir(data) + os.mkdir(config) + target = os.path.join(data, 'flags.json') + write_file(target, 'a') + make_symlink(target, os.path.join(config, 'flags.json')) + w = make_watcher([os.path.join(config, 'flags.json')]) + w.require_no_change(0.15) + watcher = w.watcher + + # Two threads resolve the links at once and both find the target's directory unwatched. The + # observer gives both the same watch, and the other thread stores it first. + with watcher._lock: + assert len(watcher._target_directories) == 1 + real_data = next(iter(watcher._target_directories)) + watcher._watches.pop(real_data) + schedule = watcher._schedule + + def schedule_while_another_thread_stores_the_watch(directory): + watch = schedule(directory) + with watcher._lock: + watcher._watches[directory] = watch + return watch + + monkeypatch.setattr(watcher, '_schedule', schedule_while_another_thread_stores_the_watch) + unschedule = mock.Mock(wraps=watcher._observer.unschedule) + monkeypatch.setattr(watcher._observer, 'unschedule', unschedule) + watcher._refresh_links() + + unschedule.assert_not_called() + write_file(target, 'bb') + w.require_change() + + +@watchdog_required +def test_watcher_warns_once_while_a_directory_cannot_be_watched_and_again_once_it_was_watched(tmp_path, make_watcher, monkeypatch, caplog): + monkeypatch.setattr(filedata, '_WATCH_RETRY_INTERVAL', SHORT_DELAY) + caplog.set_level(logging.DEBUG) + directory = os.path.join(str(tmp_path), 'later') + path = os.path.join(directory, 'data.json') + message = 'Cannot watch directory "%s" for changes yet because it does not exist' % directory + + def records(level): + return [r for r in caplog.records if r.levelno == level and r.getMessage() == message] + + w = make_watcher([path]) + # Several retries fail the same way. + time.sleep(SHORT_DELAY * 6) + assert len(records(logging.WARNING)) == 1 + assert len(records(logging.DEBUG)) >= 2 + + os.mkdir(directory) + write_file(path, 'a') + w.require_change() + w.drain() + if not sys.platform.startswith('linux'): + # The loss of a watched directory is reported through inotify. + return + + # Once the directory was watched, losing it again is reported again. + os.remove(path) + os.rmdir(directory) + time.sleep(SHORT_DELAY * 6) + assert len(records(logging.WARNING)) == 2 + + +@watchdog_required +def test_watcher_watches_the_target_of_a_link_once_the_target_directory_appears(tmp_path, make_watcher, monkeypatch): + monkeypatch.setattr(filedata, '_WATCH_RETRY_INTERVAL', SHORT_DELAY) + config = os.path.join(str(tmp_path), 'config') + data = os.path.join(str(tmp_path), 'data') + os.mkdir(config) + target = os.path.join(data, 'flags.json') + make_symlink(target, os.path.join(config, 'flags.json')) + w = make_watcher([os.path.join(config, 'flags.json')]) + w.require_no_change(0.15) + + os.mkdir(data) + write_file(target, 'a') + # The retry that watches the target's directory signals a change so the file is read. + w.require_change() + w.drain() + + write_file(target, 'bb') + w.require_change() + + +@watchdog_required +@pytest.mark.skipif(not sys.platform.startswith('linux'), reason='the directory loss is reported through inotify') +def test_watcher_watches_the_target_of_a_link_again_after_its_directory_is_deleted_and_recreated(tmp_path, make_watcher, monkeypatch): + monkeypatch.setattr(filedata, '_WATCH_RETRY_INTERVAL', SHORT_DELAY) + config = os.path.join(str(tmp_path), 'config') + data = os.path.join(str(tmp_path), 'data') + os.mkdir(config) + os.mkdir(data) + target = os.path.join(data, 'flags.json') + write_file(target, 'a') + make_symlink(target, os.path.join(config, 'flags.json')) + w = make_watcher([os.path.join(config, 'flags.json')]) + w.require_no_change(0.15) + + shutil.rmtree(data) + w.require_change() + w.drain() + + os.mkdir(data) + write_file(target, 'bb') + w.require_change() + w.drain() + + write_file(target, 'ccc') + w.require_change() + + +@watchdog_required +def test_watcher_picks_up_directory_that_appears_later(tmp_path, make_watcher): + directory = os.path.join(str(tmp_path), 'later') + path = os.path.join(directory, 'data.json') + w = make_watcher([path]) + w.require_no_change(0.15) + os.mkdir(directory) + write_file(path, 'a') + # The retry that watches the new directory signals a change so the file is read. + w.require_change() + w.drain() + write_file(path, 'bb') + w.require_change() + + +@watchdog_required +def test_watcher_watches_a_directory_again_after_it_is_deleted_and_recreated(tmp_path, make_watcher, monkeypatch): + monkeypatch.setattr(filedata, '_WATCH_RETRY_INTERVAL', SHORT_DELAY) + directory = os.path.join(str(tmp_path), 'later') + path = os.path.join(directory, 'data.json') + os.mkdir(directory) + write_file(path, 'a') + w = make_watcher([path]) + + # The file and then its directory are removed. + os.remove(path) + os.rmdir(directory) + w.drain() + + # The directory comes back with the file. The retry that watches it again signals a change. + os.mkdir(directory) + write_file(path, 'bb') + w.require_change() + w.drain() + + # Notifications from the recreated directory are delivered. + write_file(path, 'ccc') + w.require_change() + + +@watchdog_required +def test_watcher_retries_when_the_observer_cannot_start(tmp_path, make_watcher, monkeypatch, caplog): + import watchdog.observers + + monkeypatch.setattr(filedata, '_WATCH_RETRY_INTERVAL', SHORT_DELAY) + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + + # The observer cannot start while the watcher is constructed. + with caplog.at_level(logging.ERROR): + with mock.patch.object(watchdog.observers.Observer, 'start', side_effect=RuntimeError("cannot start thread")): + w = make_watcher([path]) + assert len([r for r in caplog.records if r.levelno == logging.ERROR]) > 0 + + # Once the observer can start, the retry sets up the watch and signals a change, and later + # edits are detected. + w.require_change() + w.drain() + write_file(path, 'bb') + w.require_change() + + +@watchdog_required +def test_watcher_close_succeeds_when_the_observer_never_started(tmp_path): + import watchdog.observers + + path = os.path.join(str(tmp_path), 'data.json') + with mock.patch.object(watchdog.observers.Observer, 'start', side_effect=RuntimeError("cannot start thread")): + watcher = Watcher([path], lambda: None) + watcher.close() + + +@watchdog_required +def test_watcher_close_stops_notifications(tmp_path, make_watcher): + path = os.path.join(str(tmp_path), 'data.json') + write_file(path, 'a') + w = make_watcher([path]) + w.watcher.close() + write_file(path, 'bb') + w.require_no_change() diff --git a/ldclient/testing/mock_components.py b/ldclient/testing/mock_components.py index ad93b32b..a569f829 100644 --- a/ldclient/testing/mock_components.py +++ b/ldclient/testing/mock_components.py @@ -1,9 +1,16 @@ +import threading import time -from typing import Callable +from typing import Any, Callable, Dict, Optional +from ldclient.impl.util import _Success from ldclient.interfaces import ( + Basis, BigSegmentStore, BigSegmentStoreMetadata, + ChangeSetBuilder, + IntentCode, + ObjectKind, + OverrideSink, Selector ) @@ -54,3 +61,142 @@ def __init__(self, selector: Selector): def selector(self) -> Selector: return self._selector + + +class MockOverrideSource: + """ + An override source for tests. It pushes its current contents to the sink when started and + on every later call to set_overrides, and records its lifecycle calls. + """ + + def __init__( + self, + flags: Optional[Dict[str, Any]] = None, + segments: Optional[Dict[str, Any]] = None, + start_error: Optional[Exception] = None, + supply_before_start_error: bool = False, + close_error: Optional[Exception] = None, + ): + """ + :param start_error: when given, start raises it instead of supplying the contents + :param supply_before_start_error: with start_error, start supplies the contents and then raises + :param close_error: when given, close raises it after recording the call + """ + self._flags: Dict[str, Any] = dict(flags or {}) + self._segments: Dict[str, Any] = dict(segments or {}) + self._start_error = start_error + self._supply_before_start_error = supply_before_start_error + self._close_error = close_error + self._sink: Optional[OverrideSink] = None + self.start_count = 0 + self.close_count = 0 + + def start(self, sink: OverrideSink) -> None: + self.start_count += 1 + if self._start_error is not None: + if self._supply_before_start_error: + self._sink = sink + sink.set_overrides(self._flags, self._segments) + raise self._start_error + self._sink = sink + sink.set_overrides(self._flags, self._segments) + + def close(self) -> None: + self.close_count += 1 + if self._close_error is not None: + raise self._close_error + + def set_overrides(self, flags: Optional[Dict[str, Any]] = None, segments: Optional[Dict[str, Any]] = None) -> None: + """Replaces the source's contents and, once started, pushes them to the sink.""" + self._flags = dict(flags or {}) + self._segments = dict(segments or {}) + if self._sink is not None: + self._sink.set_overrides(self._flags, self._segments) + + @property + def builder(self) -> 'MockOverrideSourceBuilder': + return MockOverrideSourceBuilder(self) + + +class MockOverrideSourceBuilder: + """Wraps a MockOverrideSource so it can be passed to the data system configuration.""" + + def __init__(self, source: MockOverrideSource): + self._source = source + self.build_count = 0 + + def build(self, config) -> MockOverrideSource: + self.build_count += 1 + return self._source + + +class FailingOverrideSourceBuilder: + """A builder whose build raises, to simulate invalid override source configuration.""" + + def build(self, config): + raise ValueError("invalid override source configuration") + + +class HangingSynchronizer: + """ + A synchronizer that connects but never yields data, so the client stays uninitialized + until it is stopped. + """ + + def __init__(self): + self._stop = threading.Event() + + @property + def name(self) -> str: + return "HangingSynchronizer" + + def sync(self, ss): + self._stop.wait() + yield from () + + def stop(self): + self._stop.set() + + @property + def builder(self) -> 'MockDataSourceBuilder': + return MockDataSourceBuilder(self) + + +class StaticInitializer: + """ + An initializer that supplies fixed flag and segment definitions as a full-transfer basis + with a defined selector, so the client reports full data availability once it has applied + the data. + """ + + def __init__(self, flags: Optional[Dict[str, dict]] = None, segments: Optional[Dict[str, dict]] = None): + self._flags = dict(flags or {}) + self._segments = dict(segments or {}) + + @property + def name(self) -> str: + return "StaticInitializer" + + def fetch(self, ss): + builder = ChangeSetBuilder() + builder.start(IntentCode.TRANSFER_FULL) + for key, flag in self._flags.items(): + builder.add_put(ObjectKind.FLAG, key, flag.get('version', 1), flag) + for key, segment in self._segments.items(): + builder.add_put(ObjectKind.SEGMENT, key, segment.get('version', 1), segment) + change_set = builder.finish(Selector.new_selector('test-state', 1)) + return _Success(Basis(change_set=change_set, persist=False, environment_id=None)) + + @property + def builder(self) -> 'MockDataSourceBuilder': + return MockDataSourceBuilder(self) + + +class MockDataSourceBuilder: + """Wraps a ready-made data source so it can be passed to the data system configuration.""" + + def __init__(self, source): + self._source = source + + def build(self, config): + return self._source diff --git a/ldclient/testing/test_async_client_override_vectors.py b/ldclient/testing/test_async_client_override_vectors.py new file mode 100644 index 00000000..3eab7e3f --- /dev/null +++ b/ldclient/testing/test_async_client_override_vectors.py @@ -0,0 +1,71 @@ +""" +Runs the OVERRIDE specification test vectors through the async client. The vectors and the +checks are those of the sync runner: each vector sets up LaunchDarkly data, an override layer, +and an initialization state, and the test evaluates one flag through the full client stack. +""" +from typing import Any, Dict + +import pytest + +from ldclient.async_client import AsyncLDClient +from ldclient.async_config import AsyncConfig, AsyncDataSystemConfig +from ldclient.context import Context +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.testing.mock_async_components import MockAsyncEventProcessor +from ldclient.testing.mock_components import ( + MockDataSourceBuilder, + MockOverrideSource +) +from ldclient.testing.test_async_client_overrides import ( + AsyncHangingSynchronizer, + AsyncStaticInitializer +) +from ldclient.testing.test_ldclient_override_vectors import ( + assert_reason, + load_vectors, + override_flags, + vector_id +) + + +async def make_client(vector: Dict[str, Any]) -> AsyncLDClient: + source = MockOverrideSource(flags=override_flags(vector['overrides']), segments=dict(vector['overrides'].get('segments', {}))) + ld_data = vector['launchDarklyData'] + if ld_data['initialized']: + initializer = AsyncStaticInitializer(ld_data.get('flags', {}), ld_data.get('segments', {})) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + start_wait = 5 + else: + # With no sources at all, the client would consider cached data available rather than + # applying its not-initialized handling. A synchronizer that never delivers avoids that. + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) + start_wait = 0 + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=start_wait) + return client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("vector", load_vectors(), ids=vector_id) +async def test_override_spec_vector(vector: Dict[str, Any]): + client = await make_client(vector) + try: + assert await client.is_initialized() is vector['launchDarklyData']['initialized'] + evaluate = vector['evaluate'] + detail = await client.variation_detail(evaluate['flagKey'], Context.from_dict(evaluate['context']), evaluate['defaultValue']) + + expect = vector['expect'] + assert detail.value == expect['value'], "value" + assert detail.variation_index == expect['variationIndex'], "variationIndex" + assert_reason(expect['reason'], detail.reason) + + # summaryOverrideAffected is the marking the client hands to the event processor for this + # evaluation. The event processor keys individual-event suppression and the summary + # counter marker on that scalar, not on the reason. + if 'summaryOverrideAffected' in expect: + records = [e for e in client._event_processor.events if isinstance(e, EventInputEvaluation) and e.key == evaluate['flagKey']] + assert len(records) == 1, "expected exactly one evaluation record for the flag" + assert records[0].override_affected is expect['summaryOverrideAffected'], "summaryOverrideAffected" + finally: + await client.close() diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py new file mode 100644 index 00000000..e8f456f8 --- /dev/null +++ b/ldclient/testing/test_async_client_overrides.py @@ -0,0 +1,704 @@ +""" +Tests for flag overrides through the async client. These mirror the key scenarios of the sync +client tests. +""" +import asyncio +import json +import logging +import threading +import time +from typing import Any, Dict, List, Optional + +import pytest + +from ldclient.async_client import AsyncLDClient +from ldclient.async_config import AsyncConfig, AsyncDataSystemConfig +from ldclient.async_feature_store import AsyncInMemoryFeatureStore +from ldclient.context import Context +from ldclient.evaluation import EvaluationDetail +from ldclient.hook import EvaluationSeriesContext, Hook, Metadata +from ldclient.impl.aio.concurrency import AsyncEvent +from ldclient.impl.events.async_event_processor import ( + DefaultAsyncEventProcessor +) +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.interfaces import AsyncFeatureStore, DataStoreMode +from ldclient.migrations import Stage +from ldclient.testing.builders import FlagBuilder +from ldclient.testing.impl.events.test_async_event_processor import MockAioHttp +from ldclient.testing.mock_async_components import MockAsyncEventProcessor +from ldclient.testing.mock_components import ( + FailingOverrideSourceBuilder, + MockDataSourceBuilder, + MockOverrideSource, + StaticInitializer +) +from ldclient.versioned_data_kind import FEATURES + +user = Context.create('user-key') + +# A context whose key is empty is invalid. +invalid_context = Context.create('') + + +class AsyncHangingSynchronizer: + """An async synchronizer that connects but never yields data.""" + + def __init__(self): + self._stop = AsyncEvent() + + @property + def name(self) -> str: + return "AsyncHangingSynchronizer" + + async def sync(self, ss): + await self._stop.wait() + return + yield + + async def stop(self) -> None: + self._stop.set() + + +class AsyncStaticInitializer(StaticInitializer): + async def fetch(self, ss): # type: ignore[override] + return super().fetch(ss) + + +class UnavailableAsyncStore(AsyncInMemoryFeatureStore): + """A persistent store whose every read fails, as during a store outage.""" + + async def is_initialized(self) -> bool: + raise RuntimeError("store unreachable") + + async def get(self, kind, key): + raise RuntimeError("store unreachable") + + async def all(self, kind): + raise RuntimeError("store unreachable") + + +class UninitializedAsyncStore(AsyncInMemoryFeatureStore): + """A persistent store that holds data but was never initialized by an SDK.""" + + @property + def initialized(self) -> bool: + return False + + async def is_initialized(self) -> bool: + return False + + +class AsyncStoreThatCannotListItems(AsyncInMemoryFeatureStore): + """An initialized persistent store whose enumeration fails.""" + + @property + def initialized(self) -> bool: + return True + + async def is_initialized(self) -> bool: + return True + + async def all(self, kind): + raise RuntimeError("store cannot list items") + + +def records_containing(caplog, level: int, text: str): + return [r for r in caplog.records if r.levelno == level and text in r.getMessage()] + + +async def uninitialized_store_holding(key: str) -> UninitializedAsyncStore: + store = UninitializedAsyncStore() + await store.upsert(FEATURES, FlagBuilder(key).version(1).on(False).off_variation(0).variations('ld-value').build().to_json_dict()) + return store + + +def single_value_flag(key: str, value: Any) -> dict: + return make_flag_with_value(key, value).to_json_dict() + + +def recorded_keys(client: AsyncLDClient) -> List[str]: + """The flag keys of the evaluations the client handed to the event processor.""" + processor: Any = client._event_processor + return [event.key for event in processor.events] + + +async def make_uninitialized_client(source: MockOverrideSource, store: Optional[AsyncFeatureStore] = None) -> AsyncLDClient: + """ + A client whose data system can never obtain LaunchDarkly data, with the given override + source and, when given, a read-only persistent store. + """ + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder, data_store=store, data_store_mode=DataStoreMode.READ_ONLY) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=0) + return client + + +async def make_initialized_client(flags: Dict[str, dict], source: MockOverrideSource, segments: Optional[Dict[str, dict]] = None) -> AsyncLDClient: + initializer = AsyncStaticInitializer(flags, segments or {}) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=5) + assert await client.is_initialized() is True + return client + + +@pytest.mark.asyncio +async def test_override_is_served_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.is_initialized() is False + detail = await client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + finally: + await client.close() + assert source.close_count == 1 + + +@pytest.mark.asyncio +async def test_non_overridden_flag_still_short_circuits_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + detail = await client.variation_detail('other-flag', user, False) + assert detail.value is False + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_removal_restores_short_circuit(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.variation('overridden-flag', user, False) is True + source.set_overrides({}, {}) + detail = await client.variation_detail('overridden-flag', user, False) + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + finally: + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('overrides', [{}, {'other-flag': single_value_flag('other-flag', True)}], ids=['empty-layer', 'layer-without-the-key']) +async def test_not_initialized_client_returns_not_ready_for_an_invalid_context_when_the_layer_lacks_the_key(overrides): + client = await make_uninitialized_client(MockOverrideSource(flags=overrides)) + try: + detail = await client.variation_detail('requested-flag', invalid_context, False) + # The not-ready handling runs before the context check, as it does without overrides, + # and records the evaluation of the unknown flag. + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_not_initialized_client_checks_the_context_before_serving_an_override(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + detail = await client.variation_detail('overridden-flag', invalid_context, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'USER_NOT_SPECIFIED'}) + assert recorded_keys(client) == [] + finally: + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('store_outage', [True, False], ids=['store-outage', 'uninitialized-store']) +async def test_not_initialized_client_returns_not_ready_when_the_store_has_no_launchdarkly_data_and_the_layer_lacks_the_key(store_outage): + store: AsyncFeatureStore = UnavailableAsyncStore() if store_outage else await uninitialized_store_holding('requested-flag') + source = MockOverrideSource(flags={'other-flag': single_value_flag('other-flag', True)}) + client = await make_uninitialized_client(source, store=store) + try: + detail = await client.variation_detail('requested-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_is_served_during_a_persistent_store_outage(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source, store=UnavailableAsyncStore()) + try: + detail = await client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_the_layer_is_empty(): + client = await make_uninitialized_client(MockOverrideSource(), store=await uninitialized_store_holding('ld-flag')) + try: + state = await client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_before_initialization_excludes_launchdarkly_data_from_an_uninitialized_store(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source, store=await uninitialized_store_holding('ld-flag')) + try: + state = await client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_holds_the_overrides_and_is_invalid_when_the_store_fails(caplog): + # A daemon-mode client whose store cannot list its flags logs the failure, as it does without + # a source. Overrides only add to the state, so it holds the override flags, and it stays + # invalid because the store's flags are missing. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = AsyncDataSystemConfig(override_source=source.builder, data_store=AsyncStoreThatCannotListItems(), data_store_mode=DataStoreMode.READ_ONLY) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + state = await client.all_flags_state(user, with_reasons=True) + assert state.valid is False + assert state.to_values_map() == {'overridden-flag': True} + assert state.get_flag_reason('overridden-flag') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records_containing(caplog, logging.ERROR, 'Unable to read flags for all_flag_state') != [] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_is_invalid_and_empty_when_the_store_fails_and_the_layer_holds_no_flags(caplog): + source = MockOverrideSource() + datasystem = AsyncDataSystemConfig(override_source=source.builder, data_store=AsyncStoreThatCannotListItems(), data_store_mode=DataStoreMode.READ_ONLY) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + state = await client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + assert records_containing(caplog, logging.ERROR, 'Unable to read flags for all_flag_state') != [] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_source_start_failure_is_logged_and_the_client_runs_without_overrides(caplog): + # The SDK's own source never raises from start. A custom source could, and that must not + # keep the client from starting: the failure is logged, the source is closed, the snapshot + # it supplied before failing is removed, and a later snapshot is ignored. + flags = {'overridden-flag': single_value_flag('overridden-flag', True)} + source = MockOverrideSource(flags=flags, start_error=RuntimeError("cannot start"), supply_before_start_error=True) + client = await make_uninitialized_client(source) + try: + assert source.close_count == 1 + assert records_containing(caplog, logging.ERROR, 'Unable to start the override source; the client runs without overrides: cannot start') != [] + detail = await client.variation_detail('overridden-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + + source.set_overrides(flags) + assert await client.variation('overridden-flag', user, False) is False + finally: + await client.close() + assert source.close_count == 1 + + +def deleted_flag(key: str) -> dict: + return {'key': key, 'version': 2, 'deleted': True} + + +def flag_and_dependent() -> Dict[str, dict]: + """LaunchDarkly data: the flag "flag", and the flag "dependent", which has it as a prerequisite.""" + return { + 'flag': FlagBuilder('flag').version(1).on(True).variations('ld-value').fallthrough_variation(0).build().to_json_dict(), + 'dependent': FlagBuilder('dependent').version(1).on(True).variations('dependent-on', 'dependent-off').fallthrough_variation(0).off_variation(1).prerequisite('flag', 0).build().to_json_dict(), + } + + +DELETED_FLAG_REJECTED = 'Rejected an override snapshot; the overrides in effect stay: flag "flag" is marked as deleted' + + +@pytest.mark.asyncio +async def test_override_source_that_supplies_a_deleted_flag_at_start_leaves_no_overrides(caplog): + # The sink rejects the whole snapshot, and the error propagates out of the source's start, + # which fails it as an invalid definition does. The client serves LaunchDarkly data. + source = MockOverrideSource(flags={'flag': deleted_flag('flag')}) + client = await make_initialized_client(flag_and_dependent(), source) + try: + assert await client.variation('flag', user, 'default') == 'ld-value' + assert await client.variation_detail('flag', user, 'default') == EvaluationDetail('ld-value', 0, {'kind': 'FALLTHROUGH'}) + assert await client.variation_detail('dependent', user, 'default') == EvaluationDetail('dependent-on', 0, {'kind': 'FALLTHROUGH'}) + assert records_containing(caplog, logging.ERROR, DELETED_FLAG_REJECTED) != [] + assert records_containing(caplog, logging.ERROR, 'Unable to start the override source; the client runs without overrides') != [] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_snapshot_with_a_deleted_flag_is_rejected_and_the_overrides_in_effect_stay(caplog): + source = MockOverrideSource(flags={'flag': single_value_flag('flag', 'override-value')}) + client = await make_initialized_client(flag_and_dependent(), source) + try: + changes: List[Any] = [] + client.flag_tracker.add_listener(changes.append) + + with pytest.raises(ValueError): + await asyncio.to_thread(source.set_overrides, {'flag': deleted_flag('flag'), 'other': single_value_flag('other', True)}) + + assert await client.variation('flag', user, 'default') == 'override-value' + assert await client.variation_detail('flag', user, 'default') == EvaluationDetail('override-value', 0, {'kind': 'FALLTHROUGH', 'overrideAffected': True}) + assert await client.variation_detail('dependent', user, 'default') == EvaluationDetail('dependent-on', 0, {'kind': 'FALLTHROUGH', 'overrideAffected': True}) + assert await client.variation('other', user, 'default') == 'default' + assert records_containing(caplog, logging.ERROR, DELETED_FLAG_REJECTED) != [] + # A notification would be delivered on the event loop. + await asyncio.sleep(0.3) + assert changes == [] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_snapshot_supplied_after_the_client_is_closed_is_ignored(): + source = MockOverrideSource() + client = await make_uninitialized_client(source) + await client.close() + + source.set_overrides({'late-flag': single_value_flag('late-flag', True)}) + + assert client._data_system.override_layer.get(FEATURES, 'late-flag') is None + + +@pytest.mark.asyncio +async def test_override_source_is_not_started_when_offline(caplog): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = AsyncDataSystemConfig(override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, offline=True) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + assert source.start_count == 0 + assert await client.variation('overridden-flag', user, False) is False + assert records_containing(caplog, logging.WARNING, 'Offline mode is enabled; the override source is not started and no overrides are served') != [] + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_invalid_override_source_configuration_fails_construction(): + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=FailingOverrideSourceBuilder()) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + with pytest.raises(ValueError): + AsyncLDClient(config) + + +@pytest.mark.asyncio +async def test_override_takes_precedence_over_launchdarkly_data_and_all_flags_reflects_it(): + ld_flag = FlagBuilder('flag-precedence').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').build().to_json_dict() + source = MockOverrideSource(flags={'flag-precedence': single_value_flag('flag-precedence', 'override-value')}) + client = await make_initialized_client({'flag-precedence': ld_flag, 'flag-normal': normal}, source) + try: + detail = await client.variation_detail('flag-precedence', user, 'default') + assert detail.value == 'override-value' + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + detail = await client.variation_detail('flag-normal', user, 'default') + assert detail.reason == {'kind': 'OFF'} + + state = await client.all_flags_state(user, with_reasons=True) + assert state.to_values_map() == {'flag-precedence': 'override-value', 'flag-normal': 'normal-value'} + assert state.get_flag_reason('flag-precedence') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_contains_only_overrides_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + state = await client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + source.set_overrides({}, {}) + assert (await client.all_flags_state(user)).valid is False + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_flag_tracker_is_notified_of_override_changes_on_the_event_loop(): + source = MockOverrideSource() + client = await make_uninitialized_client(source) + try: + changes: asyncio.Queue = asyncio.Queue() + loop_thread = threading.current_thread() + listener_threads = [] + + def listener(change): + listener_threads.append(threading.current_thread()) + changes.put_nowait(change) + + client.flag_tracker.add_listener(listener) + + # The source pushes from a worker thread, as the file source does. The notification + # is delivered on the event loop's thread. + await asyncio.to_thread(source.set_overrides, {'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + change = await asyncio.wait_for(changes.get(), 5) + assert change.key == 'overridden-flag' + assert listener_threads[0] is loop_thread + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_flag_value_change_listener_sees_override_value_changes(): + ld_flag = FlagBuilder('flag').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + source = MockOverrideSource() + client = await make_initialized_client({'flag': ld_flag}, source) + try: + changes: asyncio.Queue = asyncio.Queue() + await client.flag_tracker.add_flag_value_change_listener('flag', user, changes.put_nowait) + await asyncio.to_thread(source.set_overrides, {'flag': single_value_flag('flag', 'override-value')}, {}) + change = await asyncio.wait_for(changes.get(), 5) + assert change.old_value == 'ld-value' + assert change.new_value == 'override-value' + finally: + await client.close() + + +async def raise_evaluation_failure(*args): + raise RuntimeError("evaluation failure") + + +@pytest.mark.asyncio +async def test_failed_evaluation_reason_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + # The evaluation read the flag definition before it failed, so the failure of the override + # flag is marked and the failure of the ordinary flag is not, in a single evaluation and in + # the all-flags state. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_initialized_client({'plain-flag': single_value_flag('plain-flag', True)}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + assert await client.variation_detail('overridden-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + assert await client.variation_detail('plain-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + state = await client.all_flags_state(user, with_reasons=True) + assert state.get_flag_reason('overridden-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert state.get_flag_reason('plain-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_evaluation_events_carry_override_affected_marking(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.variation('overridden-flag', user, False) is True + records = [e for e in client._event_processor.events if isinstance(e, EventInputEvaluation)] + assert [e.key for e in records] == ['overridden-flag'] + assert records[0].override_affected is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): + overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).debug_events_until_date(int(time.time() * 1000) + 100000).build().to_json_dict() + plain = FlagBuilder('plain-tracked').version(1).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + client = await make_initialized_client({'plain-tracked': plain}, source) + try: + state = await client.all_flags_state(user, with_reasons=True) + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['plain-tracked']['trackEvents'] is True + assert 'trackEvents' not in flags_state['overridden-flag'] + assert 'debugEventsUntilDate' not in flags_state['overridden-flag'] + assert flags_state['overridden-flag']['reason']['overrideAffected'] is True + assert state.get_flag_value('overridden-flag') is True + finally: + await client.close() + + +def test_override_update_after_the_client_and_its_loop_are_gone_is_dropped(): + source = MockOverrideSource() + + async def run_client() -> None: + client = await make_uninitialized_client(source) + client.flag_tracker.add_listener(lambda change: None) + await client.close() + + asyncio.run(run_client()) + # A reload that finishes after the client closed finds the event loop closed. The + # notification has nobody left to reach and is dropped. + source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +def tracked_bool_flag(key: str) -> FlagBuilder: + return FlagBuilder(key).version(100).variations(False, True).off_variation(0).fallthrough_variation(1).track_events(True) + + +def evaluation_events_by_key(client: AsyncLDClient) -> Dict[str, EventInputEvaluation]: + """The evaluation records the client handed to the event processor, keyed by flag key.""" + processor: Any = client._event_processor + return {event.key: event for event in processor.events if isinstance(event, EventInputEvaluation)} + + +@pytest.mark.asyncio +async def test_overridden_prerequisite_marks_the_dependent_evaluation_records(): + # top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + # --> plain-flag (LaunchDarkly) + # The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + ld_data = { + 'top-flag': tracked_bool_flag('top-flag').on(True).prerequisite('mid-flag', 1).prerequisite('plain-flag', 1).build().to_json_dict(), + 'mid-flag': tracked_bool_flag('mid-flag').on(True).prerequisite('leaf-flag', 1).build().to_json_dict(), + 'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict(), + 'leaf-flag': tracked_bool_flag('leaf-flag').on(False).build().to_json_dict(), + } + source = MockOverrideSource(flags={'leaf-flag': tracked_bool_flag('leaf-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client(ld_data, source) + try: + detail = await client.variation_detail('top-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + records = evaluation_events_by_key(client) + assert sorted(records.keys()) == ['leaf-flag', 'mid-flag', 'plain-flag', 'top-flag'] + assert records['top-flag'].override_affected is True + assert records['mid-flag'].override_affected is True + assert records['leaf-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['leaf-flag'].reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records['plain-flag'].reason == {'kind': 'FALLTHROUGH'} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_wrong_type_result_of_overridden_flag_stays_marked(): + details = [] + + class CapturingHook(Hook): + @property + def metadata(self) -> Metadata: + return Metadata(name='capturing-hook') + + def before_evaluation(self, series_context: EvaluationSeriesContext, data: dict) -> dict: + return data + + def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, detail: EvaluationDetail) -> dict: + details.append(detail) + return data + + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', 'not-a-stage')}) + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor(), hooks=[CapturingHook()]) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + stage, _ = await client.migration_variation('overridden-flag', user, Stage.OFF) + assert stage == Stage.OFF + assert len(details) == 1 + assert details[0].value == 'off' + assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_failed_evaluation_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + # The failure of the override flag is marked. The failure of the ordinary flag is not. + detail = await client.variation_detail('overridden-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + detail = await client.variation_detail('plain-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + # The marked record produces no individual event. The ordinary record keeps its tracking. + records = evaluation_events_by_key(client) + assert records['overridden-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['plain-flag'].track_events is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_turns_off_event_tracking_for_an_override_flag_whose_evaluation_fails(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + state = await client.all_flags_state(user, with_reasons=True) + + # The failed override flag stays in the state with a marked reason and no tracking + # fields. The failed ordinary flag keeps its tracking fields. + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['overridden-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert 'trackEvents' not in flags_state['overridden-flag'] + assert flags_state['plain-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + assert flags_state['plain-flag']['trackEvents'] is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_affected_evaluations_appear_only_in_summary_output(): + # End to end through the real event processor: the overridden flag requests individual + # feature events and debug events, and an ordinary flag requests feature events. + debug_until = int(time.time() * 1000) + 100000 + overridden = FlagBuilder('flag-tracked-override').version(300).on(False).off_variation(0).variations('override-value').track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'flag-tracked-override': overridden}) + mock_http = MockAioHttp() + + initializer = AsyncStaticInitializer({'flag-normal': normal}, {}) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, diagnostic_opt_out=True, event_processor_class=lambda config: DefaultAsyncEventProcessor(config, mock_http)) + client = AsyncLDClient(config) + await client.start(start_wait=5) + try: + assert await client.is_initialized() is True + for _ in range(2): + assert await client.variation('flag-tracked-override', user, 'default1') == 'override-value' + assert await client.variation('flag-normal', user, 'default2') == 'normal-value' + assert await client._event_processor.flush_and_wait(5) is True + finally: + await client.close() + + assert mock_http.request_data is not None + output = json.loads(mock_http.request_data) + kinds = sorted(e['kind'] for e in output) + assert kinds == ['feature', 'index', 'summary'] + feature = [e for e in output if e['kind'] == 'feature'][0] + assert feature['key'] == 'flag-normal' + summary = [e for e in output if e['kind'] == 'summary'][0] + assert summary['features']['flag-tracked-override']['default'] == 'default1' + assert summary['features']['flag-tracked-override']['counters'] == [ + {'count': 2, 'value': 'override-value', 'variation': 0, 'version': 300, 'overrideAffected': True} + ] + assert summary['features']['flag-normal']['counters'] == [{'count': 1, 'value': 'normal-value', 'variation': 0, 'version': 100}] diff --git a/ldclient/testing/test_ldclient_override_vectors.py b/ldclient/testing/test_ldclient_override_vectors.py new file mode 100644 index 00000000..a55ea87d --- /dev/null +++ b/ldclient/testing/test_ldclient_override_vectors.py @@ -0,0 +1,95 @@ +""" +Runs the OVERRIDE specification test vectors. Each vector sets up LaunchDarkly data, an +override layer, and an initialization state. The test evaluates one flag through the full +client stack and checks the value, the variation index, the reason, and the marking handed to the +event processor. +""" +import json +import os +from typing import Any, Dict + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.datasystem import custom +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.testing.mock_components import ( + HangingSynchronizer, + MockOverrideSource, + StaticInitializer +) +from ldclient.testing.stub_util import MockEventProcessor + +VECTORS_PATH = os.path.join(os.path.dirname(__file__), 'testdata', 'override-vectors', 'vectors.json') + +# The vectors' semantics are versioned. A schema change means this runner needs review. +SUPPORTED_SCHEMA_VERSION = '0.4.0' + + +def load_vectors(): + with open(VECTORS_PATH, 'r') as f: + document = json.load(f) + assert document['schemaVersion'] == SUPPORTED_SCHEMA_VERSION, "the vectors changed schema; review this runner against the new schema before updating" + assert len(document['vectors']) > 0 + return document['vectors'] + + +def vector_id(vector: Dict[str, Any]) -> str: + return "%s: %s" % (vector['group'], vector['description']) + + +def override_flags(overrides: Dict[str, Any]) -> Dict[str, Any]: + flags = dict(overrides.get('flags', {})) + for key, value in overrides.get('flagValues', {}).items(): + flags[key] = make_flag_with_value(key, value) + return flags + + +def make_client(vector: Dict[str, Any]) -> LDClient: + source = MockOverrideSource(flags=override_flags(vector['overrides']), segments=dict(vector['overrides'].get('segments', {}))) + ld_data = vector['launchDarklyData'] + datasystem = custom().overrides(source.builder) + if ld_data['initialized']: + datasystem.initializers([StaticInitializer(ld_data.get('flags', {}), ld_data.get('segments', {})).builder]) + start_wait = 5 + else: + # With no sources at all, the client would consider cached data available rather than + # applying its not-initialized handling. A synchronizer that never delivers avoids that. + datasystem.synchronizers(HangingSynchronizer().builder) + start_wait = 0 + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem.build(), event_processor_class=MockEventProcessor) + return LDClient(config, start_wait=start_wait) + + +def assert_reason(expected: Dict[str, Any], actual: Dict[str, Any]) -> None: + """ + The reason is compared only on the fields present in the expected reason. The + override-affected indicator is a tri-state collapse: an expected reason that omits it + requires the actual reason to omit it or report it as false. + """ + for name, value in expected.items(): + assert actual.get(name) == value, "reason property %s: expected %r in %r" % (name, value, actual) + if 'overrideAffected' not in expected: + assert actual.get('overrideAffected', False) is False, "reason must not be override-affected: %r" % actual + + +@pytest.mark.parametrize("vector", load_vectors(), ids=vector_id) +def test_override_spec_vector(vector: Dict[str, Any]): + with make_client(vector) as client: + assert client.is_initialized() is vector['launchDarklyData']['initialized'] + evaluate = vector['evaluate'] + detail = client.variation_detail(evaluate['flagKey'], Context.from_dict(evaluate['context']), evaluate['defaultValue']) + + expect = vector['expect'] + assert detail.value == expect['value'], "value" + assert detail.variation_index == expect['variationIndex'], "variationIndex" + assert_reason(expect['reason'], detail.reason) + + # summaryOverrideAffected is the marking the client hands to the event processor for this + # evaluation. The event processor keys individual-event suppression and the summary + # counter marker on that scalar, not on the reason. + if 'summaryOverrideAffected' in expect: + records = [e for e in client._event_processor._events if isinstance(e, EventInputEvaluation) and e.key == evaluate['flagKey']] + assert len(records) == 1, "expected exactly one evaluation record for the flag" + assert records[0].override_affected is expect['summaryOverrideAffected'], "summaryOverrideAffected" diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py new file mode 100644 index 00000000..74cca252 --- /dev/null +++ b/ldclient/testing/test_ldclient_overrides.py @@ -0,0 +1,780 @@ +""" +Tests for flag overrides through the client: the override source lifecycle, the overlay at the +store read boundary, the not-initialized gate, the all-flags state, and flag change notifications. +""" +import json +import logging +import time +from queue import Empty, Queue +from typing import Any, Dict, List, Optional + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.datasystem import custom, daemon +from ldclient.evaluation import EvaluationDetail +from ldclient.feature_store import InMemoryFeatureStore +from ldclient.hook import EvaluationSeriesContext, Hook, Metadata +from ldclient.impl.big_segments import BigSegmentStoreManager +from ldclient.impl.datasystem.fdv1 import FDv1 +from ldclient.impl.datasystem.fdv2 import FDv2 +from ldclient.impl.events.event_processor import DefaultEventProcessor +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.impl.integrations.files.filedata import make_flag_with_value +from ldclient.integrations.test_datav2 import TestDataV2 +from ldclient.interfaces import ( + DataSourceState, + DataStoreMode, + FeatureStore, + FlagChange +) +from ldclient.migrations import Stage +from ldclient.testing.builders import ( + FlagBuilder, + FlagRuleBuilder, + SegmentBuilder, + make_clause_matching_segment_key +) +from ldclient.testing.mock_components import ( + FailingOverrideSourceBuilder, + HangingSynchronizer, + MockOverrideSource, + StaticInitializer +) +from ldclient.testing.stub_util import MockEventProcessor, MockHttp +from ldclient.versioned_data_kind import FEATURES + +user = Context.create('user-key') + +# A context whose key is empty is invalid. +invalid_context = Context.create('') + + +def single_value_flag(key: str, value: Any) -> dict: + return make_flag_with_value(key, value).to_json_dict() + + +def make_uninitialized_client(source: MockOverrideSource, store: Optional[FeatureStore] = None) -> LDClient: + """ + A client whose data system can never obtain LaunchDarkly data, with the given override + source and, when given, a read-only persistent store. + """ + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder) + if store is not None: + datasystem.data_store(store, DataStoreMode.READ_ONLY) + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem.build(), event_processor_class=MockEventProcessor) + return LDClient(config, start_wait=0) + + +def make_initialized_client(flags: Dict[str, dict], source: MockOverrideSource, segments: Optional[Dict[str, dict]] = None) -> LDClient: + """A client that has initialized with the given LaunchDarkly data, with the given override source.""" + initializer = StaticInitializer(flags, segments or {}) + datasystem = custom().initializers([initializer.builder]).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + client = LDClient(config, start_wait=5) + assert client.is_initialized() is True + return client + + +def warnings_containing(caplog, text: str): + return [r for r in caplog.records if r.levelno == logging.WARNING and text in r.getMessage()] + + +def recorded_keys(client: LDClient) -> List[str]: + """The flag keys of the evaluations the client handed to the event processor.""" + processor: Any = client._event_processor + return [event.key for event in processor._events] + + +class UnavailableStore(InMemoryFeatureStore): + """A persistent store whose every read fails, as during a store outage.""" + + @property + def initialized(self) -> bool: + raise RuntimeError("store unreachable") + + def get(self, kind, key, callback=lambda x: x): + raise RuntimeError("store unreachable") + + def all(self, kind, callback=lambda x: x): + raise RuntimeError("store unreachable") + + +class UninitializedStore(InMemoryFeatureStore): + """A persistent store that holds data but was never initialized by an SDK.""" + + @property + def initialized(self) -> bool: + return False + + +def uninitialized_store_holding(key: str) -> UninitializedStore: + store = UninitializedStore() + store.upsert(FEATURES, FlagBuilder(key).version(1).on(False).off_variation(0).variations('ld-value').build().to_json_dict()) + return store + + +class StoreThatCannotListItems(InMemoryFeatureStore): + """An initialized persistent store whose enumeration fails.""" + + @property + def initialized(self) -> bool: + return True + + def all(self, kind, callback=lambda x: x): + raise RuntimeError("store cannot list items") + + +def errors_containing(caplog, text: str): + return [r for r in caplog.records if r.levelno == logging.ERROR and text in r.getMessage()] + + +def test_override_is_served_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.is_initialized() is False + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.variation_index == 0 + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + +def test_non_overridden_flag_still_short_circuits_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + detail = client.variation_detail('other-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + + +def test_override_removal_restores_short_circuit(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.variation('overridden-flag', user, False) is True + source.set_overrides({}, {}) + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is False + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + + +@pytest.mark.parametrize('overrides', [{}, {'other-flag': single_value_flag('other-flag', True)}], ids=['empty-layer', 'layer-without-the-key']) +def test_not_initialized_client_returns_not_ready_for_an_invalid_context_when_the_layer_lacks_the_key(overrides): + with make_uninitialized_client(MockOverrideSource(flags=overrides)) as client: + detail = client.variation_detail('requested-flag', invalid_context, False) + # The not-ready handling runs before the context check, as it does without overrides, + # and records the evaluation of the unknown flag. + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + + +def test_not_initialized_client_checks_the_context_before_serving_an_override(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + detail = client.variation_detail('overridden-flag', invalid_context, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'USER_NOT_SPECIFIED'}) + assert recorded_keys(client) == [] + + +@pytest.mark.parametrize('make_store', [UnavailableStore, lambda: uninitialized_store_holding('requested-flag')], ids=['store-outage', 'uninitialized-store']) +def test_not_initialized_client_returns_not_ready_when_the_store_has_no_launchdarkly_data_and_the_layer_lacks_the_key(make_store): + source = MockOverrideSource(flags={'other-flag': single_value_flag('other-flag', True)}) + with make_uninitialized_client(source, store=make_store()) as client: + detail = client.variation_detail('requested-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert recorded_keys(client) == ['requested-flag'] + + +def test_override_is_served_during_a_persistent_store_outage(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source, store=UnavailableStore()) as client: + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + +def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_the_layer_is_empty(): + with make_uninitialized_client(MockOverrideSource(), store=uninitialized_store_holding('ld-flag')) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + + +def test_all_flags_state_is_invalid_when_the_store_is_uninitialized_and_the_layer_holds_no_flags(): + # A layer that holds only segments has no flags to report, so the LaunchDarkly data in the + # store stays out, as it does without an override source. + source = MockOverrideSource(segments={'segment1': SegmentBuilder('segment1').version(1).build().to_json_dict()}) + with make_uninitialized_client(source, store=uninitialized_store_holding('ld-flag')) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + + +def test_all_flags_state_before_initialization_excludes_launchdarkly_data_from_an_uninitialized_store(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source, store=uninitialized_store_holding('ld-flag')) as client: + state = client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + + +def test_all_flags_state_holds_the_overrides_and_is_invalid_when_the_store_fails(caplog): + # A daemon-mode client whose store cannot list its flags logs the failure, as it does without + # a source. Overrides only add to the state, so it holds the override flags, and it stays + # invalid because the store's flags are missing. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = daemon(StoreThatCannotListItems()).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=0) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is False + assert state.to_values_map() == {'overridden-flag': True} + assert state.get_flag_reason('overridden-flag') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert errors_containing(caplog, 'Unable to read flags for all_flag_state') != [] + + +def test_all_flags_state_is_invalid_and_empty_when_the_store_fails_and_the_layer_holds_no_flags(caplog): + source = MockOverrideSource() + datasystem = daemon(StoreThatCannotListItems()).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=0) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + assert errors_containing(caplog, 'Unable to read flags for all_flag_state') != [] + + +def test_override_source_is_started_before_the_constructor_returns(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = make_uninitialized_client(source) + try: + assert source.start_count == 1 + assert client.variation('overridden-flag', user, False) is True + finally: + client.close() + + +def test_override_source_is_closed_when_the_client_is_closed(): + source = MockOverrideSource() + client = make_uninitialized_client(source) + assert source.close_count == 0 + client.close() + assert source.close_count == 1 + + +def test_invalid_override_source_configuration_fails_construction(): + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(FailingOverrideSourceBuilder()).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with pytest.raises(ValueError): + LDClient(config, start_wait=0) + + +def test_override_source_start_failure_is_logged_and_the_client_runs_without_overrides(caplog): + # The SDK's own source never raises from start. A custom source could, and that must not + # keep the client from starting: the failure is logged, the source is closed, and the + # client behaves as it does without overrides. + source = MockOverrideSource(start_error=RuntimeError("cannot start")) + with make_uninitialized_client(source) as client: + assert source.close_count == 1 + assert errors_containing(caplog, 'Unable to start the override source; the client runs without overrides: cannot start') != [] + detail = client.variation_detail('any-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + # Closing the client does not close the failed source again. + assert source.close_count == 1 + + +def test_override_source_that_supplies_a_snapshot_and_then_fails_to_start_leaves_no_overrides(): + # The snapshot the source supplied before it failed is removed, so the client runs without + # overrides, as the log says, and a later snapshot from the closed source is ignored. + flags = {'overridden-flag': single_value_flag('overridden-flag', True)} + source = MockOverrideSource(flags=flags, start_error=RuntimeError("cannot start"), supply_before_start_error=True) + with make_uninitialized_client(source) as client: + detail = client.variation_detail('overridden-flag', user, False) + assert detail == EvaluationDetail(False, None, {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'}) + assert client.all_flags_state(user).valid is False + + source.set_overrides(flags) + assert client.variation('overridden-flag', user, False) is False + + +def deleted_flag(key: str) -> dict: + return {'key': key, 'version': 2, 'deleted': True} + + +def flag_and_dependent() -> Dict[str, dict]: + """LaunchDarkly data: the flag "flag", and the flag "dependent", which has it as a prerequisite.""" + return { + 'flag': FlagBuilder('flag').version(1).on(True).variations('ld-value').fallthrough_variation(0).build().to_json_dict(), + 'dependent': FlagBuilder('dependent').version(1).on(True).variations('dependent-on', 'dependent-off').fallthrough_variation(0).off_variation(1).prerequisite('flag', 0).build().to_json_dict(), + } + + +DELETED_FLAG_REJECTED = 'Rejected an override snapshot; the overrides in effect stay: flag "flag" is marked as deleted' + + +def test_override_source_that_supplies_a_deleted_flag_at_start_leaves_no_overrides(caplog): + # The sink rejects the whole snapshot, and the error propagates out of the source's start, + # which fails it as an invalid definition does. The client serves LaunchDarkly data. + source = MockOverrideSource(flags={'flag': deleted_flag('flag')}) + with make_initialized_client(flag_and_dependent(), source) as client: + assert client.variation('flag', user, 'default') == 'ld-value' + assert client.variation_detail('flag', user, 'default') == EvaluationDetail('ld-value', 0, {'kind': 'FALLTHROUGH'}) + assert client.variation_detail('dependent', user, 'default') == EvaluationDetail('dependent-on', 0, {'kind': 'FALLTHROUGH'}) + assert errors_containing(caplog, DELETED_FLAG_REJECTED) != [] + assert errors_containing(caplog, 'Unable to start the override source; the client runs without overrides') != [] + + +def test_snapshot_with_a_deleted_flag_is_rejected_and_the_overrides_in_effect_stay(caplog): + source = MockOverrideSource(flags={'flag': single_value_flag('flag', 'override-value')}) + with make_initialized_client(flag_and_dependent(), source) as client: + changes: Queue = Queue() + client.flag_tracker.add_listener(lambda change: changes.put(change)) + + with pytest.raises(ValueError): + source.set_overrides({'flag': deleted_flag('flag'), 'other': single_value_flag('other', True)}) + + assert client.variation('flag', user, 'default') == 'override-value' + assert client.variation_detail('flag', user, 'default') == EvaluationDetail('override-value', 0, {'kind': 'FALLTHROUGH', 'overrideAffected': True}) + assert client.variation_detail('dependent', user, 'default') == EvaluationDetail('dependent-on', 0, {'kind': 'FALLTHROUGH', 'overrideAffected': True}) + assert client.variation('other', user, 'default') == 'default' + assert errors_containing(caplog, DELETED_FLAG_REJECTED) != [] + with pytest.raises(Empty): + changes.get(timeout=0.3) + + +def test_snapshot_supplied_after_the_client_is_closed_is_ignored(): + source = MockOverrideSource() + client = make_uninitialized_client(source) + client.close() + + source.set_overrides({'late-flag': single_value_flag('late-flag', True)}) + + assert client._data_system.override_layer.get(FEATURES, 'late-flag') is None + + +def test_override_source_is_not_started_when_offline(caplog): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + datasystem = custom().overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, offline=True) + with LDClient(config, start_wait=0) as client: + assert source.start_count == 0 + assert client.variation('overridden-flag', user, False) is False + assert warnings_containing(caplog, 'Offline mode is enabled; the override source is not started and no overrides are served') != [] + + +def test_data_system_without_override_source_has_no_override_layer(): + datasystem = custom().synchronizers(HangingSynchronizer().builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=0) as client: + assert client._data_system.override_layer is None + detail = client.variation_detail('any-flag', user, False) + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + + +def test_legacy_data_system_has_no_override_layer(): + config = Config(sdk_key='SDK_KEY', offline=True) + with LDClient(config) as client: + assert isinstance(client._data_system, FDv1) + assert client._data_system.override_layer is None + + +def test_override_takes_precedence_over_launchdarkly_data(): + ld_flag = FlagBuilder('flag-precedence').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').build().to_json_dict() + source = MockOverrideSource(flags={'flag-precedence': single_value_flag('flag-precedence', 'override-value')}) + with make_initialized_client({'flag-precedence': ld_flag, 'flag-normal': normal}, source) as client: + detail = client.variation_detail('flag-precedence', user, 'default') + assert detail.value == 'override-value' + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + detail = client.variation_detail('flag-normal', user, 'default') + assert detail.value == 'normal-value' + assert detail.reason == {'kind': 'OFF'} + + +def test_full_flag_override_evaluates_targeting_rules_and_referenced_override_segment(): + segment_flag = FlagBuilder('flag-segment').version(1).on(True).off_variation(0).fallthrough_variation(0).variations('not-included', 'included').rules( + FlagRuleBuilder().id('segment-rule').variation(1).clauses(make_clause_matching_segment_key('overridden-segment')).build() + ).build().to_json_dict() + ld_segment = SegmentBuilder('overridden-segment').version(100).build().to_json_dict() + override_segment = SegmentBuilder('overridden-segment').version(101).included(user.key).build().to_json_dict() + source = MockOverrideSource(flags={'flag-segment': segment_flag}, segments={'overridden-segment': override_segment}) + with make_initialized_client({}, source, segments={'overridden-segment': ld_segment}) as client: + detail = client.variation_detail('flag-segment', user, 'default') + assert detail.value == 'included' + assert detail.reason == {'kind': 'RULE_MATCH', 'ruleIndex': 0, 'ruleId': 'segment-rule', 'overrideAffected': True} + + +def test_overridden_prerequisite_marks_the_dependent_flag(): + dependent = FlagBuilder('dependent').version(1).on(True).off_variation(0).fallthrough_variation(1).variations('prereq-failed', 'affected-value').prerequisite('prereq', 1).build().to_json_dict() + ld_prereq = FlagBuilder('prereq').version(100).on(False).off_variation(0).variations('a', 'b').build().to_json_dict() + override_prereq = FlagBuilder('prereq').version(200).on(True).off_variation(0).fallthrough_variation(1).variations('a', 'b').build().to_json_dict() + source = MockOverrideSource(flags={'prereq': override_prereq}) + with make_initialized_client({'dependent': dependent, 'prereq': ld_prereq}, source) as client: + detail = client.variation_detail('dependent', user, 'default') + assert detail.value == 'affected-value' + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + +def test_all_flags_state_contains_only_overrides_when_client_is_not_initialized(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + state = client.all_flags_state(user) + assert state.valid is True + assert state.to_values_map() == {'overridden-flag': True} + + +def test_all_flags_state_overrides_only_warning_is_logged_once(caplog): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + with caplog.at_level(logging.WARNING): + assert client.all_flags_state(user).valid is True + assert client.all_flags_state(user).valid is True + assert len(warnings_containing(caplog, 'Returning only flags from the override layer')) == 1 + + +def test_all_flags_state_is_invalid_when_not_initialized_and_override_layer_is_empty(): + source = MockOverrideSource() + with make_uninitialized_client(source) as client: + state = client.all_flags_state(user) + assert state.valid is False + assert state.to_values_map() == {} + + +def test_all_flags_state_reflects_overrides_when_initialized(): + ld_flag = FlagBuilder('flag-precedence').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').build().to_json_dict() + source = MockOverrideSource(flags={ + 'flag-precedence': single_value_flag('flag-precedence', 'override-value'), + 'override-only': single_value_flag('override-only', 'only-value'), + }) + with make_initialized_client({'flag-precedence': ld_flag, 'flag-normal': normal}, source) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is True + assert state.to_values_map() == {'flag-precedence': 'override-value', 'flag-normal': 'normal-value', 'override-only': 'only-value'} + assert state.get_flag_reason('flag-precedence') == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert state.get_flag_reason('flag-normal') == {'kind': 'OFF'} + + +def test_flag_tracker_is_notified_of_override_changes(): + source = MockOverrideSource() + with make_uninitialized_client(source) as client: + changes: Queue = Queue() + client.flag_tracker.add_listener(lambda change: changes.put(change)) + + source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + change = changes.get(timeout=5) + assert isinstance(change, FlagChange) + assert change.key == 'overridden-flag' + + # An identical snapshot notifies nothing. + source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + with pytest.raises(Empty): + changes.get(timeout=0.3) + + source.set_overrides({}, {}) + assert changes.get(timeout=5).key == 'overridden-flag' + + +def test_launchdarkly_change_to_a_prerequisite_notifies_the_override_flag_that_depends_on_it(): + td = TestDataV2.data_source() + td.update(td.flag('ld-flag').on(True)) + override_flag = FlagBuilder('override-flag').version(1).on(True).variations(True, False).fallthrough_variation(0).prerequisite('ld-flag', 0).build().to_json_dict() + source = MockOverrideSource({'override-flag': override_flag}, {}) + datasystem = custom().synchronizers(td.builder).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with LDClient(config, start_wait=5) as client: + assert client.is_initialized() is True + changes: Queue = Queue() + client.flag_tracker.add_listener(lambda change: changes.put(change)) + + td.update(td.flag('ld-flag').on(False)) + keys = {changes.get(timeout=5).key, changes.get(timeout=5).key} + assert keys == {'ld-flag', 'override-flag'} + with pytest.raises(Empty): + changes.get(timeout=0.3) + + +def test_flag_value_change_listener_sees_override_value_changes(): + ld_flag = FlagBuilder('flag').version(100).on(False).off_variation(0).variations('ld-value').build().to_json_dict() + source = MockOverrideSource() + with make_initialized_client({'flag': ld_flag}, source) as client: + changes: Queue = Queue() + client.flag_tracker.add_flag_value_change_listener('flag', user, lambda change: changes.put(change)) + + source.set_overrides({'flag': single_value_flag('flag', 'override-value')}, {}) + change = changes.get(timeout=5) + assert change.old_value == 'ld-value' + assert change.new_value == 'override-value' + + source.set_overrides({}, {}) + change = changes.get(timeout=5) + assert change.old_value == 'override-value' + assert change.new_value == 'ld-value' + + +def test_data_source_status_is_unaffected_by_overrides(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.is_initialized() is False + assert client.data_source_status_provider.status.state == DataSourceState.INITIALIZING + + +def raise_evaluation_failure(*args): + raise RuntimeError("evaluation failure") + + +def test_failed_evaluation_reason_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + # The evaluation read the flag definition before it failed, so the failure of the override + # flag is marked and the failure of the ordinary flag is not, in a single evaluation and in + # the all-flags state. + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_initialized_client({'plain-flag': single_value_flag('plain-flag', True)}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + assert client.variation_detail('overridden-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + assert client.variation_detail('plain-flag', user, 'default') == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + state = client.all_flags_state(user, with_reasons=True) + assert state.get_flag_reason('overridden-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert state.get_flag_reason('plain-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + + +def test_snapshot_with_a_definition_under_another_key_is_rejected_and_the_overrides_stay(): + # Evaluation reads an override by its key in the snapshot, while the all-flags report and + # events use the definition's key. A definition keyed under another key would serve one + # flag's override under the other's key, so the snapshot is rejected and nothing changes. + ld_flags = {'a': single_value_flag('a', 'ld-a'), 'b': single_value_flag('b', 'ld-b')} + source = MockOverrideSource(flags={'a': single_value_flag('a', 'override-a')}) + with make_initialized_client(ld_flags, source) as client: + with pytest.raises(ValueError): + source.set_overrides({'a': single_value_flag('b', 'override-b')}) + + assert client.variation('a', user, 'default') == 'override-a' + assert client.variation('b', user, 'default') == 'ld-b' + assert client.all_flags_state(user).to_values_map() == {'a': 'override-a', 'b': 'ld-b'} + + +def test_failed_start_stops_every_component_and_raises_the_start_failure(monkeypatch, caplog): + # A component that fails to stop must not hide the start-up failure or keep the components + # after it from stopping. + stopped: List[str] = [] + + def failing_start(self, set_on_ready): + raise RuntimeError('start failed') + + def failing_stop(self): + stopped.append('data system') + raise RuntimeError('stop failed') + + monkeypatch.setattr(FDv2, 'start', failing_start) + monkeypatch.setattr(FDv2, 'stop', failing_stop) + monkeypatch.setattr(BigSegmentStoreManager, 'stop', lambda self: stopped.append('big segment store manager')) + monkeypatch.setattr(MockEventProcessor, 'stop', lambda self: stopped.append('event processor')) + + datasystem = custom().synchronizers(HangingSynchronizer().builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + with pytest.raises(RuntimeError, match='start failed'): + LDClient(config, start_wait=0) + + assert stopped == ['data system', 'big segment store manager', 'event processor'] + assert warnings_containing(caplog, 'Error stopping the data system after a failed start: stop failed') != [] + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +def evaluation_events_by_key(client: LDClient) -> Dict[str, EventInputEvaluation]: + """The evaluation records the client handed to the event processor, keyed by flag key.""" + records = {} + processor: Any = client._event_processor + for event in processor._events: + if isinstance(event, EventInputEvaluation): + records[event.key] = event + return records + + +def test_override_evaluation_events_carry_override_affected_marking(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.variation('overridden-flag', user, False) is True + records = evaluation_events_by_key(client) + assert list(records.keys()) == ['overridden-flag'] + assert records['overridden-flag'].override_affected is True + + +def test_ordinary_evaluation_events_are_not_marked(): + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'other': single_value_flag('other', True)}) + with make_initialized_client({'flag-normal': normal}, source) as client: + client.variation('flag-normal', user, 'default') + records = evaluation_events_by_key(client) + assert records['flag-normal'].override_affected is False + assert records['flag-normal'].track_events is True + + +def tracked_bool_flag(key: str) -> FlagBuilder: + return FlagBuilder(key).version(100).variations(False, True).off_variation(0).fallthrough_variation(1).track_events(True) + + +def test_overridden_prerequisite_marks_the_dependent_evaluation_records(): + # top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + # --> plain-flag (LaunchDarkly) + # The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + ld_data = { + 'top-flag': tracked_bool_flag('top-flag').on(True).prerequisite('mid-flag', 1).prerequisite('plain-flag', 1).build().to_json_dict(), + 'mid-flag': tracked_bool_flag('mid-flag').on(True).prerequisite('leaf-flag', 1).build().to_json_dict(), + 'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict(), + 'leaf-flag': tracked_bool_flag('leaf-flag').on(False).build().to_json_dict(), + } + source = MockOverrideSource(flags={'leaf-flag': tracked_bool_flag('leaf-flag').on(True).build().to_json_dict()}) + with make_initialized_client(ld_data, source) as client: + detail = client.variation_detail('top-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + records = evaluation_events_by_key(client) + assert sorted(records.keys()) == ['leaf-flag', 'mid-flag', 'plain-flag', 'top-flag'] + assert records['top-flag'].override_affected is True + assert records['mid-flag'].override_affected is True + assert records['leaf-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['mid-flag'].prereq_of is not None and records['mid-flag'].prereq_of.key == 'top-flag' + assert records['leaf-flag'].prereq_of is not None and records['leaf-flag'].prereq_of.key == 'mid-flag' + assert records['leaf-flag'].reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records['plain-flag'].reason == {'kind': 'FALLTHROUGH'} + + +def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): + debug_until = int(time.time() * 1000) + 100000 + ld_data = { + 'plain-tracked': FlagBuilder('plain-tracked').version(1).on(False).off_variation(0).variations(True).track_events(True).debug_events_until_date(debug_until).build().to_json_dict(), + 'dependent-tracked': FlagBuilder('dependent-tracked').version(1).on(True).variations(False, True).fallthrough_variation(1).prerequisite('overridden-flag', 0).track_events(True).debug_events_until_date(debug_until).build().to_json_dict(), + } + # The overridden flag is on and serves variation 0, so the dependent flag's prerequisite passes. + overridden = FlagBuilder('overridden-flag').version(7).on(True).fallthrough_variation(0).variations(True).track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + with make_initialized_client(ld_data, source) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + + # A flag with no override keeps its tracking fields. + assert flags_state['plain-tracked']['trackEvents'] is True + assert flags_state['plain-tracked']['debugEventsUntilDate'] == debug_until + + # The overridden flag and the flag that depends on it stay in the state with their values + # and marked reasons, but with no tracking fields. + for key in ('overridden-flag', 'dependent-tracked'): + assert flags_state[key]['reason']['overrideAffected'] is True, key + assert 'trackEvents' not in flags_state[key], key + assert 'trackReason' not in flags_state[key], key + assert 'debugEventsUntilDate' not in flags_state[key], key + assert state.get_flag_value('overridden-flag') is True + assert state.get_flag_value('dependent-tracked') is True + assert flags_state['overridden-flag']['version'] == 7 + + +def test_all_flags_state_omits_details_of_override_affected_flags_when_details_only_for_tracked_flags(): + # With details only for tracked flags, an override-affected flag counts as untracked, so its + # version and reason are omitted like any other untracked flag, and its value stays. + overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + with make_initialized_client({}, source) as client: + state = client.all_flags_state(user, with_reasons=True, details_only_for_tracked_flags=True) + flags_state = state.to_json_dict()['$flagsState'] + assert state.get_flag_value('overridden-flag') is True + assert 'version' not in flags_state['overridden-flag'] + assert 'reason' not in flags_state['overridden-flag'] + + +def test_wrong_type_result_of_overridden_flag_stays_marked(): + details = [] + + class CapturingHook(Hook): + @property + def metadata(self) -> Metadata: + return Metadata(name='capturing-hook') + + def before_evaluation(self, series_context: EvaluationSeriesContext, data: dict) -> dict: + return data + + def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, detail: EvaluationDetail) -> dict: + details.append(detail) + return data + + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', 'not-a-stage')}) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor, hooks=[CapturingHook()]) + with LDClient(config, start_wait=0) as client: + stage, _ = client.migration_variation('overridden-flag', user, Stage.OFF) + assert stage == Stage.OFF + assert len(details) == 1 + assert details[0].value == 'off' + assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True} + + +def test_failed_evaluation_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + with make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + # The failure of the override flag is marked. The failure of the ordinary flag is not. + detail = client.variation_detail('overridden-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + detail = client.variation_detail('plain-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + # The marked record produces no individual event. The ordinary record keeps its tracking. + records = evaluation_events_by_key(client) + assert records['overridden-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['plain-flag'].track_events is True + + +def test_all_flags_state_turns_off_event_tracking_for_an_override_flag_whose_evaluation_fails(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + with make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + state = client.all_flags_state(user, with_reasons=True) + + # The failed override flag stays in the state with a marked reason and no tracking + # fields. The failed ordinary flag keeps its tracking fields. + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['overridden-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert 'trackEvents' not in flags_state['overridden-flag'] + assert flags_state['plain-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + assert flags_state['plain-flag']['trackEvents'] is True + + +def test_override_affected_evaluations_appear_only_in_summary_output(): + # End to end through the real event processor: the overridden flag requests individual + # feature events and debug events, and an ordinary flag requests feature events. + debug_until = int(time.time() * 1000) + 100000 + overridden = FlagBuilder('flag-tracked-override').version(300).on(False).off_variation(0).variations('override-value').track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'flag-tracked-override': overridden}) + mock_http = MockHttp() + + initializer = StaticInitializer({'flag-normal': normal}, {}) + datasystem = custom().initializers([initializer.builder]).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, diagnostic_opt_out=True, event_processor_class=lambda config: DefaultEventProcessor(config, mock_http)) + with LDClient(config, start_wait=5) as client: + assert client.is_initialized() is True + for _ in range(2): + assert client.variation('flag-tracked-override', user, 'default1') == 'override-value' + assert client.variation('flag-normal', user, 'default2') == 'normal-value' + client.flush() + client._event_processor._wait_until_inactive() + + assert mock_http.request_data is not None + output = json.loads(mock_http.request_data) + kinds = sorted(e['kind'] for e in output) + assert kinds == ['feature', 'index', 'summary'] + feature = [e for e in output if e['kind'] == 'feature'][0] + assert feature['key'] == 'flag-normal' + summary = [e for e in output if e['kind'] == 'summary'][0] + assert summary['features']['flag-tracked-override']['default'] == 'default1' + assert summary['features']['flag-tracked-override']['counters'] == [ + {'count': 2, 'value': 'override-value', 'variation': 0, 'version': 300, 'overrideAffected': True} + ] + assert summary['features']['flag-normal']['counters'] == [{'count': 1, 'value': 'normal-value', 'variation': 0, 'version': 100}] diff --git a/ldclient/testing/testdata/override-vectors/vectors.json b/ldclient/testing/testdata/override-vectors/vectors.json new file mode 100644 index 00000000..82dc62da --- /dev/null +++ b/ldclient/testing/testdata/override-vectors/vectors.json @@ -0,0 +1,316 @@ +{ + "schemaVersion": "0.4.0", + "vectors": [ + { + "description": "An override takes precedence over LaunchDarkly data for the same flag key", + "group": "precedence", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-a": { + "key": "flag-a", + "version": 7, + "on": true, + "variations": ["ld-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "salt": "salt-a" + } + }, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-a", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "override-value", + "variationIndex": 0, + "reason": { "kind": "FALLTHROUGH", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "An overridden flag is served when the SDK has not initialized from LaunchDarkly", + "group": "uninitialized", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-a", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "override-value", + "variationIndex": 0, + "reason": { "kind": "FALLTHROUGH", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A flag absent from the override layer is evaluated from LaunchDarkly data and is not marked", + "group": "passthrough", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-b": { + "key": "flag-b", + "version": 3, + "on": true, + "variations": ["off-value", "ld-value"], + "fallthrough": { "variation": 1 }, + "offVariation": 0, + "salt": "salt-b" + } + }, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-b", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "ld-value", + "variationIndex": 1, + "reason": { "kind": "FALLTHROUGH" }, + "summaryOverrideAffected": false + } + }, + { + "description": "A flag absent from the override layer still short-circuits when the SDK has not initialized", + "group": "passthrough", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flagValues": { "flag-a": "override-value" } + }, + "evaluate": { + "flagKey": "flag-b", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "fallback", + "variationIndex": null, + "reason": { "kind": "ERROR", "errorKind": "CLIENT_NOT_READY" } + } + }, + { + "description": "A full flag override with a matching targeting rule serves the rule's variation and is marked", + "group": "full-evaluation", + "launchDarklyData": { + "initialized": true, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-c": { + "key": "flag-c", + "version": 1, + "on": true, + "variations": ["default-value", "eu-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-eu", + "variation": 1, + "clauses": [ + { + "contextKind": "user", + "attribute": "region", + "op": "in", + "values": ["eu"], + "negate": false + } + ] + } + ], + "salt": "salt-c" + } + } + }, + "evaluate": { + "flagKey": "flag-c", + "context": { "kind": "user", "key": "user-1", "region": "eu" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "eu-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-eu", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "An override flag resolves a referenced segment through the merged view while the SDK is uninitialized", + "group": "merged-view", + "launchDarklyData": { + "initialized": false, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-d": { + "key": "flag-d", + "version": 1, + "on": true, + "variations": ["default-value", "member-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-seg", + "variation": 1, + "clauses": [ + { + "attribute": "", + "op": "segmentMatch", + "values": ["seg-1"], + "negate": false + } + ] + } + ], + "salt": "salt-d" + } + }, + "segments": { + "seg-1": { + "key": "seg-1", + "version": 1, + "included": ["user-1"], + "excluded": [], + "salt": "seg-salt-1" + } + } + }, + "evaluate": { + "flagKey": "flag-d", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "member-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-seg", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A flag is marked overrideAffected when it matches via an overridden segment", + "group": "marking", + "launchDarklyData": { + "initialized": true, + "flags": { + "flag-f": { + "key": "flag-f", + "version": 4, + "on": true, + "variations": ["default-value", "member-value"], + "fallthrough": { "variation": 0 }, + "offVariation": 0, + "rules": [ + { + "id": "rule-seg", + "variation": 1, + "clauses": [ + { + "attribute": "", + "op": "segmentMatch", + "values": ["seg-2"], + "negate": false + } + ] + } + ], + "salt": "salt-f" + } + }, + "segments": { + "seg-2": { + "key": "seg-2", + "version": 1, + "included": [], + "excluded": [], + "salt": "seg-salt-2" + } + } + }, + "overrides": { + "segments": { + "seg-2": { + "key": "seg-2", + "version": 2, + "included": ["user-1"], + "excluded": [], + "salt": "seg-salt-2" + } + } + }, + "evaluate": { + "flagKey": "flag-f", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "member-value", + "variationIndex": 1, + "reason": { "kind": "RULE_MATCH", "ruleIndex": 0, "ruleId": "rule-seg", "overrideAffected": true }, + "summaryOverrideAffected": true + } + }, + { + "description": "A malformed override flag returns the caller default with an error reason and is marked overrideAffected", + "group": "error-handling", + "launchDarklyData": { + "initialized": true, + "flags": {}, + "segments": {} + }, + "overrides": { + "flags": { + "flag-e": { + "key": "flag-e", + "version": 1, + "on": true, + "variations": ["only-value"], + "fallthrough": { "variation": 5 }, + "offVariation": 0, + "salt": "salt-e" + } + } + }, + "evaluate": { + "flagKey": "flag-e", + "context": { "kind": "user", "key": "user-1" }, + "defaultValue": "fallback" + }, + "expect": { + "value": "fallback", + "variationIndex": null, + "reason": { "kind": "ERROR", "errorKind": "MALFORMED_FLAG", "overrideAffected": true }, + "summaryOverrideAffected": true + } + } + ] +}