From a3aa953e09d095deb94fbc7705eb1b0af2e93f81 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:57:19 +0000 Subject: [PATCH] fix: Keep deleted flags and segments deleted in the persistent store after an FDv2 store outage After a persistent store outage, the FDv2 store writes its in-memory data back to the persistent store. Store.commit() and AsyncStore.commit() read that data with InMemoryFeatureStore.all(), which leaves out deleted items. The write-back replaces the whole store, so it removed the tombstone of each item deleted during the outage, and an update with an older version could then add the item again. The write-back now includes deleted items. The contract test services now advertise persistent-data-store-recovery, so that the store recovery tests run. Generated-by: termag --- contract-tests/async_service.py | 1 + contract-tests/service.py | 1 + ldclient/impl/datasystem/async_store.py | 5 +- ldclient/impl/datasystem/store.py | 15 +++- .../datasystem/test_fdv2_async_persistence.py | 41 +++++++++++ .../impl/datasystem/test_fdv2_persistence.py | 70 +++++++++++++++++++ 6 files changed, 130 insertions(+), 3 deletions(-) diff --git a/contract-tests/async_service.py b/contract-tests/async_service.py index dae59b19..4fa56dbb 100644 --- a/contract-tests/async_service.py +++ b/contract-tests/async_service.py @@ -68,6 +68,7 @@ async def handle_status(request: aiohttp.web.Request) -> aiohttp.web.Response: 'flag-value-change-listeners', 'migrations', 'persistent-data-store-redis', + 'persistent-data-store-recovery', 'fdv1-fallback', 'flag-overrides', 'retry-conformance-fdv1-streaming', diff --git a/contract-tests/service.py b/contract-tests/service.py index eb977734..916b8813 100644 --- a/contract-tests/service.py +++ b/contract-tests/service.py @@ -83,6 +83,7 @@ def status(): 'persistent-data-store-redis', 'persistent-data-store-dynamodb', 'persistent-data-store-consul', + 'persistent-data-store-recovery', 'flag-change-listeners', 'flag-value-change-listeners', 'fdv1-fallback', diff --git a/ldclient/impl/datasystem/async_store.py b/ldclient/impl/datasystem/async_store.py index 248726d1..486ca5f8 100644 --- a/ldclient/impl/datasystem/async_store.py +++ b/ldclient/impl/datasystem/async_store.py @@ -173,9 +173,12 @@ def __mapping(data: Dict[str, ModelEntity]) -> Dict[str, Dict[str, Any]]: all_data: Optional[Collections] = None with self._lock: if self._should_persist(): + # Include the deleted items, so that the store keeps the + # version of each deletion and an older update cannot add + # the item again. all_data = {} for kind in [FEATURES, SEGMENTS]: - all_data[kind] = self._memory_store.all(kind, __mapping_from_kind(kind)) + all_data[kind] = self._memory_store.all_including_deleted(kind, __mapping_from_kind(kind)) if all_data is None: return None diff --git a/ldclient/impl/datasystem/store.py b/ldclient/impl/datasystem/store.py index 3904312b..be69ff36 100644 --- a/ldclient/impl/datasystem/store.py +++ b/ldclient/impl/datasystem/store.py @@ -89,6 +89,14 @@ def all(self, kind: VersionedDataKind, callback: Callable[[Any], Any] = lambda x ) ) + def all_including_deleted(self, kind: VersionedDataKind, callback: Callable[[Any], Any] = lambda x: x) -> Any: + """ + Returns all items of a kind, including deleted items. A deleted item is + a placeholder that keeps the version of the deletion. + """ + with self._lock.read(): + return callback(dict(self._items[kind])) + def set_basis(self, collections: Collections) -> bool: """ Initializes the store with a full set of data, replacing any existing data. @@ -554,10 +562,13 @@ def __mapping(data: Dict[str, ModelEntity]) -> Dict[str, Dict[str, Any]]: store = self._persistent_store assert store is not None try: - # Get all data from memory store and write to persistent store + # Get all data from memory store and write to persistent store. + # Include the deleted items, so that the persistent store keeps + # the version of each deletion and an older update cannot add + # the item again. all_data: Collections = {} for kind in [FEATURES, SEGMENTS]: - all_data[kind] = self._memory_store.all(kind, __mapping_from_kind(kind)) + all_data[kind] = self._memory_store.all_including_deleted(kind, __mapping_from_kind(kind)) store.init(all_data) except Exception as e: return e diff --git a/ldclient/testing/impl/datasystem/test_fdv2_async_persistence.py b/ldclient/testing/impl/datasystem/test_fdv2_async_persistence.py index dcfab21b..aa95e92f 100644 --- a/ldclient/testing/impl/datasystem/test_fdv2_async_persistence.py +++ b/ldclient/testing/impl/datasystem/test_fdv2_async_persistence.py @@ -188,6 +188,47 @@ async def test_commit_writes_memory_to_store(): assert "flag-a" in async_store.snapshot()[FEATURES] +@pytest.mark.asyncio +async def test_commit_writes_deleted_items_to_store(): + async_store = FakeAsyncFeatureStore() + store = AsyncStore(Listeners(), Listeners()) + store.with_async_persistence(async_store, True, None) + + await store.apply( + ChangeSet( + intent_code=IntentCode.TRANSFER_FULL, + changes=[ + Change(action=ChangeType.PUT, kind=ObjectKind.FLAG, key="flag-a", version=1, object=_flag("flag-a", 1, True)), + Change(action=ChangeType.PUT, kind=ObjectKind.FLAG, key="flag-b", version=1, object=_flag("flag-b", 1, True)), + Change(action=ChangeType.PUT, kind=ObjectKind.SEGMENT, key="segment-a", version=1, object={"key": "segment-a", "version": 1}), + ], + selector=Selector.no_selector(), + ), + True, + ) + await store.apply( + ChangeSet( + intent_code=IntentCode.TRANSFER_CHANGES, + changes=[ + Change(action=ChangeType.DELETE, kind=ObjectKind.FLAG, key="flag-b", version=2), + Change(action=ChangeType.DELETE, kind=ObjectKind.SEGMENT, key="segment-a", version=2), + ], + selector=Selector.no_selector(), + ), + True, + ) + + # Write the memory store back to the async store. + err = await store.commit() + + # The async store holds the kept flag and a tombstone for each deleted item. + assert err is None + snapshot = async_store.snapshot() + assert snapshot[FEATURES]["flag-a"]["version"] == 1 + assert snapshot[FEATURES].get("flag-b") == {"key": "flag-b", "deleted": True, "version": 2} + assert snapshot[SEGMENTS].get("segment-a") == {"key": "segment-a", "deleted": True, "version": 2} + + @pytest.mark.asyncio async def test_commit_returns_error_on_failure(): class FailingStore(FakeAsyncFeatureStore): diff --git a/ldclient/testing/impl/datasystem/test_fdv2_persistence.py b/ldclient/testing/impl/datasystem/test_fdv2_persistence.py index d9f68bd5..9122f0bd 100644 --- a/ldclient/testing/impl/datasystem/test_fdv2_persistence.py +++ b/ldclient/testing/impl/datasystem/test_fdv2_persistence.py @@ -728,6 +728,76 @@ def test_persistent_store_commit_encodes_data_correctly(): assert flag_in_store["on"] is True +def test_persistent_store_commit_writes_deleted_items(): + """Test that Store.commit() writes deleted items to the persistent store as tombstones""" + from ldclient.impl.datasystem.store import Store + from ldclient.impl.listeners import Listeners + from ldclient.interfaces import ( + Change, + ChangeSet, + ChangeType, + IntentCode, + ObjectKind, + Selector + ) + + persistent_store = StubFeatureStore() + store = Store(Listeners(), Listeners()) + store.with_persistence(persistent_store, True, None) + + store.apply( + ChangeSet( + intent_code=IntentCode.TRANSFER_FULL, + changes=[ + Change( + action=ChangeType.PUT, + kind=ObjectKind.FLAG, + key="kept-flag", + version=1, + object={"key": "kept-flag", "version": 1, "on": True}, + ), + Change( + action=ChangeType.PUT, + kind=ObjectKind.FLAG, + key="deleted-flag", + version=1, + object={"key": "deleted-flag", "version": 1, "on": True}, + ), + Change( + action=ChangeType.PUT, + kind=ObjectKind.SEGMENT, + key="deleted-segment", + version=1, + object={"key": "deleted-segment", "version": 1}, + ), + ], + selector=Selector.no_selector(), + ), + True, + ) + store.apply( + ChangeSet( + intent_code=IntentCode.TRANSFER_CHANGES, + changes=[ + Change(action=ChangeType.DELETE, kind=ObjectKind.FLAG, key="deleted-flag", version=2), + Change(action=ChangeType.DELETE, kind=ObjectKind.SEGMENT, key="deleted-segment", version=2), + ], + selector=Selector.no_selector(), + ), + True, + ) + + # Write the memory store back to the persistent store. + err = store.commit() + + # The persistent store holds the kept flag and a tombstone for each deleted item. + assert err is None + snapshot = persistent_store.get_data_snapshot() + assert snapshot[FEATURES]["kept-flag"]["version"] == 1 + assert snapshot[FEATURES].get("deleted-flag") == {"key": "deleted-flag", "deleted": True, "version": 2} + assert snapshot[SEGMENTS].get("deleted-segment") == {"key": "deleted-segment", "deleted": True, "version": 2} + + def test_persistent_store_commit_with_no_persistent_store(): """Test that Store.commit() safely handles the case where there's no persistent store""" from ldclient.impl.datasystem.store import Store