diff --git a/contract-tests/async_service.py b/contract-tests/async_service.py index dae59b19..4fa56dbb 100644 --- a/contract-tests/async_service.py +++ b/contract-tests/async_service.py @@ -68,6 +68,7 @@ async def handle_status(request: aiohttp.web.Request) -> aiohttp.web.Response: 'flag-value-change-listeners', 'migrations', 'persistent-data-store-redis', + 'persistent-data-store-recovery', 'fdv1-fallback', 'flag-overrides', 'retry-conformance-fdv1-streaming', diff --git a/contract-tests/service.py b/contract-tests/service.py index eb977734..916b8813 100644 --- a/contract-tests/service.py +++ b/contract-tests/service.py @@ -83,6 +83,7 @@ def status(): 'persistent-data-store-redis', 'persistent-data-store-dynamodb', 'persistent-data-store-consul', + 'persistent-data-store-recovery', 'flag-change-listeners', 'flag-value-change-listeners', 'fdv1-fallback', diff --git a/ldclient/impl/datasystem/async_store.py b/ldclient/impl/datasystem/async_store.py index 248726d1..486ca5f8 100644 --- a/ldclient/impl/datasystem/async_store.py +++ b/ldclient/impl/datasystem/async_store.py @@ -173,9 +173,12 @@ def __mapping(data: Dict[str, ModelEntity]) -> Dict[str, Dict[str, Any]]: all_data: Optional[Collections] = None with self._lock: if self._should_persist(): + # Include the deleted items, so that the store keeps the + # version of each deletion and an older update cannot add + # the item again. all_data = {} for kind in [FEATURES, SEGMENTS]: - all_data[kind] = self._memory_store.all(kind, __mapping_from_kind(kind)) + all_data[kind] = self._memory_store.all_including_deleted(kind, __mapping_from_kind(kind)) if all_data is None: return None diff --git a/ldclient/impl/datasystem/store.py b/ldclient/impl/datasystem/store.py index 3904312b..be69ff36 100644 --- a/ldclient/impl/datasystem/store.py +++ b/ldclient/impl/datasystem/store.py @@ -89,6 +89,14 @@ def all(self, kind: VersionedDataKind, callback: Callable[[Any], Any] = lambda x ) ) + def all_including_deleted(self, kind: VersionedDataKind, callback: Callable[[Any], Any] = lambda x: x) -> Any: + """ + Returns all items of a kind, including deleted items. A deleted item is + a placeholder that keeps the version of the deletion. + """ + with self._lock.read(): + return callback(dict(self._items[kind])) + def set_basis(self, collections: Collections) -> bool: """ Initializes the store with a full set of data, replacing any existing data. @@ -554,10 +562,13 @@ def __mapping(data: Dict[str, ModelEntity]) -> Dict[str, Dict[str, Any]]: store = self._persistent_store assert store is not None try: - # Get all data from memory store and write to persistent store + # Get all data from memory store and write to persistent store. + # Include the deleted items, so that the persistent store keeps + # the version of each deletion and an older update cannot add + # the item again. all_data: Collections = {} for kind in [FEATURES, SEGMENTS]: - all_data[kind] = self._memory_store.all(kind, __mapping_from_kind(kind)) + all_data[kind] = self._memory_store.all_including_deleted(kind, __mapping_from_kind(kind)) store.init(all_data) except Exception as e: return e diff --git a/ldclient/testing/impl/datasystem/test_fdv2_async_persistence.py b/ldclient/testing/impl/datasystem/test_fdv2_async_persistence.py index dcfab21b..aa95e92f 100644 --- a/ldclient/testing/impl/datasystem/test_fdv2_async_persistence.py +++ b/ldclient/testing/impl/datasystem/test_fdv2_async_persistence.py @@ -188,6 +188,47 @@ async def test_commit_writes_memory_to_store(): assert "flag-a" in async_store.snapshot()[FEATURES] +@pytest.mark.asyncio +async def test_commit_writes_deleted_items_to_store(): + async_store = FakeAsyncFeatureStore() + store = AsyncStore(Listeners(), Listeners()) + store.with_async_persistence(async_store, True, None) + + await store.apply( + ChangeSet( + intent_code=IntentCode.TRANSFER_FULL, + changes=[ + Change(action=ChangeType.PUT, kind=ObjectKind.FLAG, key="flag-a", version=1, object=_flag("flag-a", 1, True)), + Change(action=ChangeType.PUT, kind=ObjectKind.FLAG, key="flag-b", version=1, object=_flag("flag-b", 1, True)), + Change(action=ChangeType.PUT, kind=ObjectKind.SEGMENT, key="segment-a", version=1, object={"key": "segment-a", "version": 1}), + ], + selector=Selector.no_selector(), + ), + True, + ) + await store.apply( + ChangeSet( + intent_code=IntentCode.TRANSFER_CHANGES, + changes=[ + Change(action=ChangeType.DELETE, kind=ObjectKind.FLAG, key="flag-b", version=2), + Change(action=ChangeType.DELETE, kind=ObjectKind.SEGMENT, key="segment-a", version=2), + ], + selector=Selector.no_selector(), + ), + True, + ) + + # Write the memory store back to the async store. + err = await store.commit() + + # The async store holds the kept flag and a tombstone for each deleted item. + assert err is None + snapshot = async_store.snapshot() + assert snapshot[FEATURES]["flag-a"]["version"] == 1 + assert snapshot[FEATURES].get("flag-b") == {"key": "flag-b", "deleted": True, "version": 2} + assert snapshot[SEGMENTS].get("segment-a") == {"key": "segment-a", "deleted": True, "version": 2} + + @pytest.mark.asyncio async def test_commit_returns_error_on_failure(): class FailingStore(FakeAsyncFeatureStore): diff --git a/ldclient/testing/impl/datasystem/test_fdv2_persistence.py b/ldclient/testing/impl/datasystem/test_fdv2_persistence.py index d9f68bd5..9122f0bd 100644 --- a/ldclient/testing/impl/datasystem/test_fdv2_persistence.py +++ b/ldclient/testing/impl/datasystem/test_fdv2_persistence.py @@ -728,6 +728,76 @@ def test_persistent_store_commit_encodes_data_correctly(): assert flag_in_store["on"] is True +def test_persistent_store_commit_writes_deleted_items(): + """Test that Store.commit() writes deleted items to the persistent store as tombstones""" + from ldclient.impl.datasystem.store import Store + from ldclient.impl.listeners import Listeners + from ldclient.interfaces import ( + Change, + ChangeSet, + ChangeType, + IntentCode, + ObjectKind, + Selector + ) + + persistent_store = StubFeatureStore() + store = Store(Listeners(), Listeners()) + store.with_persistence(persistent_store, True, None) + + store.apply( + ChangeSet( + intent_code=IntentCode.TRANSFER_FULL, + changes=[ + Change( + action=ChangeType.PUT, + kind=ObjectKind.FLAG, + key="kept-flag", + version=1, + object={"key": "kept-flag", "version": 1, "on": True}, + ), + Change( + action=ChangeType.PUT, + kind=ObjectKind.FLAG, + key="deleted-flag", + version=1, + object={"key": "deleted-flag", "version": 1, "on": True}, + ), + Change( + action=ChangeType.PUT, + kind=ObjectKind.SEGMENT, + key="deleted-segment", + version=1, + object={"key": "deleted-segment", "version": 1}, + ), + ], + selector=Selector.no_selector(), + ), + True, + ) + store.apply( + ChangeSet( + intent_code=IntentCode.TRANSFER_CHANGES, + changes=[ + Change(action=ChangeType.DELETE, kind=ObjectKind.FLAG, key="deleted-flag", version=2), + Change(action=ChangeType.DELETE, kind=ObjectKind.SEGMENT, key="deleted-segment", version=2), + ], + selector=Selector.no_selector(), + ), + True, + ) + + # Write the memory store back to the persistent store. + err = store.commit() + + # The persistent store holds the kept flag and a tombstone for each deleted item. + assert err is None + snapshot = persistent_store.get_data_snapshot() + assert snapshot[FEATURES]["kept-flag"]["version"] == 1 + assert snapshot[FEATURES].get("deleted-flag") == {"key": "deleted-flag", "deleted": True, "version": 2} + assert snapshot[SEGMENTS].get("deleted-segment") == {"key": "deleted-segment", "deleted": True, "version": 2} + + def test_persistent_store_commit_with_no_persistent_store(): """Test that Store.commit() safely handles the case where there's no persistent store""" from ldclient.impl.datasystem.store import Store