Skip unchanged cached subtrees when removing nested nulls

API-INTERNAL.md

@@ -33,6 +33,9 @@ is reused by every connection.</p>
 <dt><a href="#getDefaultKeyStates">getDefaultKeyStates()</a></dt>
 <dd><p>Getter - returns the default key states.</p>
 </dd>
+<dt><a href="#getNullFreeDefaultKeyState">getNullFreeDefaultKeyState()</a></dt>
+<dd><p>Getter - returns the default key state without nested nulls, cached so clear() reuses the same references.</p>
+</dd>
 <dt><a href="#getDeferredInitTask">getDeferredInitTask()</a></dt>
 <dd><p>Getter - returns the deffered init task.</p>
 </dd>
@@ -206,6 +209,12 @@ Getter - returns the merge queue promise.
 ## getDefaultKeyStates()
 Getter - returns the default key states.
 
+**Kind**: global function  
+<a name="getNullFreeDefaultKeyState"></a>
+
+## getNullFreeDefaultKeyState()
+Getter - returns the default key state without nested nulls, cached so clear() reuses the same references.
+
 **Kind**: global function  
 <a name="getDeferredInitTask"></a>
 

lib/Onyx.ts

@@ -64,13 +64,15 @@ function init({
             const individual: Array<[OnyxKey, OnyxEntry<KeyValueMapping[OnyxKey]>]> = [];
             const collectionBatches = new Map<string, {partial: NonUndefined<OnyxCollection<KeyValueMapping[OnyxKey]>>; previous: NonUndefined<OnyxCollection<KeyValueMapping[OnyxKey]>>}>();
 
-            for (const [key, value] of pairs) {
+            for (const [key, valueFromStorage] of pairs) {
                 // RAM-only keys should never sync from storage as they may have stale persisted data
                 // from before the key was migrated to RAM-only.
                 if (OnyxKeys.isRamOnlyKey(key)) {
                     continue;
                 }
 
+                const value = utils.needsNormalization(valueFromStorage) ? utils.removeNestedNullValues(valueFromStorage) : valueFromStorage;
+
                 const collectionKey = OnyxKeys.getCollectionKey(key);
                 const isCollectionMember = !!collectionKey && OnyxKeys.isCollectionMemberKey(collectionKey, key);
 
@@ -487,7 +489,7 @@ function clear(keysToPreserve: OnyxKey[] = []): Promise<void> {
                         //      since collection key subscribers need to be updated differently
                         if (!isKeyToPreserve) {
                             const oldValue = cache.get(key);
-                            const newValue = defaultKeyStates[key] ?? null;
+                            const newValue = OnyxUtils.getNullFreeDefaultKeyState(key);
                             valuesSetByClear[key] = newValue;
                             if (newValue !== oldValue) {
                                 cache.set(key, newValue);

lib/OnyxUtils.ts

@@ -76,6 +76,9 @@ const mergeQueuesWithStaleRead = new WeakSet<Array<OnyxValue<OnyxKey>>>();
 // Optional user-provided key value states set when Onyx initializes or clears
 let defaultKeyStates: Record<OnyxKey, OnyxValue<OnyxKey>> = {};
 
+// Cache for getNullFreeDefaultKeyState()
+let nullFreeDefaultKeyStates: Record<OnyxKey, OnyxValue<OnyxKey>> = {};
+
 let snapshotKey: OnyxKey | null = null;
 
 // Connections can be made before `Onyx.init`. They would wait for this task before resolving.
@@ -123,6 +126,14 @@ function getDefaultKeyStates(): Record<OnyxKey, OnyxValue<OnyxKey>> {
     return defaultKeyStates;
 }
 
+/** Getter - returns the default key state without nested nulls, cached so clear() reuses the same references. */
+function getNullFreeDefaultKeyState(key: OnyxKey): OnyxValue<OnyxKey> {
+    if (!(key in nullFreeDefaultKeyStates)) {
+        nullFreeDefaultKeyStates[key] = utils.removeNestedNullValues(defaultKeyStates[key] ?? null);
+    }
+    return nullFreeDefaultKeyStates[key];
+}
+
 /**
  * Getter - returns the deffered init task.
  */
@@ -191,6 +202,7 @@ function initStoreValues(keys: DeepRecord<string, OnyxKey>, initialKeyStates: Pa
 
     // Set our default key states to use when initializing and clearing Onyx data
     defaultKeyStates = initialKeyStates;
+    nullFreeDefaultKeyStates = {};
 
     DevTools.initState(initialKeyStates);
 
@@ -781,7 +793,7 @@ function prepareKeyValuePairsForStorage(
             continue;
         }
 
-        const valueWithoutNestedNullValues = shouldRemoveNestedNulls ?? true ? utils.removeNestedNullValues(value) : value;
+        const valueWithoutNestedNullValues = shouldRemoveNestedNulls ?? true ? utils.removeNestedNullValues(value, cache.get(key)) : value;
 
         if (valueWithoutNestedNullValues !== undefined) {
             pairs.push([key, valueWithoutNestedNullValues, replaceNullPatches?.[key]]);
@@ -1119,7 +1131,7 @@ function setWithRetry<TKey extends OnyxKey>({key, value, options}: SetParams<TKe
         return Promise.resolve();
     }
 
-    const valueWithoutNestedNullValues = utils.removeNestedNullValues(value) as OnyxValue<TKey>;
+    const valueWithoutNestedNullValues = utils.removeNestedNullValues(value, existingValue) as OnyxValue<TKey>;
     const hasChanged = options?.skipCacheCheck ? true : cache.hasValueChanged(key, valueWithoutNestedNullValues);
 
     OnyxUtils.logKeyChanged(OnyxUtils.METHOD.SET, key, value, hasChanged);
@@ -1697,6 +1709,7 @@ const OnyxUtils = {
     getMergeQueue,
     getMergeQueuePromise,
     getDefaultKeyStates,
+    getNullFreeDefaultKeyState,
     getDeferredInitTask,
     afterInit,
     initStoreValues,

lib/storage/providers/IDBKeyValProvider/index.ts

@@ -154,7 +154,14 @@ const provider: StorageProvider<UseStore | undefined> = {
         return (
             IDB.get(key, provider.store)
                 // idb-keyval returns undefined for missing items, but this needs to return null so that idb-keyval does the same thing as SQLiteStorage.
-                .then((val) => (val === undefined ? null : val))
+                .then((val) => {
+                    if (val === undefined) {
+                        return null;
+                    }
+
+                    // Stored values can hold nested nulls, but values handed to the cache must not.
+                    return utils.needsNormalization(val) ? utils.removeNestedNullValues(val) : val;
+                })
         );
     },
     removeItem(key) {

lib/storage/providers/SQLiteProvider.ts

@@ -215,7 +215,10 @@ const provider: StorageProvider<NitroSQLiteConnection | undefined> = {
                 return null;
             }
 
-            return JSON.parse(result.valueJSON);
+            const value = JSON.parse(result.valueJSON);
+
+            // valueJSON is a number for numeric values, so the type is checked before includes().
+            return typeof value === 'object' && value !== null && result.valueJSON.includes('null') ? utils.removeNestedNullValues(value) : value;
         });
     },
     multiGet(keys) {

lib/utils.ts

@@ -1,5 +1,6 @@
 import {deepEqual} from 'fast-equals';
 import type {OnyxInput, OnyxKey} from './types';
+import * as Logger from './Logger';
 
 type EmptyObject = Record<string, never>;
 type EmptyValue = EmptyObject | null | undefined;
@@ -176,7 +177,8 @@ function mergeObject<TObject extends Record<string, unknown>>(
             // of the merged object only.
             const sourcePropertyWithoutMark = {...sourceProperty};
             delete sourcePropertyWithoutMark.ONYX_INTERNALS__REPLACE_OBJECT_MARK;
-            destination[key] = sourcePropertyWithoutMark;
+            // Built in "mark" mode, so it can still hold nested nulls.
+            destination[key] = options.shouldRemoveNestedNulls ? removeNestedNullValues(sourcePropertyWithoutMark) : sourcePropertyWithoutMark;
             continue;
         }
 
@@ -264,12 +266,25 @@ function needsNormalization(value: unknown): boolean {
     return false;
 }
 
-/** Deep removes the nested null values from the given value. Returns the original reference if no nulls were found. */
-function removeNestedNullValues<TValue extends OnyxInput<OnyxKey> | null>(value: TValue): TValue {
+/**
+ * Deep removes nested null values from the given value, leaving arrays as they are. Returns the original reference if no nulls were found.
+ *
+ * Objects that are the same reference as the object at the same path in `nullFreeReference` are returned without being traversed,
+ * so `nullFreeReference` must hold no nested nulls outside arrays (pass the live cache value).
+ */
+function removeNestedNullValues<TValue extends OnyxInput<OnyxKey> | null>(value: TValue, nullFreeReference?: unknown): TValue {
     if (value === null || value === undefined || typeof value !== 'object' || Array.isArray(value)) {
         return value;
     }
 
+    if (value === nullFreeReference) {
+        if (process.env.NODE_ENV === 'development' && needsNormalization(value)) {
+            Logger.logAlert('removeNestedNullValues skipped a cached subtree that holds nested null values. Onyx values must not be mutated in place.');
+        }
+        return value;
+    }
+
+    const reference = nullFreeReference !== null && typeof nullFreeReference === 'object' && !Array.isArray(nullFreeReference) ? (nullFreeReference as Record<string, unknown>) : undefined;
     let hasChanged = false;
     const result: Record<string, unknown> = {};
 
@@ -283,7 +298,7 @@ function removeNestedNullValues<TValue extends OnyxInput<OnyxKey> | null>(value:
         }
 
         if (typeof propertyValue === 'object' && !Array.isArray(propertyValue)) {
-            const cleaned = removeNestedNullValues(propertyValue);
+            const cleaned = removeNestedNullValues(propertyValue, reference?.[key]);
             if (cleaned !== propertyValue) {
                 hasChanged = true;
             }