Web Experience Backstops & Telemetry

app/web/src/newhotness/Workspace.vue

@@ -33,7 +33,7 @@
         :changeSetId="changeSetId"
         :componentId="componentId"
         :viewId="viewId"
-        connected
+        :connected="haveWSConn"
       />
 
       <!-- Right -->
@@ -236,10 +236,9 @@ const changeSetId = computed(() => props.changeSetId);
 
 const coldStartInProgress = computed(() => heimdall.muspelheimInProgress.value);
 
-// const haveWSConn = computed<boolean>(() => {
-//   if (!heimdall.initCompleted.value) return true; // dont show error if we havent gotten started
-//   return !!heimdall.wsConnections.value[props.workspacePk];
-// });
+const haveWSConn = computed<boolean>(() => {
+  return !!heimdall.wsConnections.value[props.workspacePk];
+});
 
 // no tan stack queries hitting sqlite until after the cold start has finished
 const queriesEnabled = computed(
@@ -637,11 +636,42 @@ watch(
         return;
       }
       await heimdall.registerBearerToken(newWorkspacePk, workspaceToken);
-      heimdall.niflheim(newWorkspacePk, newChangeSetId, true, false);
+      await heimdall.niflheim(newWorkspacePk, newChangeSetId, true, false);
     }
   },
 );
 
+const EVENTUALLY_CONSISTENT_TIME = 30 * 1000; // 30 seconds
+let indexInterval: NodeJS.Timeout;
+
+watch(
+  () => [props.changeSetId],
+  () => {
+    // clear any interval for other change set
+    if (indexInterval) clearInterval(indexInterval);
+
+    // set a new interval (note: above watcher fires first cold start from the user selection change)
+    indexInterval = setInterval(() => {
+      heimdall.syncAtoms(props.workspacePk, props.changeSetId);
+    }, EVENTUALLY_CONSISTENT_TIME);
+  },
+  { immediate: true },
+);
+
+watch(
+  () => heimdall.indexTouches.get(props.changeSetId),
+  () => {
+    if (lobby.value) return; // dont do anything when in the lobby
+    // each time we get index updates, clear the interval
+    if (indexInterval) clearInterval(indexInterval);
+    // and restart it
+    indexInterval = setInterval(() => {
+      heimdall.syncAtoms(props.workspacePk, props.changeSetId);
+    }, EVENTUALLY_CONSISTENT_TIME);
+  },
+  { immediate: true },
+);
+
 const indexFailedToLoad = computed(() => {
   // NOTE: this represents a 5XX error that was retried 5 times
   // if we get here its likely because the index could not be retrieved, or even built

app/web/src/newhotness/nav/NavbarPanelCenter.vue

@@ -2,7 +2,7 @@
   <div
     class="flex flex-row flex-none items-center h-full justify-center place-items-center mx-auto overflow-hidden"
   >
-    <template v-if="connected">
+    <template v-if="isConnected">
       <NavbarButton
         tooltipText="Compose"
         icon="grid"
@@ -65,10 +65,17 @@ import { computed } from "vue";
 import { useRoute } from "vue-router";
 import { themeClasses, Icon } from "@si/vue-lib/design-system";
 import NavbarButton from "@/components/layout/navbar/NavbarButton.vue";
+import { useFeatureFlagsStore } from "@/store/feature_flags.store";
 import { useContext } from "../logic_composables/context";
 
 const ctx = useContext();
 const route = useRoute();
+const ffStore = useFeatureFlagsStore();
+
+const isConnected = computed(() => {
+  if (ffStore.SHOW_WS_DISCONNECT) return props.connected;
+  return true;
+});
 
 const props = defineProps<{
   workspaceId: string;

app/web/src/newhotness/util.ts

@@ -1,4 +1,3 @@
-import * as _ from "lodash-es";
 import { Ref, unref } from "vue";
 import {
   BRAND_COLOR_FILTER_HEX_CODES,
@@ -123,51 +122,6 @@ export const openWorkspaceMigrationDocumentation = () => {
   );
 };
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-type AnyFn = (...args: any[]) => any;
-
-export interface MemoizeDebouncedFn<F extends AnyFn>
-  extends _.DebouncedFunc<F> {
-  (...args: Parameters<F>): ReturnType<F> | undefined;
-  flush: (...args: Parameters<F>) => ReturnType<F> | undefined;
-  cancel: (...args: Parameters<F>) => void;
-}
-/**
- * Debounce based on args to the fn
- */
-export function memoizeDebounce<F extends AnyFn>(
-  func: F,
-  wait = 0,
-  options: _.DebounceSettings = {},
-  resolver?: (...args: Parameters<F>) => unknown,
-): MemoizeDebouncedFn<F> {
-  const dbMemo = _.memoize<(...args: Parameters<F>) => _.DebouncedFunc<F>>(
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    (..._args: Parameters<F>) => _.debounce(func, wait, options),
-    resolver,
-  );
-
-  function wrappedFn(
-    this: MemoizeDebouncedFn<F>,
-    ...args: Parameters<F>
-  ): ReturnType<F> | undefined {
-    return dbMemo(...args)(...args);
-  }
-
-  const flush: MemoizeDebouncedFn<F>["flush"] = (...args) => {
-    return dbMemo(...args).flush();
-  };
-
-  const cancel: MemoizeDebouncedFn<F>["cancel"] = (...args) => {
-    return dbMemo(...args).cancel();
-  };
-
-  wrappedFn.flush = flush;
-  wrappedFn.cancel = cancel;
-
-  return wrappedFn;
-}
-
 /**
  * Escapes a string segment for use in a JSON Pointer (RFC 6901).
  * JSON Pointers use '~' as an escape character:

app/web/src/store/feature_flags.store.ts

@@ -17,6 +17,7 @@ const USER_FLAG_MAPPING = {
   SQLITE_TOOLS: "sqlite-tools",
   COMPONENT_HISTORY_FUNCS: "component-history-funcs",
   GOOGLE_CLOUD_UI: "google-cloud-ui",
+  SHOW_WS_DISCONNECT: "show-ws-disconnect",
 } as const;
 const WORKSPACE_FLAG_MAPPING: Record<string, string> = {
   // STORE_FLAG_NAME: "posthogFlagName",

app/web/src/store/realtime/heimdall.ts

@@ -43,7 +43,7 @@ import { DefaultMap } from "@/utils/defaultmap";
 import * as rainbow from "@/newhotness/logic_composables/rainbow_counter";
 import { sdfApiInstance as sdf } from "@/store/apis.web";
 import { WorkspaceMetadata, WorkspacePk } from "@/api/sdf/dal/workspace";
-import { memoizeDebounce } from "@/newhotness/util";
+import { memoizeDebounce } from "@/workers/utils";
 import {
   cachedAppEmitter,
   SHOW_CACHED_APP_NOTIFICATION_EVENT,
@@ -196,28 +196,32 @@ export const init = async (
 
   // If both tabs are refreshed at the same time, this can falsely indicate that
   // a tab has the lock, but that tab has actually been refreshed just *after*
-  // this call, so *we* now have the lock.  adding 2.5 second timeout here
+  // this call, so *we* now have the lock.  adding 0.2 second timeout here
   // ensures that there is enough time for the lock to be resolved in a multitab
   // scenario before we begin cold start. (This only matters if 2+ tabs are
   // refreshed at more or less the same time, in the normal scenario we will
   // indicate lock acquisition via the broadcast channel)
   //
-  setTimeout(async () => {
-    if (!lockAcquired.value) {
-      await detectLockAcquiredByOtherTab();
-    }
-  }, 2000);
+  const p = new Promise<void>((resolve) => {
+    setTimeout(async () => {
+      if (!lockAcquired.value) {
+        await detectLockAcquiredByOtherTab();
+      }
+      resolve();
+    }, 200);
+  });
+  await p;
 
   setTimeout(async () => {
-    // If after 5 seconds total, we have not detected lock acquisition, try
+    // If after 2 seconds total, we have not detected lock acquisition, try
     // and force a leader election
     if (!lockAcquired.value) {
       await detectLockAcquiredByOtherTab();
       if (!lockAcquired.value) {
         forceLeaderElectionBroadcastChannel.postMessage(FORCE_LEADER_ELECTION);
       }
     }
-  }, 5000);
+  }, 2000);
 };
 
 export const initCompleted = computed(
@@ -323,19 +327,36 @@ const updateCache = (
 export const wsConnections = ref<Record<string, boolean>>({});
 export const _wsConnections = ref<Record<string, boolean>>({});
 
-const updateConnectionStatus: ConnStatusFn = (
+// set the connections from the shared web worker
+// when i am the only tab, this is a no-op
+// and the WS starting up will fill me in below
+// when i am a second tab without a db lock, i wont get any new WS
+// so i need to learn what i am connected to
+db.getConnections().then((conns) => {
+  Object.entries(conns).forEach(([k, v]) => {
+    _wsConnections.value[k] = v;
+    wsConnections.value[k] = v;
+  });
+});
+
+const updateConnectionStatus: ConnStatusFn = async (
   workspaceId: WorkspacePk,
   connected: boolean,
   noBroadcast?: boolean,
 ) => {
   _wsConnections.value[workspaceId] = connected;
 
-  // prevent blips in the UI
+  // always update to connected
+  if (connected) wsConnections.value[workspaceId] = connected;
+
+  await db.setConnections({ ..._wsConnections.value });
+
+  // prevent blips in the UI from quick disconnect -> connect flips
   setTimeout(() => {
     Object.entries(_wsConnections.value).forEach(([k, v]) => {
       wsConnections.value[k] = v;
     });
-  }, 7 * 1000);
+  }, 3 * 1000);
 
   if (!noBroadcast) {
     db.broadcastMessage({
@@ -400,12 +421,16 @@ const atomUpdated: UpdateFn = (
 };
 
 export const indexFailures = reactive(new Set<string>());
+export const indexTouches = reactive(new Map<string, number>());
 
 const lobbyExit: LobbyExitFn = async (
   workspaceId: WorkspacePk,
   changeSetId: ChangeSetId,
   noBroadcast?: boolean,
 ) => {
+  // keep track of each change set's index touch
+  indexTouches.set(changeSetId, Date.now());
+
   // Only navigate away from lobby if user is currently in the lobby
   // for this change set
   if (muspelheimStatuses.value[changeSetId] === true) {
@@ -827,7 +852,7 @@ export const muspelheim = async (workspaceId: WorkspacePk, force?: boolean) => {
     muspelheimStatuses.value[changeSet.id] = false;
   }
 
-  await niflheim(workspaceId, baseChangeSetId, force);
+  await niflheim(workspaceId, baseChangeSetId, force, true);
 
   niflheimQueue.add(async () => {
     await vanaheim(workspaceId);
@@ -838,7 +863,7 @@ export const muspelheim = async (workspaceId: WorkspacePk, force?: boolean) => {
     }
 
     const run = async () => {
-      await niflheim(workspaceId, changeSet.id, force);
+      await niflheim(workspaceId, changeSet.id, force, true);
     };
     niflheimQueue.add(run);
   }
@@ -857,12 +882,24 @@ export const registerBearerToken = async (
   await db.setBearer(workspaceId, bearerToken);
 };
 
+export const syncAtoms = async (
+  workspaceId: WorkspacePk,
+  changeSetId: ChangeSetId,
+): Promise<void> => {
+  await waitForInitCompletion();
+  // eslint-disable-next-line no-console
+  await db.syncAtoms(workspaceId, changeSetId);
+};
+
 // cold start
+const FIVE_MIN = 1000 * 60 * 5;
+const buildingRetryIntervals = {} as Record<string, NodeJS.Timeout>;
 export const niflheim = async (
   workspaceId: WorkspacePk,
   changeSetId: ChangeSetId,
   force = false,
   lobbyOnFailure = true,
+  retryOnBuilding = false,
 ): Promise<-1 | 0 | 1> => {
   await waitForInitCompletion();
   const start = performance.now();
@@ -880,16 +917,41 @@ export const niflheim = async (
     // Index is being rebuilt and is not ready yet.
     if (success === 0 && lobbyOnFailure) {
       muspelheimStatuses.value[changeSetId] = false;
+      // there is an outside chance that the system will fail to send a wsevent
+      // (incident, poor timing of a deploy, etc etc)
+      // to the web worker that tells it the index has been built successfully
+      // so we have a backup cold start interval here
+      // to prevent a user from being stuck in the lobby perpetually
+      if (retryOnBuilding) {
+        const interval = setInterval(async () => {
+          const success = await niflheim(
+            workspaceId,
+            changeSetId,
+            force,
+            lobbyOnFailure,
+            false,
+          );
+          if (success === 1) {
+            const interval = buildingRetryIntervals[changeSetId];
+            clearInterval(interval);
+          }
+        }, FIVE_MIN);
+        buildingRetryIntervals[changeSetId] = interval;
+      }
     } else if (success === 1 || success === -1) {
       // add a failure state for this change set
       if (success === -1) indexFailures.add(changeSetId);
       // we could not retrieve the index, because it failed to build for various reasons
       // so we shouldn't block them in the lobby b/c this is never going to load.
       muspelheimStatuses.value[changeSetId] = true;
     }
+    const interval = buildingRetryIntervals[changeSetId];
+    if (success === 1 && interval) clearInterval(interval);
     return success;
   }
 
+  const interval = buildingRetryIntervals[changeSetId];
+  if (interval) clearInterval(interval);
   return 1;
 };
 

app/web/src/workers/shared_webworker.ts

@@ -120,6 +120,7 @@ async function withLeader<R>(
 }
 
 const clientInterest: Record<string, number> = {};
+const connectionStatus: Record<string, boolean> = {};
 
 const dbInterface: SharedDBInterface = {
   async broadcastMessage(message: BroadcastMessage) {
@@ -236,6 +237,16 @@ const dbInterface: SharedDBInterface = {
     });
   },
 
+  async setConnections(connections: Record<string, boolean>) {
+    Object.entries(connections).forEach(([k, v]) => {
+      connectionStatus[k] = v;
+    });
+  },
+
+  async getConnections() {
+    return connectionStatus;
+  },
+
   async initSocket(workspaceId: string): Promise<void> {
     await withLeader(async (remote) => await remote.initSocket(workspaceId));
   },
@@ -434,6 +445,12 @@ const dbInterface: SharedDBInterface = {
     );
   },
 
+  async syncAtoms(workspaceId: string, changeSetId: string): Promise<void> {
+    return await withLeader(
+      async (remote) => await remote.syncAtoms(workspaceId, changeSetId),
+    );
+  },
+
   async vanaheim(workspaceId: string): Promise<boolean> {
     return await withLeader(
       async (remote) => await remote.vanaheim(workspaceId),