feat: store library to IndexedDB & support storage adapters (#7655)

excalidraw-app/App.tsx

@@ -30,7 +30,6 @@ import {
 } from "../packages/excalidraw/index";
 import {
   AppState,
-  LibraryItems,
   ExcalidrawImperativeAPI,
   BinaryFiles,
   ExcalidrawInitialDataState,
@@ -64,7 +63,6 @@ import {
   loadScene,
 } from "./data";
 import {
-  getLibraryItemsFromStorage,
   importFromLocalStorage,
   importUsernameFromLocalStorage,
 } from "./data/localStorage";
@@ -82,7 +80,11 @@ import { updateStaleImageStatuses } from "./data/FileManager";
 import { newElementWith } from "../packages/excalidraw/element/mutateElement";
 import { isInitializedImageElement } from "../packages/excalidraw/element/typeChecks";
 import { loadFilesFromFirebase } from "./data/firebase";
-import { LocalData } from "./data/LocalData";
+import {
+  LibraryIndexedDBAdapter,
+  LibraryLocalStorageMigrationAdapter,
+  LocalData,
+} from "./data/LocalData";
 import { isBrowserStorageStateNewer } from "./data/tabSync";
 import clsx from "clsx";
 import { reconcileElements } from "./collab/reconciliation";
@@ -315,7 +317,9 @@ const ExcalidrawWrapper = () => {
 
   useHandleLibrary({
     excalidrawAPI,
-    getInitialLibraryItems: getLibraryItemsFromStorage,
+    adapter: LibraryIndexedDBAdapter,
+    // TODO maybe remove this in several months (shipped: 24-02-07)
+    migrationAdapter: LibraryLocalStorageMigrationAdapter,
   });
 
   useEffect(() => {
@@ -445,8 +449,12 @@ const ExcalidrawWrapper = () => {
           excalidrawAPI.updateScene({
             ...localDataState,
           });
-          excalidrawAPI.updateLibrary({
+          LibraryIndexedDBAdapter.load().then((data) => {
-            libraryItems: getLibraryItemsFromStorage(),
+            if (data) {
+              excalidrawAPI.updateLibrary({
+                libraryItems: data.libraryItems,
+              });
+            }
           });
           collabAPI?.setUsername(username || "");
         }
@@ -658,15 +666,6 @@ const ExcalidrawWrapper = () => {
     );
   };
 
-  const onLibraryChange = async (items: LibraryItems) => {
-    if (!items.length) {
-      localStorage.removeItem(STORAGE_KEYS.LOCAL_STORAGE_LIBRARY);
-      return;
-    }
-    const serializedItems = JSON.stringify(items);
-    localStorage.setItem(STORAGE_KEYS.LOCAL_STORAGE_LIBRARY, serializedItems);
-  };
-
   const isOffline = useAtomValue(isOfflineAtom);
 
   const onCollabDialogOpen = useCallback(
@@ -742,7 +741,6 @@ const ExcalidrawWrapper = () => {
         renderCustomStats={renderCustomStats}
         detectScroll={false}
         handleKeyboardGlobally={true}
-        onLibraryChange={onLibraryChange}
         autoFocus={true}
         theme={theme}
         renderTopRightUI={(isMobile) => {

excalidraw-app/app_constants.ts

@@ -39,10 +39,14 @@ export const STORAGE_KEYS = {
   LOCAL_STORAGE_ELEMENTS: "excalidraw",
   LOCAL_STORAGE_APP_STATE: "excalidraw-state",
   LOCAL_STORAGE_COLLAB: "excalidraw-collab",
-  LOCAL_STORAGE_LIBRARY: "excalidraw-library",
   LOCAL_STORAGE_THEME: "excalidraw-theme",
   VERSION_DATA_STATE: "version-dataState",
   VERSION_FILES: "version-files",
+
+  IDB_LIBRARY: "excalidraw-library",
+
+  // do not use apart from migrations
+  __LEGACY_LOCAL_STORAGE_LIBRARY: "excalidraw-library",
 } as const;
 
 export const COOKIES = {

excalidraw-app/data/LocalData.ts

@@ -10,8 +10,18 @@
  *   (localStorage, indexedDB).
  */
 
-import { createStore, entries, del, getMany, set, setMany } from "idb-keyval";
+import {
+  createStore,
+  entries,
+  del,
+  getMany,
+  set,
+  setMany,
+  get,
+} from "idb-keyval";
 import { clearAppStateForLocalStorage } from "../../packages/excalidraw/appState";
+import { LibraryPersistedData } from "../../packages/excalidraw/data/library";
+import { ImportedDataState } from "../../packages/excalidraw/data/types";
 import { clearElementsForLocalStorage } from "../../packages/excalidraw/element";
 import {
   ExcalidrawElement,
@@ -22,6 +32,7 @@ import {
   BinaryFileData,
   BinaryFiles,
 } from "../../packages/excalidraw/types";
+import { MaybePromise } from "../../packages/excalidraw/utility-types";
 import { debounce } from "../../packages/excalidraw/utils";
 import { SAVE_TO_LOCAL_STORAGE_TIMEOUT, STORAGE_KEYS } from "../app_constants";
 import { FileManager } from "./FileManager";
@@ -183,3 +194,52 @@ export class LocalData {
     },
   });
 }
+export class LibraryIndexedDBAdapter {
+  /** IndexedDB database and store name */
+  private static idb_name = STORAGE_KEYS.IDB_LIBRARY;
+  /** library data store key */
+  private static key = "libraryData";
+
+  private static store = createStore(
+    `${LibraryIndexedDBAdapter.idb_name}-db`,
+    `${LibraryIndexedDBAdapter.idb_name}-store`,
+  );
+
+  static async load() {
+    const IDBData = await get<LibraryPersistedData>(
+      LibraryIndexedDBAdapter.key,
+      LibraryIndexedDBAdapter.store,
+    );
+
+    return IDBData || null;
+  }
+
+  static save(data: LibraryPersistedData): MaybePromise<void> {
+    return set(
+      LibraryIndexedDBAdapter.key,
+      data,
+      LibraryIndexedDBAdapter.store,
+    );
+  }
+}
+
+/** LS Adapter used only for migrating LS library data
+ * to indexedDB */
+export class LibraryLocalStorageMigrationAdapter {
+  static load() {
+    const LSData = localStorage.getItem(
+      STORAGE_KEYS.__LEGACY_LOCAL_STORAGE_LIBRARY,
+    );
+    if (LSData != null) {
+      const libraryItems: ImportedDataState["libraryItems"] =
+        JSON.parse(LSData);
+      if (libraryItems) {
+        return { libraryItems };
+      }
+    }
+    return null;
+  }
+  static clear() {
+    localStorage.removeItem(STORAGE_KEYS.__LEGACY_LOCAL_STORAGE_LIBRARY);
+  }
+}

excalidraw-app/data/localStorage.ts

@@ -6,7 +6,6 @@ import {
 } from "../../packages/excalidraw/appState";
 import { clearElementsForLocalStorage } from "../../packages/excalidraw/element";
 import { STORAGE_KEYS } from "../app_constants";
-import { ImportedDataState } from "../../packages/excalidraw/data/types";
 
 export const saveUsernameToLocalStorage = (username: string) => {
   try {
@@ -88,28 +87,13 @@ export const getTotalStorageSize = () => {
   try {
     const appState = localStorage.getItem(STORAGE_KEYS.LOCAL_STORAGE_APP_STATE);
     const collab = localStorage.getItem(STORAGE_KEYS.LOCAL_STORAGE_COLLAB);
-    const library = localStorage.getItem(STORAGE_KEYS.LOCAL_STORAGE_LIBRARY);
 
     const appStateSize = appState?.length || 0;
     const collabSize = collab?.length || 0;
-    const librarySize = library?.length || 0;
 
-    return appStateSize + collabSize + librarySize + getElementsStorageSize();
+    return appStateSize + collabSize + getElementsStorageSize();
   } catch (error: any) {
     console.error(error);
     return 0;
   }
 };
-
-export const getLibraryItemsFromStorage = () => {
-  try {
-    const libraryItems: ImportedDataState["libraryItems"] = JSON.parse(
-      localStorage.getItem(STORAGE_KEYS.LOCAL_STORAGE_LIBRARY) as string,
-    );
-
-    return libraryItems || [];
-  } catch (error) {
-    console.error(error);
-    return [];
-  }
-};

packages/excalidraw/CHANGELOG.md

@@ -15,6 +15,10 @@ Please add the latest change on the top under the correct section.
 
 ### Features
 
+- Add `useHandleLibrary`'s `opts.adapter` as the new recommended pattern to handle library initialization and persistence on library updates. [#7655](https://github.com/excalidraw/excalidraw/pull/7655)
+- Add `useHandleLibrary`'s `opts.migrationAdapter` adapter to handle library migration during init, when migrating from one data store to another (e.g. from LocalStorage to IndexedDB). [#7655](https://github.com/excalidraw/excalidraw/pull/7655)
+- Soft-deprecate `useHandleLibrary`'s `opts.getInitialLibraryItems` in favor of `opts.adapter`. [#7655](https://github.com/excalidraw/excalidraw/pull/7655)
+
 - Add `onPointerUp` prop [#7638](https://github.com/excalidraw/excalidraw/pull/7638).
 
 - Expose `getVisibleSceneBounds` helper to get scene bounds of visible canvas area. [#7450](https://github.com/excalidraw/excalidraw/pull/7450)

packages/excalidraw/data/library.ts

@@ -4,6 +4,7 @@ import {
   LibraryItem,
   ExcalidrawImperativeAPI,
   LibraryItemsSource,
+  LibraryItems_anyVersion,
 } from "../types";
 import { restoreLibraryItems } from "./restore";
 import type App from "../components/App";
@@ -23,13 +24,72 @@ import {
   LIBRARY_SIDEBAR_TAB,
 } from "../constants";
 import { libraryItemSvgsCache } from "../hooks/useLibraryItemSvg";
-import { cloneJSON } from "../utils";
+import {
+  arrayToMap,
+  cloneJSON,
+  preventUnload,
+  promiseTry,
+  resolvablePromise,
+} from "../utils";
+import { MaybePromise } from "../utility-types";
+import { Emitter } from "../emitter";
+import { Queue } from "../queue";
+import { hashElementsVersion, hashString } from "../element";
+
+type LibraryUpdate = {
+  /** deleted library items since last onLibraryChange event */
+  deletedItems: Map<LibraryItem["id"], LibraryItem>;
+  /** newly added items in the library */
+  addedItems: Map<LibraryItem["id"], LibraryItem>;
+};
+
+// an object so that we can later add more properties to it without breaking,
+// such as schema version
+export type LibraryPersistedData = { libraryItems: LibraryItems };
+
+const onLibraryUpdateEmitter = new Emitter<
+  [update: LibraryUpdate, libraryItems: LibraryItems]
+>();
+
+export interface LibraryPersistenceAdapter {
+  /**
+   * Should load data that were previously saved into the database using the
+   * `save` method. Should throw if saving fails.
+   *
+   * Will be used internally in multiple places, such as during save to
+   * in order to reconcile changes with latest store data.
+   */
+  load(metadata: {
+    /**
+     * Priority 1 indicates we're loading latest data with intent
+     * to reconcile with before save.
+     * Priority 2 indicates we're loading for read-only purposes, so
+     * host app can implement more aggressive caching strategy.
+     */
+    priority: 1 | 2;
+  }): MaybePromise<{ libraryItems: LibraryItems_anyVersion } | null>;
+  /** Should persist to the database as is (do no change the data structure). */
+  save(libraryData: LibraryPersistedData): MaybePromise<void>;
+}
+
+export interface LibraryMigrationAdapter {
+  /**
+   * loads data from legacy data source. Returns `null` if no data is
+   * to be migrated.
+   */
+  load(): MaybePromise<{ libraryItems: LibraryItems_anyVersion } | null>;
+
+  /** clears entire storage afterwards */
+  clear(): MaybePromise<void>;
+}
 
 export const libraryItemsAtom = atom<{
   status: "loading" | "loaded";
+  /** indicates whether library is initialized with library items (has gone
+   * through at least one update). Used in UI. Specific to this atom only. */
   isInitialized: boolean;
   libraryItems: LibraryItems;
-}>({ status: "loaded", isInitialized: true, libraryItems: [] });
+}>({ status: "loaded", isInitialized: false, libraryItems: [] });
 
 const cloneLibraryItems = (libraryItems: LibraryItems): LibraryItems =>
   cloneJSON(libraryItems);
@@ -74,12 +134,45 @@ export const mergeLibraryItems = (
   return [...newItems, ...localItems];
 };
 
+/**
+ * Returns { deletedItems, addedItems } maps of all added and deleted items
+ * since last onLibraryChange event.
+ *
+ * Host apps are recommended to diff with the latest state they have.
+ */
+const createLibraryUpdate = (
+  prevLibraryItems: LibraryItems,
+  nextLibraryItems: LibraryItems,
+): LibraryUpdate => {
+  const nextItemsMap = arrayToMap(nextLibraryItems);
+
+  const update: LibraryUpdate = {
+    deletedItems: new Map<LibraryItem["id"], LibraryItem>(),
+    addedItems: new Map<LibraryItem["id"], LibraryItem>(),
+  };
+
+  for (const item of prevLibraryItems) {
+    if (!nextItemsMap.has(item.id)) {
+      update.deletedItems.set(item.id, item);
+    }
+  }
+
+  const prevItemsMap = arrayToMap(prevLibraryItems);
+
+  for (const item of nextLibraryItems) {
+    if (!prevItemsMap.has(item.id)) {
+      update.addedItems.set(item.id, item);
+    }
+  }
+
+  return update;
+};
+
 class Library {
   /** latest libraryItems */
-  private lastLibraryItems: LibraryItems = [];
+  private currLibraryItems: LibraryItems = [];
-  /** indicates whether library is initialized with library items (has gone
+  /** snapshot of library items since last onLibraryChange call */
-   * though at least one update) */
+  private prevLibraryItems = cloneLibraryItems(this.currLibraryItems);
-  private isInitialized = false;
 
   private app: App;
 
@@ -95,21 +188,29 @@ class Library {
 
   private notifyListeners = () => {
     if (this.updateQueue.length > 0) {
-      jotaiStore.set(libraryItemsAtom, {
+      jotaiStore.set(libraryItemsAtom, (s) => ({
         status: "loading",
-        libraryItems: this.lastLibraryItems,
+        libraryItems: this.currLibraryItems,
-        isInitialized: this.isInitialized,
+        isInitialized: s.isInitialized,
-      });
+      }));
     } else {
-      this.isInitialized = true;
       jotaiStore.set(libraryItemsAtom, {
         status: "loaded",
-        libraryItems: this.lastLibraryItems,
+        libraryItems: this.currLibraryItems,
-        isInitialized: this.isInitialized,
+        isInitialized: true,
       });
       try {
-        this.app.props.onLibraryChange?.(
+        const prevLibraryItems = this.prevLibraryItems;
-          cloneLibraryItems(this.lastLibraryItems),
+        this.prevLibraryItems = cloneLibraryItems(this.currLibraryItems);
+
+        const nextLibraryItems = cloneLibraryItems(this.currLibraryItems);
+
+        this.app.props.onLibraryChange?.(nextLibraryItems);
+
+        // for internal use in `useHandleLibrary` hook
+        onLibraryUpdateEmitter.trigger(
+          createLibraryUpdate(prevLibraryItems, nextLibraryItems),
+          nextLibraryItems,
         );
       } catch (error) {
         console.error(error);
@@ -119,9 +220,8 @@ class Library {
 
   /** call on excalidraw instance unmount */
   destroy = () => {
-    this.isInitialized = false;
     this.updateQueue = [];
-    this.lastLibraryItems = [];
+    this.currLibraryItems = [];
     jotaiStore.set(libraryItemSvgsCache, new Map());
     // TODO uncomment after/if we make jotai store scoped to each excal instance
     // jotaiStore.set(libraryItemsAtom, {
@@ -142,14 +242,14 @@ class Library {
     return new Promise(async (resolve) => {
       try {
         const libraryItems = await (this.getLastUpdateTask() ||
-          this.lastLibraryItems);
+          this.currLibraryItems);
         if (this.updateQueue.length > 0) {
           resolve(this.getLatestLibrary());
         } else {
           resolve(cloneLibraryItems(libraryItems));
         }
       } catch (error) {
-        return resolve(this.lastLibraryItems);
+        return resolve(this.currLibraryItems);
       }
     });
   };
@@ -181,7 +281,7 @@ class Library {
         try {
           const source = await (typeof libraryItems === "function" &&
           !(libraryItems instanceof Blob)
-            ? libraryItems(this.lastLibraryItems)
+            ? libraryItems(this.currLibraryItems)
             : libraryItems);
 
           let nextItems;
@@ -207,7 +307,7 @@ class Library {
             }
 
             if (merge) {
-              resolve(mergeLibraryItems(this.lastLibraryItems, nextItems));
+              resolve(mergeLibraryItems(this.currLibraryItems, nextItems));
             } else {
               resolve(nextItems);
             }
@@ -244,12 +344,12 @@ class Library {
         await this.getLastUpdateTask();
 
         if (typeof libraryItems === "function") {
-          libraryItems = libraryItems(this.lastLibraryItems);
+          libraryItems = libraryItems(this.currLibraryItems);
         }
 
-        this.lastLibraryItems = cloneLibraryItems(await libraryItems);
+        this.currLibraryItems = cloneLibraryItems(await libraryItems);
 
-        resolve(this.lastLibraryItems);
+        resolve(this.currLibraryItems);
       } catch (error: any) {
         reject(error);
       }
@@ -257,7 +357,7 @@ class Library {
       .catch((error) => {
         if (error.name === "AbortError") {
           console.warn("Library update aborted by user");
-          return this.lastLibraryItems;
+          return this.currLibraryItems;
         }
         throw error;
       })
@@ -382,20 +482,165 @@ export const parseLibraryTokensFromUrl = () => {
   return libraryUrl ? { libraryUrl, idToken } : null;
 };
 
-export const useHandleLibrary = ({
+class AdapterTransaction {
-  excalidrawAPI,
+  static queue = new Queue();
-  getInitialLibraryItems,
+
-}: {
+  static async getLibraryItems(
-  excalidrawAPI: ExcalidrawImperativeAPI | null;
+    adapter: LibraryPersistenceAdapter,
-  getInitialLibraryItems?: () => LibraryItemsSource;
+    priority: 1 | 2,
-}) => {
+    _queue = true,
-  const getInitialLibraryRef = useRef(getInitialLibraryItems);
+  ): Promise<LibraryItems> {
+    const task = () =>
+      new Promise<LibraryItems>(async (resolve, reject) => {
+        try {
+          const data = await adapter.load({ priority });
+          resolve(restoreLibraryItems(data?.libraryItems || [], "published"));
+        } catch (error: any) {
+          reject(error);
+        }
+      });
+
+    if (_queue) {
+      return AdapterTransaction.queue.push(task);
+    }
+
+    return task();
+  }
+
+  static run = async <T>(
+    adapter: LibraryPersistenceAdapter,
+    fn: (transaction: AdapterTransaction) => Promise<T>,
+  ) => {
+    const transaction = new AdapterTransaction(adapter);
+    return AdapterTransaction.queue.push(() => fn(transaction));
+  };
+
+  // ------------------
+
+  private adapter: LibraryPersistenceAdapter;
+
+  constructor(adapter: LibraryPersistenceAdapter) {
+    this.adapter = adapter;
+  }
+
+  getLibraryItems(priority: 1 | 2) {
+    return AdapterTransaction.getLibraryItems(this.adapter, priority, false);
+  }
+}
+
+let lastSavedLibraryItemsHash = 0;
+let librarySaveCounter = 0;
+
+export const getLibraryItemsHash = (items: LibraryItems) => {
+  return hashString(
+    items
+      .map((item) => {
+        return `${item.id}:${hashElementsVersion(item.elements)}`;
+      })
+      .sort()
+      .join(),
+  );
+};
+
+const persistLibraryUpdate = async (
+  adapter: LibraryPersistenceAdapter,
+  update: LibraryUpdate,
+): Promise<LibraryItems> => {
+  try {
+    librarySaveCounter++;
+
+    return await AdapterTransaction.run(adapter, async (transaction) => {
+      const nextLibraryItemsMap = arrayToMap(
+        await transaction.getLibraryItems(1),
+      );
+
+      for (const [id] of update.deletedItems) {
+        nextLibraryItemsMap.delete(id);
+      }
+
+      const addedItems: LibraryItem[] = [];
+
+      // we want to merge current library items with the ones stored in the
+      // DB so that we don't lose any elements that for some reason aren't
+      // in the current editor library, which could happen when:
+      //
+      // 1. we haven't received an update deleting some elements
+      //    (in which case it's still better to keep them in the DB lest
+      //     it was due to a different reason)
+      // 2. we keep a single DB for all active editors, but the editors'
+      //    libraries aren't synced or there's a race conditions during
+      //    syncing
+      // 3. some other race condition, e.g. during init where emit updates
+      //    for partial updates (e.g. you install a 3rd party library and
+      //    init from DB only after — we emit events for both updates)
+      for (const [id, item] of update.addedItems) {
+        if (nextLibraryItemsMap.has(id)) {
+          // replace item with latest version
+          // TODO we could prefer the newer item instead
+          nextLibraryItemsMap.set(id, item);
+        } else {
+          // we want to prepend the new items with the ones that are already
+          // in DB to preserve the ordering we do in editor (newly added
+          // items are added to the beginning)
+          addedItems.push(item);
+        }
+      }
+
+      const nextLibraryItems = addedItems.concat(
+        Array.from(nextLibraryItemsMap.values()),
+      );
+
+      const version = getLibraryItemsHash(nextLibraryItems);
+
+      if (version !== lastSavedLibraryItemsHash) {
+        await adapter.save({ libraryItems: nextLibraryItems });
+      }
+
+      lastSavedLibraryItemsHash = version;
+
+      return nextLibraryItems;
+    });
+  } finally {
+    librarySaveCounter--;
+  }
+};
+
+export const useHandleLibrary = (
+  opts: {
+    excalidrawAPI: ExcalidrawImperativeAPI | null;
+  } & (
+    | {
+        /** @deprecated we recommend using `opts.adapter` instead */
+        getInitialLibraryItems?: () => MaybePromise<LibraryItemsSource>;
+      }
+    | {
+        adapter: LibraryPersistenceAdapter;
+        /**
+         * Adapter that takes care of loading data from legacy data store.
+         * Supply this if you want to migrate data on initial load from legacy
+         * data store.
+         *
+         * Can be a different LibraryPersistenceAdapter.
+         */
+        migrationAdapter?: LibraryMigrationAdapter;
+      }
+  ),
+) => {
+  const { excalidrawAPI } = opts;
+
+  const optsRef = useRef(opts);
+  optsRef.current = opts;
+
+  const isLibraryLoadedRef = useRef(false);
 
   useEffect(() => {
     if (!excalidrawAPI) {
       return;
     }
 
+    // reset on editor remount (excalidrawAPI changed)
+    isLibraryLoadedRef.current = false;
+
     const importLibraryFromURL = async ({
       libraryUrl,
       idToken,
@@ -463,23 +708,209 @@ export const useHandleLibrary = ({
     };
 
     // -------------------------------------------------------------------------
-    // ------ init load --------------------------------------------------------
+    // ---------------------------------- init ---------------------------------
-    if (getInitialLibraryRef.current) {
+    // -------------------------------------------------------------------------
-      excalidrawAPI.updateLibrary({
-        libraryItems: getInitialLibraryRef.current(),
-      });
-    }
 
     const libraryUrlTokens = parseLibraryTokensFromUrl();
 
     if (libraryUrlTokens) {
       importLibraryFromURL(libraryUrlTokens);
     }
+
+    // ------ (A) init load (legacy) -------------------------------------------
+    if (
+      "getInitialLibraryItems" in optsRef.current &&
+      optsRef.current.getInitialLibraryItems
+    ) {
+      console.warn(
+        "useHandleLibrar `opts.getInitialLibraryItems` is deprecated. Use `opts.adapter` instead.",
+      );
+
+      Promise.resolve(optsRef.current.getInitialLibraryItems())
+        .then((libraryItems) => {
+          excalidrawAPI.updateLibrary({
+            libraryItems,
+            // merge with current library items because we may have already
+            // populated it (e.g. by installing 3rd party library which can
+            // happen before the DB data is loaded)
+            merge: true,
+          });
+        })
+        .catch((error: any) => {
+          console.error(
+            `UseHandeLibrary getInitialLibraryItems failed: ${error?.message}`,
+          );
+        });
+    }
+
+    // -------------------------------------------------------------------------
     // --------------------------------------------------------- init load -----
+    // -------------------------------------------------------------------------
+
+    // ------ (B) data source adapter ------------------------------------------
+
+    if ("adapter" in optsRef.current && optsRef.current.adapter) {
+      const adapter = optsRef.current.adapter;
+      const migrationAdapter = optsRef.current.migrationAdapter;
+
+      const initDataPromise = resolvablePromise<LibraryItems | null>();
+
+      // migrate from old data source if needed
+      // (note, if `migrate` function is defined, we always migrate even
+      //  if the data has already been migrated. In that case it'll be a no-op,
+      //  though with several unnecessary steps — we will still load latest
+      //  DB data during the `persistLibraryChange()` step)
+      // -----------------------------------------------------------------------
+      if (migrationAdapter) {
+        initDataPromise.resolve(
+          promiseTry(migrationAdapter.load)
+            .then(async (libraryData) => {
+              try {
+                // if no library data to migrate, assume no migration needed
+                // and skip persisting to new data store, as well as well
+                // clearing the old store via `migrationAdapter.clear()`
+                if (!libraryData) {
+                  return AdapterTransaction.getLibraryItems(adapter, 2);
+                }
+
+                // we don't queue this operation because it's running inside
+                // a promise that's running inside Library update queue itself
+                const nextItems = await persistLibraryUpdate(
+                  adapter,
+                  createLibraryUpdate(
+                    [],
+                    restoreLibraryItems(
+                      libraryData.libraryItems || [],
+                      "published",
+                    ),
+                  ),
+                );
+                try {
+                  await migrationAdapter.clear();
+                } catch (error: any) {
+                  console.error(
+                    `couldn't delete legacy library data: ${error.message}`,
+                  );
+                }
+                // migration suceeded, load migrated data
+                return nextItems;
+              } catch (error: any) {
+                console.error(
+                  `couldn't migrate legacy library data: ${error.message}`,
+                );
+                // migration failed, load empty library
+                return [];
+              }
+            })
+            // errors caught during `migrationAdapter.load()`
+            .catch((error: any) => {
+              console.error(`error during library migration: ${error.message}`);
+              // as a default, load latest library from current data source
+              return AdapterTransaction.getLibraryItems(adapter, 2);
+            }),
+        );
+      } else {
+        initDataPromise.resolve(
+          promiseTry(AdapterTransaction.getLibraryItems, adapter, 2),
+        );
+      }
+
+      // load initial (or migrated) library
+      excalidrawAPI
+        .updateLibrary({
+          libraryItems: initDataPromise.then((libraryItems) => {
+            const _libraryItems = libraryItems || [];
+            lastSavedLibraryItemsHash = getLibraryItemsHash(_libraryItems);
+            return _libraryItems;
+          }),
+          // merge with current library items because we may have already
+          // populated it (e.g. by installing 3rd party library which can
+          // happen before the DB data is loaded)
+          merge: true,
+        })
+        .finally(() => {
+          isLibraryLoadedRef.current = true;
+        });
+    }
+    // ---------------------------------------------- data source datapter -----
 
     window.addEventListener(EVENT.HASHCHANGE, onHashChange);
     return () => {
       window.removeEventListener(EVENT.HASHCHANGE, onHashChange);
     };
-  }, [excalidrawAPI]);
+  }, [
+    // important this useEffect only depends on excalidrawAPI so it only reruns
+    // on editor remounts (the excalidrawAPI changes)
+    excalidrawAPI,
+  ]);
+
+  // This effect is run without excalidrawAPI dependency so that host apps
+  // can run this hook outside of an active editor instance and the library
+  // update queue/loop survives editor remounts
+  //
+  // This effect is still only meant to be run if host apps supply an persitence
+  // adapter. If we don't have access to it, it the update listener doesn't
+  // do anything.
+  useEffect(
+    () => {
+      // on update, merge with current library items and persist
+      // -----------------------------------------------------------------------
+      const unsubOnLibraryUpdate = onLibraryUpdateEmitter.on(
+        async (update, nextLibraryItems) => {
+          const isLoaded = isLibraryLoadedRef.current;
+          // we want to operate with the latest adapter, but we don't want this
+          // effect to rerun on every adapter change in case host apps' adapter
+          // isn't stable
+          const adapter =
+            ("adapter" in optsRef.current && optsRef.current.adapter) || null;
+          try {
+            if (adapter) {
+              if (
+                // if nextLibraryItems hash identical to previously saved hash,
+                // exit early, even if actual upstream state ends up being
+                // different (e.g. has more data than we have locally), as it'd
+                // be low-impact scenario.
+                lastSavedLibraryItemsHash !==
+                getLibraryItemsHash(nextLibraryItems)
+              ) {
+                await persistLibraryUpdate(adapter, update);
+              }
+            }
+          } catch (error: any) {
+            console.error(
+              `couldn't persist library update: ${error.message}`,
+              update,
+            );
+
+            // currently we only show error if an editor is loaded
+            if (isLoaded && optsRef.current.excalidrawAPI) {
+              optsRef.current.excalidrawAPI.updateScene({
+                appState: {
+                  errorMessage: t("errors.saveLibraryError"),
+                },
+              });
+            }
+          }
+        },
+      );
+
+      const onUnload = (event: Event) => {
+        if (librarySaveCounter) {
+          preventUnload(event);
+        }
+      };
+
+      window.addEventListener(EVENT.BEFORE_UNLOAD, onUnload);
+
+      return () => {
+        window.removeEventListener(EVENT.BEFORE_UNLOAD, onUnload);
+        unsubOnLibraryUpdate();
+        lastSavedLibraryItemsHash = 0;
+        librarySaveCounter = 0;
+      };
+    },
+    [
+      // this effect must not have any deps so it doesn't rerun
+    ],
+  );
 };

packages/excalidraw/element/index.ts

@@ -60,9 +60,36 @@ export {
 } from "./sizeHelpers";
 export { showSelectedShapeActions } from "./showSelectedShapeActions";
 
+/**
+ * @deprecated unsafe, use hashElementsVersion instead
+ */
 export const getSceneVersion = (elements: readonly ExcalidrawElement[]) =>
   elements.reduce((acc, el) => acc + el.version, 0);
 
+/**
+ * Hashes elements' versionNonce (using djb2 algo). Order of elements matters.
+ */
+export const hashElementsVersion = (
+  elements: readonly ExcalidrawElement[],
+): number => {
+  let hash = 5381;
+  for (let i = 0; i < elements.length; i++) {
+    hash = (hash << 5) + hash + elements[i].versionNonce;
+  }
+  return hash >>> 0; // Ensure unsigned 32-bit integer
+};
+
+// string hash function (using djb2). Not cryptographically secure, use only
+// for versioning and such.
+export const hashString = (s: string): number => {
+  let hash: number = 5381;
+  for (let i = 0; i < s.length; i++) {
+    const char: number = s.charCodeAt(i);
+    hash = (hash << 5) + hash + char;
+  }
+  return hash >>> 0; // Ensure unsigned 32-bit integer
+};
+
 export const getVisibleElements = (elements: readonly ExcalidrawElement[]) =>
   elements.filter(
     (el) => !el.isDeleted && !isInvisiblySmallElement(el),

packages/excalidraw/index.tsx

@@ -207,6 +207,8 @@ Excalidraw.displayName = "Excalidraw";
 
 export {
   getSceneVersion,
+  hashElementsVersion,
+  hashString,
   isInvisiblySmallElement,
   getNonDeletedElements,
 } from "./element";
@@ -232,7 +234,7 @@ export {
   loadLibraryFromBlob,
 } from "./data/blob";
 export { getFreeDrawSvgPath } from "./renderer/renderElement";
-export { mergeLibraryItems } from "./data/library";
+export { mergeLibraryItems, getLibraryItemsHash } from "./data/library";
 export { isLinearElement } from "./element/typeChecks";
 
 export { FONT_FAMILY, THEME, MIME_TYPES, ROUNDNESS } from "./constants";

packages/excalidraw/locales/en.json

@@ -216,6 +216,7 @@
     "failedToFetchImage": "Failed to fetch image.",
     "cannotResolveCollabServer": "Couldn't connect to the collab server. Please reload the page and try again.",
     "importLibraryError": "Couldn't load library",
+    "saveLibraryError": "Couldn't save library to storage. Please save your library to a file locally to make sure you don't lose changes.",
     "collabSaveFailed": "Couldn't save to the backend database. If problems persist, you should save your file locally to ensure you don't lose your work.",
     "collabSaveFailed_sizeExceeded": "Couldn't save to the backend database, the canvas seems to be too big. You should save the file locally to ensure you don't lose your work.",
     "imageToolNotSupported": "Images are disabled.",

packages/excalidraw/queue.test.ts

@@ -0,0 +1,62 @@
+import { Queue } from "./queue";
+
+describe("Queue", () => {
+  const calls: any[] = [];
+
+  const createJobFactory =
+    <T>(
+      // for purpose of this test, Error object will become a rejection value
+      resolutionOrRejectionValue: T,
+      ms = 1,
+    ) =>
+    () => {
+      return new Promise<T>((resolve, reject) => {
+        setTimeout(() => {
+          if (resolutionOrRejectionValue instanceof Error) {
+            reject(resolutionOrRejectionValue);
+          } else {
+            resolve(resolutionOrRejectionValue);
+          }
+        }, ms);
+      }).then((x) => {
+        calls.push(x);
+        return x;
+      });
+    };
+
+  beforeEach(() => {
+    calls.length = 0;
+  });
+
+  it("should await and resolve values in order of enqueueing", async () => {
+    const queue = new Queue();
+
+    const p1 = queue.push(createJobFactory("A", 50));
+    const p2 = queue.push(createJobFactory("B"));
+    const p3 = queue.push(createJobFactory("C"));
+
+    expect(await p3).toBe("C");
+    expect(await p2).toBe("B");
+    expect(await p1).toBe("A");
+
+    expect(calls).toEqual(["A", "B", "C"]);
+  });
+
+  it("should reject a job if it throws, and not affect other jobs", async () => {
+    const queue = new Queue();
+
+    const err = new Error("B");
+
+    queue.push(createJobFactory("A", 50));
+    const p2 = queue.push(createJobFactory(err));
+    const p3 = queue.push(createJobFactory("C"));
+
+    const p2err = p2.catch((err) => err);
+
+    await p3;
+
+    expect(await p2err).toBe(err);
+
+    expect(calls).toEqual(["A", "C"]);
+  });
+});

packages/excalidraw/queue.ts

@@ -0,0 +1,45 @@
+import { MaybePromise } from "./utility-types";
+import { promiseTry, ResolvablePromise, resolvablePromise } from "./utils";
+
+type Job<T, TArgs extends unknown[]> = (...args: TArgs) => MaybePromise<T>;
+
+type QueueJob<T, TArgs extends unknown[]> = {
+  jobFactory: Job<T, TArgs>;
+  promise: ResolvablePromise<T>;
+  args: TArgs;
+};
+
+export class Queue {
+  private jobs: QueueJob<any, any[]>[] = [];
+  private running = false;
+
+  private tick() {
+    if (this.running) {
+      return;
+    }
+    const job = this.jobs.shift();
+    if (job) {
+      this.running = true;
+      job.promise.resolve(
+        promiseTry(job.jobFactory, ...job.args).finally(() => {
+          this.running = false;
+          this.tick();
+        }),
+      );
+    } else {
+      this.running = false;
+    }
+  }
+
+  push<TValue, TArgs extends unknown[]>(
+    jobFactory: Job<TValue, TArgs>,
+    ...args: TArgs
+  ): Promise<TValue> {
+    const promise = resolvablePromise<TValue>();
+    this.jobs.push({ jobFactory, promise, args });
+
+    this.tick();
+
+    return promise;
+  }
+}

packages/excalidraw/types.ts

@@ -38,7 +38,7 @@ import type { FileSystemHandle } from "./data/filesystem";
 import type { IMAGE_MIME_TYPES, MIME_TYPES } from "./constants";
 import { ContextMenuItems } from "./components/ContextMenu";
 import { SnapLine } from "./snapping";
-import { Merge, ValueOf } from "./utility-types";
+import { Merge, MaybePromise, ValueOf } from "./utility-types";
 
 export type Point = Readonly<RoughPoint>;
 
@@ -380,21 +380,14 @@ export type LibraryItems_anyVersion = LibraryItems | LibraryItems_v1;
 export type LibraryItemsSource =
   | ((
       currentLibraryItems: LibraryItems,
-    ) =>
+    ) => MaybePromise<LibraryItems_anyVersion | Blob>)
-      | Blob
+  | MaybePromise<LibraryItems_anyVersion | Blob>;
-      | LibraryItems_anyVersion
-      | Promise<LibraryItems_anyVersion | Blob>)
-  | Blob
-  | LibraryItems_anyVersion
-  | Promise<LibraryItems_anyVersion | Blob>;
 // -----------------------------------------------------------------------------
 
 export type ExcalidrawInitialDataState = Merge<
   ImportedDataState,
   {
-    libraryItems?:
+    libraryItems?: MaybePromise<Required<ImportedDataState>["libraryItems"]>;
-      | Required<ImportedDataState>["libraryItems"]
-      | Promise<Required<ImportedDataState>["libraryItems"]>;
   }
 >;
 
@@ -409,10 +402,7 @@ export interface ExcalidrawProps {
     appState: AppState,
     files: BinaryFiles,
   ) => void;
-  initialData?:
+  initialData?: MaybePromise<ExcalidrawInitialDataState | null>;
-    | ExcalidrawInitialDataState
-    | null
-    | Promise<ExcalidrawInitialDataState | null>;
   excalidrawAPI?: (api: ExcalidrawImperativeAPI) => void;
   isCollaborating?: boolean;
   onPointerUpdate?: (payload: {
@@ -643,7 +633,7 @@ export type PointerDownState = Readonly<{
 
 export type UnsubscribeCallback = () => void;
 
-export type ExcalidrawImperativeAPI = {
+export interface ExcalidrawImperativeAPI {
   updateScene: InstanceType<typeof App>["updateScene"];
   updateLibrary: InstanceType<typeof Library>["updateLibrary"];
   resetScene: InstanceType<typeof App>["resetScene"];
@@ -700,7 +690,7 @@ export type ExcalidrawImperativeAPI = {
   onUserFollow: (
     callback: (payload: OnUserFollowedPayload) => void,
   ) => UnsubscribeCallback;
-};
+}
 
 export type Device = Readonly<{
   viewport: {

packages/excalidraw/utility-types.ts

@@ -62,3 +62,6 @@ export type MakeBrand<T extends string> = {
   /** @private using ~ to sort last in intellisense */
   [K in `~brand~${T}`]: T;
 };
+
+/** Maybe just promise or already fulfilled one! */
+export type MaybePromise<T> = T | Promise<T>;

packages/excalidraw/utils.ts

@@ -14,7 +14,7 @@ import {
   UnsubscribeCallback,
   Zoom,
 } from "./types";
-import { ResolutionType } from "./utility-types";
+import { MaybePromise, ResolutionType } from "./utility-types";
 
 let mockDateTime: string | null = null;
 
@@ -538,7 +538,9 @@ export const isTransparent = (color: string) => {
 };
 
 export type ResolvablePromise<T> = Promise<T> & {
-  resolve: [T] extends [undefined] ? (value?: T) => void : (value: T) => void;
+  resolve: [T] extends [undefined]
+    ? (value?: MaybePromise<Awaited<T>>) => void
+    : (value: MaybePromise<Awaited<T>>) => void;
   reject: (error: Error) => void;
 };
 export const resolvablePromise = <T>() => {
@@ -1090,3 +1092,13 @@ export const toBrandedType = <BrandedType, CurrentType = BrandedType>(
 };
 
 // -----------------------------------------------------------------------------
+
+// Promise.try, adapted from https://github.com/sindresorhus/p-try
+export const promiseTry = async <TValue, TArgs extends unknown[]>(
+  fn: (...args: TArgs) => PromiseLike<TValue> | TValue,
+  ...args: TArgs
+): Promise<TValue> => {
+  return new Promise((resolve) => {
+    resolve(fn(...args));
+  });
+};