diff --git a/src/appState.ts b/src/appState.ts
index c54b7acdd..6e15bc3bc 100644
--- a/src/appState.ts
+++ b/src/appState.ts
@@ -34,8 +34,7 @@ export function getDefaultAppState(): AppState {
     openMenu: null,
     lastPointerDownWith: "mouse",
     selectedElementIds: {},
-    remotePointers: {},
-    collaboratorCount: 0,
+    collaborators: new Map(),
   };
 }
 
@@ -47,6 +46,8 @@ export function clearAppStateForLocalStorage(appState: AppState) {
     editingElement,
     selectionElement,
     isResizing,
+    collaborators,
+    isCollaborating,
     ...exportedState
   } = appState;
   return exportedState;
diff --git a/src/components/App.tsx b/src/components/App.tsx
index 4c102fce1..5664d5ea9 100644
--- a/src/components/App.tsx
+++ b/src/components/App.tsx
@@ -193,8 +193,7 @@ export class App extends React.Component<any, AppState> {
       this.setState(state => ({
         ...res.appState,
         isCollaborating: state.isCollaborating,
-        remotePointers: state.remotePointers,
-        collaboratorCount: state.collaboratorCount,
+        collaborators: state.collaborators,
       }));
     }
   };
@@ -234,8 +233,7 @@ export class App extends React.Component<any, AppState> {
   private destroySocketClient = () => {
     this.setState({
       isCollaborating: false,
-      remotePointers: {},
-      collaboratorCount: 0,
+      collaborators: new Map(),
     });
     if (this.socket) {
       this.socket.close();
@@ -321,11 +319,14 @@ export class App extends React.Component<any, AppState> {
               break;
             case "MOUSE_LOCATION":
               const { socketID, pointerCoords } = decryptedData.payload;
-              this.setState({
-                remotePointers: {
-                  ...this.state.remotePointers,
-                  [socketID]: pointerCoords,
-                },
+              this.setState(state => {
+                if (state.collaborators.has(socketID)) {
+                  const user = state.collaborators.get(socketID)!;
+                  user.pointer = pointerCoords;
+                  state.collaborators.set(socketID, user);
+                  return state;
+                }
+                return null;
               });
               break;
           }
@@ -337,13 +338,20 @@ export class App extends React.Component<any, AppState> {
         }
         this.socketInitialized = true;
       });
-      this.socket.on("room-user-count", (collaboratorCount: number) => {
-        this.setState({ collaboratorCount });
-      });
-      this.socket.on("client-disconnected", (socketID: number) => {
+      this.socket.on("room-user-change", (clients: string[]) => {
         this.setState(state => {
-          const { [socketID]: omit, ...remotePointers } = state.remotePointers;
-          return { remotePointers };
+          const collaborators: typeof state.collaborators = new Map();
+          for (const socketID of clients) {
+            if (state.collaborators.has(socketID)) {
+              collaborators.set(socketID, state.collaborators.get(socketID)!);
+            } else {
+              collaborators.set(socketID, {});
+            }
+          }
+          return {
+            ...state,
+            collaborators,
+          };
         });
       });
       this.socket.on("new-user", async (socketID: string) => {
@@ -2095,17 +2103,19 @@ export class App extends React.Component<any, AppState> {
     const pointerViewportCoords: {
       [id: string]: { x: number; y: number };
     } = {};
-    for (const clientId in this.state.remotePointers) {
-      const remotePointerCoord = this.state.remotePointers[clientId];
-      pointerViewportCoords[clientId] = sceneCoordsToViewportCoords(
+    this.state.collaborators.forEach((user, socketID) => {
+      if (!user.pointer) {
+        return;
+      }
+      pointerViewportCoords[socketID] = sceneCoordsToViewportCoords(
         {
-          sceneX: remotePointerCoord.x,
-          sceneY: remotePointerCoord.y,
+          sceneX: user.pointer.x,
+          sceneY: user.pointer.y,
         },
         this.state,
         this.canvas,
       );
-    }
+    });
     const { atLeastOneVisibleElement, scrollBars } = renderScene(
       elements,
       this.state,
diff --git a/src/components/LayerUI.tsx b/src/components/LayerUI.tsx
index e3efb0cce..47052b0a0 100644
--- a/src/components/LayerUI.tsx
+++ b/src/components/LayerUI.tsx
@@ -116,7 +116,7 @@ export const LayerUI = React.memo(
                       {actionManager.renderAction("clearCanvas")}
                       <RoomDialog
                         isCollaborating={appState.isCollaborating}
-                        collaboratorCount={appState.collaboratorCount}
+                        collaboratorCount={appState.collaborators.size}
                         onRoomCreate={onRoomCreate}
                         onRoomDestroy={onRoomDestroy}
                       />
diff --git a/src/components/MobileMenu.tsx b/src/components/MobileMenu.tsx
index 2b28186ff..89145c4ec 100644
--- a/src/components/MobileMenu.tsx
+++ b/src/components/MobileMenu.tsx
@@ -47,7 +47,7 @@ export function MobileMenu({
               {actionManager.renderAction("clearCanvas")}
               <RoomDialog
                 isCollaborating={appState.isCollaborating}
-                collaboratorCount={appState.collaboratorCount}
+                collaboratorCount={appState.collaborators.size}
                 onRoomCreate={onRoomCreate}
                 onRoomDestroy={onRoomDestroy}
               />
diff --git a/src/components/RoomDialog.tsx b/src/components/RoomDialog.tsx
index f4720f2dc..476ff5b89 100644
--- a/src/components/RoomDialog.tsx
+++ b/src/components/RoomDialog.tsx
@@ -116,7 +116,7 @@ export function RoomDialog({
   onRoomDestroy,
 }: {
   isCollaborating: AppState["isCollaborating"];
-  collaboratorCount: AppState["collaboratorCount"];
+  collaboratorCount: number;
   onRoomCreate: () => void;
   onRoomDestroy: () => void;
 }) {
diff --git a/src/data/localStorage.ts b/src/data/localStorage.ts
index 59d1a5230..f8101e3da 100644
--- a/src/data/localStorage.ts
+++ b/src/data/localStorage.ts
@@ -36,7 +36,7 @@ export function restoreFromLocalStorage() {
       appState = JSON.parse(savedState) as AppState;
       // If we're retrieving from local storage, we should not be collaborating
       appState.isCollaborating = false;
-      appState.collaboratorCount = 0;
+      appState.collaborators = new Map();
     } catch {
       // Do nothing because appState is already null
     }
diff --git a/src/types.ts b/src/types.ts
index 457d806f2..e1bf965a8 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -35,8 +35,7 @@ export type AppState = {
   openMenu: "canvas" | "shape" | null;
   lastPointerDownWith: PointerType;
   selectedElementIds: { [id: string]: boolean };
-  remotePointers: { [id: string]: { x: number; y: number } };
-  collaboratorCount: number;
+  collaborators: Map<string, { pointer?: { x: number; y: number } }>;
 };
 
 export type PointerCoords = Readonly<{