diff --git a/src/appState.ts b/src/appState.ts
index 1c39dd448..e25c59295 100644
--- a/src/appState.ts
+++ b/src/appState.ts
@@ -26,6 +26,7 @@ export function getDefaultAppState(): AppState {
     scrollY: 0 as FlooredNumber,
     cursorX: 0,
     cursorY: 0,
+    cursorButton: "up",
     scrolledOutside: false,
     name: `excalidraw-${getDateTime()}`,
     isCollaborating: false,
diff --git a/src/components/App.tsx b/src/components/App.tsx
index 173e49b49..764fd10c8 100644
--- a/src/components/App.tsx
+++ b/src/components/App.tsx
@@ -441,6 +441,10 @@ export class App extends React.Component<any, AppState> {
     if (this.state.isCollaborating && !this.socket) {
       this.initializeSocketClient({ showLoadingState: true });
     }
+
+    const cursorButton: {
+      [id: string]: string | undefined;
+    } = {};
     const pointerViewportCoords: SceneState["remotePointerViewportCoords"] = {};
     const remoteSelectedElementIds: SceneState["remoteSelectedElementIds"] = {};
     this.state.collaborators.forEach((user, socketID) => {
@@ -464,6 +468,7 @@ export class App extends React.Component<any, AppState> {
         this.canvas,
         window.devicePixelRatio,
       );
+      cursorButton[socketID] = user.button;
     });
     const { atLeastOneVisibleElement, scrollBars } = renderScene(
       globalSceneState.getAllElements().filter((element) => {
@@ -486,6 +491,7 @@ export class App extends React.Component<any, AppState> {
         viewBackgroundColor: this.state.viewBackgroundColor,
         zoom: this.state.zoom,
         remotePointerViewportCoords: pointerViewportCoords,
+        remotePointerButton: cursorButton,
         remoteSelectedElementIds: remoteSelectedElementIds,
         shouldCacheIgnoreZoom: this.state.shouldCacheIgnoreZoom,
       },
@@ -871,6 +877,7 @@ export class App extends React.Component<any, AppState> {
               const {
                 socketID,
                 pointerCoords,
+                button,
                 selectedElementIds,
               } = decryptedData.payload;
               this.setState((state) => {
@@ -879,6 +886,7 @@ export class App extends React.Component<any, AppState> {
                 }
                 const user = state.collaborators.get(socketID)!;
                 user.pointer = pointerCoords;
+                user.button = button;
                 user.selectedElementIds = selectedElementIds;
                 state.collaborators.set(socketID, user);
                 return state;
@@ -923,6 +931,7 @@ export class App extends React.Component<any, AppState> {
 
   private broadcastMouseLocation = (payload: {
     pointerCoords: SocketUpdateDataSource["MOUSE_LOCATION"]["payload"]["pointerCoords"];
+    button: SocketUpdateDataSource["MOUSE_LOCATION"]["payload"]["button"];
   }) => {
     if (this.socket?.id) {
       const data: SocketUpdateDataSource["MOUSE_LOCATION"] = {
@@ -930,6 +939,7 @@ export class App extends React.Component<any, AppState> {
         payload: {
           socketID: this.socket.id,
           pointerCoords: payload.pointerCoords,
+          button: payload.button || "up",
           selectedElementIds: this.state.selectedElementIds,
         },
       };
@@ -1345,13 +1355,8 @@ export class App extends React.Component<any, AppState> {
   private handleCanvasPointerMove = (
     event: React.PointerEvent<HTMLCanvasElement>,
   ) => {
-    const pointerCoords = viewportCoordsToSceneCoords(
-      event,
-      this.state,
-      this.canvas,
-      window.devicePixelRatio,
-    );
-    this.savePointer(pointerCoords);
+    this.savePointer(event.clientX, event.clientY, this.state.cursorButton);
+
     if (gesture.pointers.has(event.pointerId)) {
       gesture.pointers.set(event.pointerId, {
         x: event.clientX,
@@ -1502,7 +1507,11 @@ export class App extends React.Component<any, AppState> {
       return;
     }
 
-    this.setState({ lastPointerDownWith: event.pointerType });
+    this.setState({
+      lastPointerDownWith: event.pointerType,
+      cursorButton: "down",
+    });
+    this.savePointer(event.clientX, event.clientY, "down");
 
     // pan canvas on wheel button drag or space+drag
     if (
@@ -1535,6 +1544,10 @@ export class App extends React.Component<any, AppState> {
           if (!isHoldingSpace) {
             setCursorForShape(this.state.elementType);
           }
+          this.setState({
+            cursorButton: "up",
+          });
+          this.savePointer(event.clientX, event.clientY, "up");
           window.removeEventListener("pointermove", onPointerMove);
           window.removeEventListener("pointerup", teardown);
           window.removeEventListener("blur", teardown);
@@ -1635,6 +1648,10 @@ export class App extends React.Component<any, AppState> {
         isDraggingScrollBar = false;
         setCursorForShape(this.state.elementType);
         lastPointerUp = null;
+        this.setState({
+          cursorButton: "up",
+        });
+        this.savePointer(event.clientX, event.clientY, "up");
         window.removeEventListener("pointermove", onPointerMove);
         window.removeEventListener("pointerup", onPointerUp);
       });
@@ -2398,7 +2415,7 @@ export class App extends React.Component<any, AppState> {
       }
     });
 
-    const onPointerUp = withBatchedUpdates((event: PointerEvent) => {
+    const onPointerUp = withBatchedUpdates((childEvent: PointerEvent) => {
       const {
         draggingElement,
         resizingElement,
@@ -2412,11 +2429,15 @@ export class App extends React.Component<any, AppState> {
         isRotating: false,
         resizingElement: null,
         selectionElement: null,
+        cursorButton: "up",
         editingElement: multiElement ? this.state.editingElement : null,
       });
 
+      this.savePointer(childEvent.clientX, childEvent.clientY, "up");
+
       resizeArrowFn = null;
       lastPointerUp = null;
+
       window.removeEventListener("pointermove", onPointerMove);
       window.removeEventListener("pointerup", onPointerUp);
 
@@ -2426,7 +2447,7 @@ export class App extends React.Component<any, AppState> {
         }
         if (!draggingOccurred && draggingElement && !multiElement) {
           const { x, y } = viewportCoordsToSceneCoords(
-            event,
+            childEvent,
             this.state,
             this.canvas,
             window.devicePixelRatio,
@@ -2503,7 +2524,7 @@ export class App extends React.Component<any, AppState> {
       // was added to selection (on pointerdown phase) we need to keep
       // selection unchanged
       if (hitElement && !draggingOccurred && !hitElementWasAddedToSelection) {
-        if (event.shiftKey) {
+        if (childEvent.shiftKey) {
           this.setState((prevState) => ({
             selectedElementIds: {
               ...prevState.selectedElementIds,
@@ -2735,12 +2756,26 @@ export class App extends React.Component<any, AppState> {
     }
   }
 
-  private savePointer = (pointerCoords: { x: number; y: number }) => {
+  private savePointer = (x: number, y: number, button: "up" | "down") => {
+    if (!x || !y) {
+      return;
+    }
+    const pointerCoords = viewportCoordsToSceneCoords(
+      { clientX: x, clientY: y },
+      this.state,
+      this.canvas,
+      window.devicePixelRatio,
+    );
+
     if (isNaN(pointerCoords.x) || isNaN(pointerCoords.y)) {
       // sometimes the pointer goes off screen
       return;
     }
-    this.socket && this.broadcastMouseLocation({ pointerCoords });
+    this.socket &&
+      this.broadcastMouseLocation({
+        pointerCoords,
+        button,
+      });
   };
 
   private resetShouldCacheIgnoreZoomDebounced = debounce(() => {
diff --git a/src/data/index.ts b/src/data/index.ts
index 13818b934..8b51fa29e 100644
--- a/src/data/index.ts
+++ b/src/data/index.ts
@@ -49,6 +49,7 @@ export type SocketUpdateDataSource = {
     payload: {
       socketID: string;
       pointerCoords: { x: number; y: number };
+      button: "down" | "up";
       selectedElementIds: AppState["selectedElementIds"];
     };
   };
diff --git a/src/renderer/renderScene.ts b/src/renderer/renderScene.ts
index ca42a2381..9f8ac98f7 100644
--- a/src/renderer/renderScene.ts
+++ b/src/renderer/renderScene.ts
@@ -298,6 +298,26 @@ export function renderScene(
     if (isOutOfBounds) {
       context.globalAlpha = 0.2;
     }
+
+    if (
+      sceneState.remotePointerButton &&
+      sceneState.remotePointerButton[clientId] === "down"
+    ) {
+      context.beginPath();
+      context.arc(x, y, 15, 0, 2 * Math.PI, false);
+      context.lineWidth = 3;
+      context.strokeStyle = "#ffffff88";
+      context.stroke();
+      context.closePath();
+
+      context.beginPath();
+      context.arc(x, y, 15, 0, 2 * Math.PI, false);
+      context.lineWidth = 1;
+      context.strokeStyle = stroke;
+      context.stroke();
+      context.closePath();
+    }
+
     context.beginPath();
     context.moveTo(x, y);
     context.lineTo(x + 1, y + 14);
@@ -309,6 +329,7 @@ export function renderScene(
     context.strokeStyle = strokeStyle;
     context.fillStyle = fillStyle;
     context.globalAlpha = globalAlpha;
+    context.closePath();
   }
 
   // Paint scrollbars
diff --git a/src/scene/types.ts b/src/scene/types.ts
index 7855eb4e6..6a38c5b82 100644
--- a/src/scene/types.ts
+++ b/src/scene/types.ts
@@ -9,6 +9,7 @@ export type SceneState = {
   zoom: number;
   shouldCacheIgnoreZoom: boolean;
   remotePointerViewportCoords: { [id: string]: { x: number; y: number } };
+  remotePointerButton?: { [id: string]: string | undefined };
   remoteSelectedElementIds: { [elementId: string]: string[] };
 };
 
diff --git a/src/tests/__snapshots__/regressionTests.test.tsx.snap b/src/tests/__snapshots__/regressionTests.test.tsx.snap
index ab6a7cdbe..1238676fa 100644
--- a/src/tests/__snapshots__/regressionTests.test.tsx.snap
+++ b/src/tests/__snapshots__/regressionTests.test.tsx.snap
@@ -10,6 +10,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -197,6 +198,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -307,6 +309,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#5f3dc4",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -557,6 +560,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -703,6 +707,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -885,6 +890,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -1072,6 +1078,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -1355,6 +1362,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -1951,6 +1959,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -2061,6 +2070,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -2171,6 +2181,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -2281,6 +2292,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -2413,6 +2425,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -2545,6 +2558,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -2677,6 +2691,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -2787,6 +2802,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -2897,6 +2913,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -3029,6 +3046,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -3139,6 +3157,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "down",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -3191,6 +3210,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -3877,6 +3897,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -4239,6 +4260,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -4529,6 +4551,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -4747,6 +4770,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -4893,6 +4917,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -5543,6 +5568,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -6121,6 +6147,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -6627,6 +6654,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -7061,6 +7089,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -7459,6 +7488,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -7785,6 +7815,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -8039,6 +8070,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -8221,6 +8253,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -8907,6 +8940,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -9521,6 +9555,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -10063,6 +10098,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -10533,6 +10569,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -10777,6 +10814,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -10815,7 +10853,7 @@ Object {
 
 exports[`regression tests spacebar + drag scrolls the canvas: [end of test] number of elements 1`] = `0`;
 
-exports[`regression tests spacebar + drag scrolls the canvas: [end of test] number of renders 1`] = `4`;
+exports[`regression tests spacebar + drag scrolls the canvas: [end of test] number of renders 1`] = `5`;
 
 exports[`regression tests two-finger scroll works: [end of test] appState 1`] = `
 Object {
@@ -10827,6 +10865,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "down",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -10879,6 +10918,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
@@ -11161,6 +11201,7 @@ Object {
   "currentItemRoughness": 1,
   "currentItemStrokeColor": "#000000",
   "currentItemStrokeWidth": 1,
+  "cursorButton": "up",
   "cursorX": 0,
   "cursorY": 0,
   "draggingElement": null,
diff --git a/src/types.ts b/src/types.ts
index dbe4e2fb6..44c1fca85 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -34,6 +34,7 @@ export type AppState = {
   scrollY: FlooredNumber;
   cursorX: number;
   cursorY: number;
+  cursorButton: "up" | "down";
   scrolledOutside: boolean;
   name: string;
   isCollaborating: boolean;
@@ -50,6 +51,7 @@ export type AppState = {
         x: number;
         y: number;
       };
+      button?: "up" | "down";
       selectedElementIds?: AppState["selectedElementIds"];
     }
   >;