diff --git a/src/components/App.tsx b/src/components/App.tsx
index a6c58cef6..3446fcb30 100644
--- a/src/components/App.tsx
+++ b/src/components/App.tsx
@@ -2046,7 +2046,7 @@ class App extends React.Component<any, AppState> {
     let resizeArrowDirection: "origin" | "end" = "origin";
     let isResizingElements = false;
     let draggingOccurred = false;
-    let dragOffsetXY: [number, number] = [0, 0];
+    let dragOffsetXY: [number, number] | null = null;
     let hitElement: ExcalidrawElement | null = null;
     let hitElementWasAddedToSelection = false;
 
@@ -2130,20 +2130,6 @@ class App extends React.Component<any, AppState> {
           hitElement ||
           getElementAtPosition(elements, this.state, x, y, this.state.zoom);
 
-        if (hitElement && isNonDeletedElement(hitElement)) {
-          if (this.state.selectedElementIds[hitElement.id]) {
-            dragOffsetXY = getDragOffsetXY(selectedElements, x, y);
-          } else if (event.shiftKey) {
-            dragOffsetXY = getDragOffsetXY(
-              [...selectedElements, hitElement],
-              x,
-              y,
-            );
-          } else {
-            dragOffsetXY = getDragOffsetXY([hitElement], x, y);
-          }
-        }
-
         // clear selection if shift is not clicked
         if (
           !(hitElement && this.state.selectedElementIds[hitElement.id]) &&
@@ -2369,6 +2355,18 @@ class App extends React.Component<any, AppState> {
     let selectedElementWasDuplicated = false;
 
     const onPointerMove = withBatchedUpdates((event: PointerEvent) => {
+      // We need to initialize dragOffsetXY only after we've updated
+      // `state.selectedElementIds` on pointerDown. Doing it here in pointerMove
+      // event handler should hopefully ensure we're already working with
+      // the updated state.
+      if (dragOffsetXY === null) {
+        dragOffsetXY = getDragOffsetXY(
+          getSelectedElements(globalSceneState.getElements(), this.state),
+          originX,
+          originY,
+        );
+      }
+
       const target = event.target;
       if (!(target instanceof HTMLElement)) {
         return;