diff --git a/packages/react-native-sortables/src/components/shared/CustomHandle.tsx b/packages/react-native-sortables/src/components/shared/CustomHandle.tsx
index 7c45a3a4..614ec8e8 100644
--- a/packages/react-native-sortables/src/components/shared/CustomHandle.tsx
+++ b/packages/react-native-sortables/src/components/shared/CustomHandle.tsx
@@ -1,12 +1,10 @@
-import { type PropsWithChildren, useCallback, useEffect, useMemo } from 'react';
+import { type PropsWithChildren, useCallback, useEffect } from 'react';
 import { View } from 'react-native';
 import { GestureDetector } from 'react-native-gesture-handler';
-import { measure, useAnimatedRef } from 'react-native-reanimated';
+import { useAnimatedRef } from 'react-native-reanimated';
 
 import {
-  useCommonValuesContext,
   useCustomHandleContext,
-  useDragContext,
   useItemContext,
   usePortalOutletContext
 } from '../../providers';
@@ -49,85 +47,27 @@ function CustomHandleComponent({
     );
   }
 
-  const { activeItemKey, containerRef, itemPositions } =
-    useCommonValuesContext();
-  const { setDragStartValues } = useDragContext();
-  const { gesture, itemKey } = useItemContext();
-  const viewRef = useAnimatedRef<View>();
+  const { gesture, isActive, itemKey } = useItemContext();
+  const handleRef = useAnimatedRef<View>();
 
-  const {
-    activeHandleMeasurements,
-    activeHandleOffset,
-    makeItemFixed,
-    removeFixedItem
-  } = customHandleContext;
+  const { registerHandle, updateActiveHandleMeasurements } =
+    customHandleContext;
   const dragEnabled = mode === 'draggable';
 
   useEffect(() => {
-    if (mode === 'fixed') {
-      makeItemFixed(itemKey);
-    }
-
-    return () => removeFixedItem(itemKey);
-  }, [mode, itemKey, makeItemFixed, removeFixedItem]);
-
-  const measureHandle = useCallback(
-    (mustBeActive: boolean) => {
-      'worklet';
-      if (mustBeActive && activeItemKey.value !== itemKey) {
-        return;
-      }
-
-      const handleMeasurements = measure(viewRef);
-      const containerMeasurements = measure(containerRef);
-      const itemPosition = itemPositions.value[itemKey];
-
-      if (!handleMeasurements || !containerMeasurements || !itemPosition) {
-        return;
-      }
-
-      const { pageX, pageY } = handleMeasurements;
-      const { pageX: containerPageX, pageY: containerPageY } =
-        containerMeasurements;
-      const { x: activeX, y: activeY } = itemPosition;
+    return registerHandle(itemKey, handleRef, mode === 'fixed');
+  }, [handleRef, itemKey, registerHandle, mode]);
 
-      activeHandleMeasurements.value = handleMeasurements;
-      activeHandleOffset.value = {
-        x: pageX - containerPageX - activeX,
-        y: pageY - containerPageY - activeY
-      };
-
-      setDragStartValues(itemKey);
-    },
-    [
-      activeHandleOffset,
-      activeHandleMeasurements,
-      activeItemKey,
-      containerRef,
-      itemPositions,
-      itemKey,
-      setDragStartValues,
-      viewRef
-    ]
-  );
-
-  const gestureWithMeasure = useMemo(
-    () =>
-      gesture.onBegin(() => {
-        'worklet';
-        measureHandle(false);
-      }),
-    [gesture, measureHandle]
-  );
+  const onLayout = useCallback(() => {
+    'worklet';
+    if (isActive.value) {
+      updateActiveHandleMeasurements(itemKey);
+    }
+  }, [itemKey, isActive, updateActiveHandleMeasurements]);
 
   return (
-    <GestureDetector
-      gesture={gestureWithMeasure.enabled(dragEnabled)}
-      userSelect='none'>
-      <View
-        collapsable={false}
-        ref={viewRef}
-        onLayout={dragEnabled ? () => measureHandle(true) : undefined}>
+    <GestureDetector gesture={gesture.enabled(dragEnabled)} userSelect='none'>
+      <View collapsable={false} ref={handleRef} onLayout={onLayout}>
         {children}
       </View>
     </GestureDetector>
diff --git a/packages/react-native-sortables/src/components/shared/DraggableView/DraggableView.tsx b/packages/react-native-sortables/src/components/shared/DraggableView/DraggableView.tsx
index 7d6a7416..1e7a9994 100644
--- a/packages/react-native-sortables/src/components/shared/DraggableView/DraggableView.tsx
+++ b/packages/react-native-sortables/src/components/shared/DraggableView/DraggableView.tsx
@@ -62,7 +62,6 @@ function DraggableView({
     activationAnimationProgress,
     portalState
   );
-
   const gesture = useItemPanGesture(key, activationAnimationProgress);
 
   useEffect(() => {
diff --git a/packages/react-native-sortables/src/providers/shared/CustomHandleProvider.ts b/packages/react-native-sortables/src/providers/shared/CustomHandleProvider.ts
index 1aa93f98..8c5f293a 100644
--- a/packages/react-native-sortables/src/providers/shared/CustomHandleProvider.ts
+++ b/packages/react-native-sortables/src/providers/shared/CustomHandleProvider.ts
@@ -1,11 +1,12 @@
-import type { ReactNode } from 'react';
-import type { MeasuredDimensions } from 'react-native-reanimated';
-import { useSharedValue } from 'react-native-reanimated';
+import { type ReactNode, useCallback } from 'react';
+import type { View } from 'react-native';
+import type { AnimatedRef, MeasuredDimensions } from 'react-native-reanimated';
+import { measure, runOnUI, useSharedValue } from 'react-native-reanimated';
 
-import { useUIStableCallback } from '../../hooks';
 import type { CustomHandleContextType, Vector } from '../../types';
 import { useAnimatedDebounce } from '../../utils';
 import { createProvider } from '../utils';
+import { useCommonValuesContext } from './CommonValuesProvider';
 
 type CustomHandleProviderProps = {
   children?: ReactNode;
@@ -15,32 +16,80 @@ const { CustomHandleProvider, useCustomHandleContext } = createProvider(
   'CustomHandle',
   { guarded: false }
 )<CustomHandleProviderProps, CustomHandleContextType>(() => {
-  const activeHandleOffset = useSharedValue<Vector | null>(null);
+  const { containerRef, itemPositions } = useCommonValuesContext();
+  const debounce = useAnimatedDebounce();
+
+  const fixedItemKeys = useSharedValue<Record<string, boolean>>({});
+  const handleRefs = useSharedValue<Record<string, AnimatedRef<View>>>({});
   const activeHandleMeasurements = useSharedValue<MeasuredDimensions | null>(
     null
   );
-  const fixedItemKeys = useSharedValue<Record<string, boolean>>({});
-  const debounce = useAnimatedDebounce();
+  const activeHandleOffset = useSharedValue<Vector | null>(null);
+
+  const registerHandle = useCallback(
+    (key: string, handleRef: AnimatedRef<View>, fixed: boolean) => {
+      runOnUI(() => {
+        'worklet';
+        handleRefs.value[key] = handleRef;
+        if (fixed) {
+          fixedItemKeys.value[key] = true;
+          debounce(fixedItemKeys.modify, 100);
+        }
+      })();
 
-  const makeItemFixed = useUIStableCallback((key: string) => {
-    'worklet';
-    fixedItemKeys.value[key] = true;
-    debounce(fixedItemKeys.modify, 100);
-  });
+      const unregister = () => {
+        'worklet';
+        delete handleRefs.value[key];
+        if (fixed) {
+          fixedItemKeys.value[key] = false;
+          debounce(fixedItemKeys.modify, 100);
+        }
+      };
 
-  const removeFixedItem = useUIStableCallback((key: string) => {
-    'worklet';
-    delete fixedItemKeys.value[key];
-    debounce(fixedItemKeys.modify, 100);
-  });
+      return runOnUI(unregister);
+    },
+    [debounce, fixedItemKeys, handleRefs]
+  );
+
+  const updateActiveHandleMeasurements = useCallback(
+    (key: string) => {
+      'worklet';
+      const handleRef = handleRefs.value[key];
+      if (!handleRef) {
+        return;
+      }
+
+      const handleMeasurements = measure(handleRef);
+      const containerMeasurements = measure(containerRef);
+      const itemPosition = itemPositions.value[key];
+
+      if (!handleMeasurements || !containerMeasurements || !itemPosition) {
+        return;
+      }
+
+      activeHandleMeasurements.value = handleMeasurements;
+      const { x: itemX, y: itemY } = itemPosition;
+      activeHandleOffset.value = {
+        x: handleMeasurements.pageX - containerMeasurements.pageX - itemX,
+        y: handleMeasurements.pageY - containerMeasurements.pageY - itemY
+      };
+    },
+    [
+      activeHandleMeasurements,
+      activeHandleOffset,
+      containerRef,
+      handleRefs,
+      itemPositions
+    ]
+  );
 
   return {
     value: {
       activeHandleMeasurements,
       activeHandleOffset,
       fixedItemKeys,
-      makeItemFixed,
-      removeFixedItem
+      registerHandle,
+      updateActiveHandleMeasurements
     }
   };
 });
diff --git a/packages/react-native-sortables/src/providers/shared/DragProvider.ts b/packages/react-native-sortables/src/providers/shared/DragProvider.ts
index a2e70228..da435cbd 100644
--- a/packages/react-native-sortables/src/providers/shared/DragProvider.ts
+++ b/packages/react-native-sortables/src/providers/shared/DragProvider.ts
@@ -70,7 +70,6 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
     containerHeight,
     containerRef,
     containerWidth,
-    customHandle,
     dragActivationDelay,
     dragActivationFailOffset,
     dropAnimationDuration,
@@ -92,8 +91,11 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
   const { updateLayer } = useLayerContext() ?? {};
   const { scrollOffsetDiff, updateStartScrollOffset } =
     useAutoScrollContext() ?? {};
-  const { activeHandleMeasurements, activeHandleOffset } =
-    useCustomHandleContext() ?? {};
+  const {
+    activeHandleMeasurements,
+    activeHandleOffset,
+    updateActiveHandleMeasurements
+  } = useCustomHandleContext() ?? {};
   const { activeItemAbsolutePosition } = usePortalContext() ?? {};
 
   const haptics = useHaptics(hapticsEnabled);
@@ -131,9 +133,8 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
       offsetX: snapOffsetX.value,
       offsetY: snapOffsetY.value,
       progress: activeAnimationProgress.value,
-      snapDimensions: customHandle
-        ? activeHandleMeasurements?.value
-        : activeItemDimensions.value,
+      snapDimensions:
+        activeHandleMeasurements?.value ?? activeItemDimensions.value,
       snapOffset: activeHandleOffset?.value,
       startTouch: touchStartTouch.value,
       startTouchPosition: dragStartTouchPosition.value,
@@ -228,53 +229,9 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
    * DRAG HANDLERS
    */
 
-  // If custom handle is used, it must be called after handle is measured
-  const setDragStartValues = useCallback(
-    (key: string) => {
-      'worklet';
-      const itemPosition = itemPositions.value[key];
-
-      if (!itemPosition || !currentTouch.value) {
-        return;
-      }
-
-      let touchItemPosition = itemPosition;
-      if (customHandle) {
-        const containerMeasurements = measure(containerRef);
-        if (!activeHandleMeasurements?.value || !containerMeasurements) {
-          return;
-        }
-
-        touchItemPosition = {
-          x: activeHandleMeasurements.value.pageX - containerMeasurements.pageX,
-          y: activeHandleMeasurements.value.pageY - containerMeasurements.pageY
-        };
-      }
-
-      const touchX = touchItemPosition.x + currentTouch.value.x;
-      const touchY = touchItemPosition.y + currentTouch.value.y;
-
-      touchPosition.value = { x: touchX, y: touchY };
-      dragStartTouchPosition.value = touchPosition.value;
-      dragStartItemTouchOffset.value = {
-        x: touchX - itemPosition.x,
-        y: touchY - itemPosition.y
-      };
-    },
-    [
-      activeHandleMeasurements,
-      containerRef,
-      currentTouch,
-      customHandle,
-      dragStartItemTouchOffset,
-      dragStartTouchPosition,
-      itemPositions,
-      touchPosition
-    ]
-  );
-
   const handleDragStart = useCallback(
     (
+      touch: TouchData,
       key: string,
       activationAnimationProgress: SharedValue<number>,
       fail: () => void
@@ -282,8 +239,9 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
       'worklet';
       const itemPosition = itemPositions.value[key];
       const dimensions = itemDimensions.value[key];
+      const containerMeasurements = measure(containerRef);
 
-      if (!itemPosition || !dimensions) {
+      if (!itemPosition || !dimensions || !containerMeasurements) {
         fail();
         return;
       }
@@ -300,12 +258,31 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
       updateLayer?.(LayerState.FOCUSED);
       updateStartScrollOffset?.();
 
-      // If a custom handle is used, these values will be set in the
-      // handle component after the handle is measured
-      if (!customHandle) {
-        setDragStartValues(key);
+      let touchedItemPosition = itemPosition;
+
+      // We need to update the custom handle measurements if the custom handle
+      // is used (touch position is relative to the handle in this case)
+      updateActiveHandleMeasurements?.(key);
+      if (activeHandleMeasurements?.value) {
+        const { pageX, pageY } = activeHandleMeasurements.value;
+        touchedItemPosition = {
+          x: pageX - containerMeasurements.pageX,
+          y: pageY - containerMeasurements.pageY
+        };
       }
 
+      // Touch position relative to the top-left corner of the sortable
+      // container
+      const touchX = touchedItemPosition.x + touch.x;
+      const touchY = touchedItemPosition.y + touch.y;
+
+      touchPosition.value = { x: touchX, y: touchY };
+      dragStartTouchPosition.value = touchPosition.value;
+      dragStartItemTouchOffset.value = {
+        x: touchX - itemPosition.x,
+        y: touchY - itemPosition.y
+      };
+
       const hasInactiveAnimation =
         inactiveItemOpacity.value !== 1 || inactiveItemScale.value !== 1;
 
@@ -327,13 +304,16 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
     [
       activationAnimationDuration,
       activeAnimationProgress,
+      activeHandleMeasurements,
       activeItemDimensions,
       activeItemDropped,
       activationState,
       activeItemKey,
       activeItemPosition,
-      customHandle,
+      containerRef,
       dragStartIndex,
+      dragStartItemTouchOffset,
+      dragStartTouchPosition,
       haptics,
       inactiveAnimationProgress,
       inactiveItemOpacity,
@@ -343,9 +323,10 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
       itemPositions,
       keyToIndex,
       prevActiveItemKey,
-      setDragStartValues,
       stableOnDragStart,
+      touchPosition,
       updateLayer,
+      updateActiveHandleMeasurements,
       updateStartScrollOffset
     ]
   );
@@ -389,7 +370,7 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
         if (absoluteLayoutState.value !== AbsoluteLayoutState.COMPLETE) {
           return;
         }
-        handleDragStart(key, activationAnimationProgress, fail);
+        handleDragStart(touch, key, activationAnimationProgress, fail);
         activate();
       }, dragActivationDelay.value);
     },
@@ -401,8 +382,8 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
       currentTouch,
       dragActivationDelay,
       handleDragStart,
-      sortEnabled,
       measureContainer,
+      sortEnabled,
       touchStartTouch
     ]
   );
@@ -572,8 +553,7 @@ const { DragProvider, useDragContext } = createProvider('Drag')<
       handleDragEnd,
       handleOrderChange,
       handleTouchStart,
-      handleTouchesMove,
-      setDragStartValues
+      handleTouchesMove
     }
   };
 });
diff --git a/packages/react-native-sortables/src/providers/shared/hooks/useItemPanGesture.ts b/packages/react-native-sortables/src/providers/shared/hooks/useItemPanGesture.ts
index 4459dfe2..6b670804 100644
--- a/packages/react-native-sortables/src/providers/shared/hooks/useItemPanGesture.ts
+++ b/packages/react-native-sortables/src/providers/shared/hooks/useItemPanGesture.ts
@@ -36,11 +36,11 @@ export default function useItemPanGesture(
           handleDragEnd(key, activationAnimationProgress);
         }),
     [
-      key,
-      activationAnimationProgress,
       handleDragEnd,
       handleTouchStart,
-      handleTouchesMove
+      handleTouchesMove,
+      key,
+      activationAnimationProgress
     ]
   );
 }
diff --git a/packages/react-native-sortables/src/types/providers/shared.ts b/packages/react-native-sortables/src/types/providers/shared.ts
index 3944f034..2102d372 100644
--- a/packages/react-native-sortables/src/types/providers/shared.ts
+++ b/packages/react-native-sortables/src/types/providers/shared.ts
@@ -133,7 +133,6 @@ export type DragContextType = {
     toIndex: number,
     newOrder: Array<string>
   ) => void;
-  setDragStartValues: (key: string) => void;
 };
 
 // ITEM
@@ -161,11 +160,15 @@ export type LayerContextType = {
 // CUSTOM HANDLE
 
 export type CustomHandleContextType = {
-  activeHandleOffset: SharedValue<Vector | null>;
-  activeHandleMeasurements: SharedValue<MeasuredDimensions | null>;
   fixedItemKeys: SharedValue<Record<string, boolean>>;
-  makeItemFixed: (key: string) => void;
-  removeFixedItem: (key: string) => void;
+  activeHandleMeasurements: SharedValue<MeasuredDimensions | null>;
+  activeHandleOffset: SharedValue<Vector | null>;
+  registerHandle: (
+    key: string,
+    handleRef: AnimatedRef<View>,
+    fixed: boolean
+  ) => () => void;
+  updateActiveHandleMeasurements: (key: string) => void;
 };
 
 // PORTAL