diff --git a/library/src/main/java/me/zhanghai/android/fastscroll/FastScroller.java b/library/src/main/java/me/zhanghai/android/fastscroll/FastScroller.java
index 2849dc7..e993cce 100644
--- a/library/src/main/java/me/zhanghai/android/fastscroll/FastScroller.java
+++ b/library/src/main/java/me/zhanghai/android/fastscroll/FastScroller.java
@@ -294,14 +294,9 @@ private boolean onTouchEvent(@NonNull MotionEvent event) {
                 mDownX = eventX;
                 mDownY = eventY;
 
-                if (mTrackView.getAlpha() > 0 && isInView(mTrackView, eventX, eventY)) {
+                if (mThumbView.getAlpha() > 0 && isInViewTouchTarget(mThumbView, eventX, eventY)) {
                     mDragStartY = eventY;
-                    if (isInViewTouchTarget(mThumbView, eventX, eventY)) {
-                        mDragStartThumbOffset = mThumbOffset;
-                    } else {
-                        mDragStartThumbOffset = (int) (eventY - padding.top - mThumbHeight / 2f);
-                        scrollToThumbOffset(mDragStartThumbOffset);
-                    }
+                    mDragStartThumbOffset = mThumbOffset;
                     setDragging(true);
                 }
                 break;