diff --git a/packages/block-editor/src/components/rich-text/format-toolbar-container.js b/packages/block-editor/src/components/rich-text/format-toolbar-container.js
index fc857311706db..15fb664ded11d 100644
--- a/packages/block-editor/src/components/rich-text/format-toolbar-container.js
+++ b/packages/block-editor/src/components/rich-text/format-toolbar-container.js
@@ -9,31 +9,7 @@ import { Popover } from '@wordpress/components';
 import BlockFormatControls from '../block-format-controls';
 import FormatToolbar from './format-toolbar';
 
-function getAnchorRect( anchorObj ) {
-	const { current } = anchorObj;
-	const rect = current.getBoundingClientRect();
-
-	// Add some space.
-	const buffer = 6;
-
-	// Subtract padding if any.
-	let { paddingTop } = window.getComputedStyle( current );
-
-	paddingTop = parseInt( paddingTop, 10 );
-
-	return {
-		x: rect.left,
-		y: rect.top + paddingTop - buffer,
-		width: rect.width,
-		height: rect.height - paddingTop + buffer,
-		left: rect.left,
-		right: rect.right,
-		top: rect.top + paddingTop - buffer,
-		bottom: rect.bottom,
-	};
-}
-
-const FormatToolbarContainer = ( { inline, anchorObj } ) => {
+const FormatToolbarContainer = ( { inline, anchorRef } ) => {
 	if ( inline ) {
 		// Render in popover
 		return (
@@ -41,7 +17,8 @@ const FormatToolbarContainer = ( { inline, anchorObj } ) => {
 				noArrow
 				position="top center"
 				focusOnMount={ false }
-				getAnchorRect={ () => getAnchorRect( anchorObj ) }
+				anchorVerticalBuffer={ 6 }
+				anchorRef={ anchorRef }
 				className="block-editor-rich-text__inline-format-toolbar"
 			>
 				<FormatToolbar />
diff --git a/packages/block-editor/src/components/rich-text/index.js b/packages/block-editor/src/components/rich-text/index.js
index 63ceffb803232..ca047c7467308 100644
--- a/packages/block-editor/src/components/rich-text/index.js
+++ b/packages/block-editor/src/components/rich-text/index.js
@@ -404,7 +404,7 @@ class RichTextWrapper extends Component {
 				{ ( { isSelected, value, onChange, Editable } ) =>
 					<>
 						{ children && children( { value, onChange } ) }
-						{ isSelected && hasFormats && ( <FormatToolbarContainer inline={ inlineToolbar } anchorObj={ this.ref } /> ) }
+						{ isSelected && hasFormats && ( <FormatToolbarContainer inline={ inlineToolbar } anchorRef={ this.ref.current } /> ) }
 						{ isSelected && <RemoveBrowserShortcuts /> }
 						<Autocomplete
 							onReplace={ onReplace }
diff --git a/packages/components/src/autocomplete/index.js b/packages/components/src/autocomplete/index.js
index b12f97b30367b..b95bdf641252c 100644
--- a/packages/components/src/autocomplete/index.js
+++ b/packages/components/src/autocomplete/index.js
@@ -18,7 +18,6 @@ import {
 	isCollapsed,
 	getTextContent,
 } from '@wordpress/rich-text';
-import { getRectangleFromRange } from '@wordpress/dom';
 
 /**
  * Internal dependencies
@@ -130,13 +129,9 @@ function filterOptions( search, options = [], maxResults = 10 ) {
 	return filtered;
 }
 
-function getCaretRect() {
+function getRange() {
 	const selection = window.getSelection();
-	const range = selection.rangeCount ? selection.getRangeAt( 0 ) : null;
-
-	if ( range ) {
-		return getRectangleFromRange( range );
-	}
+	return selection.rangeCount ? selection.getRangeAt( 0 ) : null;
 }
 
 export class Autocomplete extends Component {
@@ -426,7 +421,7 @@ export class Autocomplete extends Component {
 						onClose={ this.reset }
 						position="top right"
 						className="components-autocomplete__popover"
-						getAnchorRect={ getCaretRect }
+						anchorRef={ getRange() }
 					>
 						<div
 							id={ listBoxId }
diff --git a/packages/components/src/popover/index.js b/packages/components/src/popover/index.js
index 420ac78b76bdb..790a6a84b2fe2 100644
--- a/packages/components/src/popover/index.js
+++ b/packages/components/src/popover/index.js
@@ -7,7 +7,7 @@ import classnames from 'classnames';
  * WordPress dependencies
  */
 import { useRef, useState, useEffect } from '@wordpress/element';
-import { focus } from '@wordpress/dom';
+import { focus, getRectangleFromRange } from '@wordpress/dom';
 import { ESCAPE } from '@wordpress/keycodes';
 import isShallowEqual from '@wordpress/is-shallow-equal';
 import deprecated from '@wordpress/deprecated';
@@ -62,48 +62,135 @@ function useThrottledWindowScrollOrResize( handler, ignoredScrollableRef ) {
 	}, [] );
 }
 
+function computeAnchorRect(
+	anchorRefFallback,
+	anchorRect,
+	getAnchorRect,
+	anchorRef = false,
+	anchorIncludePadding
+) {
+	if ( anchorRect ) {
+		return anchorRect;
+	}
+
+	if ( getAnchorRect ) {
+		if ( ! anchorRefFallback.current ) {
+			return;
+		}
+
+		return getAnchorRect( anchorRefFallback.current );
+	}
+
+	if ( anchorRef !== false ) {
+		if ( ! anchorRef ) {
+			return;
+		}
+
+		if ( anchorRef instanceof window.Range ) {
+			return getRectangleFromRange( anchorRef );
+		}
+
+		const rect = anchorRef.getBoundingClientRect();
+
+		if ( anchorIncludePadding ) {
+			return rect;
+		}
+
+		return withoutPadding( rect, anchorRef );
+	}
+
+	if ( ! anchorRefFallback.current ) {
+		return;
+	}
+
+	const { parentNode } = anchorRefFallback.current;
+	const rect = parentNode.getBoundingClientRect();
+
+	if ( anchorIncludePadding ) {
+		return rect;
+	}
+
+	return withoutPadding( rect, parentNode );
+}
+
+function addBuffer( rect, verticalBuffer = 0, horizontalBuffer = 0 ) {
+	return {
+		x: rect.left - horizontalBuffer,
+		y: rect.top - verticalBuffer,
+		width: rect.width + ( 2 * horizontalBuffer ),
+		height: rect.height + ( 2 * verticalBuffer ),
+		left: rect.left - horizontalBuffer,
+		right: rect.right + horizontalBuffer,
+		top: rect.top - verticalBuffer,
+		bottom: rect.bottom + verticalBuffer,
+	};
+}
+
+function withoutPadding( rect, element ) {
+	const {
+		paddingTop,
+		paddingBottom,
+		paddingLeft,
+		paddingRight,
+	} = window.getComputedStyle( element );
+	const top = paddingTop ? parseInt( paddingTop, 10 ) : 0;
+	const bottom = paddingBottom ? parseInt( paddingBottom, 10 ) : 0;
+	const left = paddingLeft ? parseInt( paddingLeft, 10 ) : 0;
+	const right = paddingRight ? parseInt( paddingRight, 10 ) : 0;
+
+	return {
+		x: rect.left + left,
+		y: rect.top + top,
+		width: rect.width - left - right,
+		height: rect.height - top - bottom,
+		left: rect.left + left,
+		right: rect.right - right,
+		top: rect.top + top,
+		bottom: rect.bottom - bottom,
+	};
+}
+
 /**
  * Hook used to compute and update the anchor position properly.
  *
- * @param {Object} anchorRef       reference to the popover anchor element.
- * @param {Object} contentRef      reference to the popover content element.
- * @param {Object} anchorRect      anchor Rect prop used to override the computed value.
- * @param {Function} getAnchorRect function used to override the anchor value computation algorithm.
+ * @param {Object}   anchorRefFallback      Reference to the popover anchor fallback element.
+ * @param {Object}   contentRef             Reference to the popover content element.
+ * @param {Object}   anchorRect             Anchor Rect prop used to override the computed value.
+ * @param {Function} getAnchorRect          Function used to override the anchor value computation algorithm.
+ * @param {Object}   anchorRef              Reference to the popover anchor fallback element.
+ * @param {Object}   anchorIncludePadding   Whether to include the anchor padding.
+ * @param {Object}   anchorVerticalBuffer   Vertical buffer for the anchor.
+ * @param {Object}   anchorHorizontalBuffer Horizontal buffer for the anchor.
  *
  * @return {Object} Anchor position.
  */
-function useAnchor( anchorRef, contentRef, anchorRect, getAnchorRect ) {
+function useAnchor(
+	anchorRefFallback,
+	contentRef,
+	anchorRect,
+	getAnchorRect,
+	anchorRef,
+	anchorIncludePadding,
+	anchorVerticalBuffer,
+	anchorHorizontalBuffer
+) {
 	const [ anchor, setAnchor ] = useState( null );
 	const refreshAnchorRect = () => {
-		if ( ! anchorRef.current ) {
-			return;
-		}
+		let newAnchor = computeAnchorRect(
+			anchorRefFallback,
+			anchorRect,
+			getAnchorRect,
+			anchorRef,
+			anchorIncludePadding
+		);
 
-		let newAnchor;
-		if ( anchorRect ) {
-			newAnchor = anchorRect;
-		} else if ( getAnchorRect ) {
-			newAnchor = getAnchorRect( anchorRef.current );
-		} else {
-			const rect = anchorRef.current.parentNode.getBoundingClientRect();
-			// subtract padding
-			const { paddingTop, paddingBottom } = window.getComputedStyle( anchorRef.current.parentNode );
-			const topPad = parseInt( paddingTop, 10 );
-			const bottomPad = parseInt( paddingBottom, 10 );
-			newAnchor = {
-				x: rect.left,
-				y: rect.top + topPad,
-				width: rect.width,
-				height: rect.height - topPad - bottomPad,
-				left: rect.left,
-				right: rect.right,
-				top: rect.top + topPad,
-				bottom: rect.bottom - bottomPad,
-			};
-		}
+		newAnchor = addBuffer(
+			newAnchor,
+			anchorVerticalBuffer,
+			anchorHorizontalBuffer
+		);
 
-		const didAnchorRectChange = ! isShallowEqual( newAnchor, anchor );
-		if ( didAnchorRectChange ) {
+		if ( ! isShallowEqual( newAnchor, anchor ) ) {
 			setAnchor( newAnchor );
 		}
 	};
@@ -259,6 +346,10 @@ const Popover = ( {
 	position = 'top',
 	range,
 	focusOnMount = 'firstElement',
+	anchorRef,
+	anchorIncludePadding,
+	anchorVerticalBuffer,
+	anchorHorizontalBuffer,
 	anchorRect,
 	getAnchorRect,
 	expandOnMobile,
@@ -268,14 +359,23 @@ const Popover = ( {
 	/* eslint-enable no-unused-vars */
 	...contentProps
 } ) => {
-	const anchorRef = useRef( null );
+	const anchorRefFallback = useRef( null );
 	const contentRef = useRef( null );
 
 	// Animation
 	const [ isReadyToAnimate, setIsReadyToAnimate ] = useState( false );
 
 	// Anchor position
-	const anchor = useAnchor( anchorRef, contentRef, anchorRect, getAnchorRect );
+	const anchor = useAnchor(
+		anchorRefFallback,
+		contentRef,
+		anchorRect,
+		getAnchorRect,
+		anchorRef,
+		anchorIncludePadding,
+		anchorVerticalBuffer,
+		anchorHorizontalBuffer
+	);
 
 	// Content size
 	const contentSize = useInitialContentSize( contentRef );
@@ -438,7 +538,7 @@ const Popover = ( {
 				}
 
 				return (
-					<span ref={ anchorRef }>
+					<span ref={ anchorRefFallback }>
 						{ content }
 						{ popoverPosition.isMobile && expandOnMobile && <ScrollLock /> }
 					</span>
diff --git a/packages/format-library/src/image/index.js b/packages/format-library/src/image/index.js
index 15cea7ecee4e3..c7a46a804b636 100644
--- a/packages/format-library/src/image/index.js
+++ b/packages/format-library/src/image/index.js
@@ -3,11 +3,10 @@
  */
 import { Path, SVG, TextControl, Popover, IconButton } from '@wordpress/components';
 import { __ } from '@wordpress/i18n';
-import { Component, useMemo } from '@wordpress/element';
+import { Component } from '@wordpress/element';
 import { insertObject } from '@wordpress/rich-text';
 import { MediaUpload, RichTextToolbarButton, MediaUploadCheck } from '@wordpress/block-editor';
 import { LEFT, RIGHT, UP, DOWN, BACKSPACE, ENTER } from '@wordpress/keycodes';
-import { computeCaretRect } from '@wordpress/dom';
 
 const ALLOWED_MEDIA_TYPES = [ 'image' ];
 
@@ -16,16 +15,10 @@ const title = __( 'Inline image' );
 
 const stopKeyPropagation = ( event ) => event.stopPropagation();
 
-const PopoverAtImage = ( { dependencies, ...props } ) => {
-	return (
-		<Popover
-			position="bottom center"
-			focusOnMount={ false }
-			anchorRect={ useMemo( () => computeCaretRect(), dependencies ) }
-			{ ...props }
-		/>
-	);
-};
+function getRange() {
+	const selection = window.getSelection();
+	return selection.rangeCount ? selection.getRangeAt( 0 ) : null;
+}
 
 export const image = {
 	name,
@@ -93,7 +86,6 @@ export const image = {
 
 		render() {
 			const { value, onChange, isObjectActive, activeObjectAttributes } = this.props;
-			const { style } = activeObjectAttributes;
 
 			return (
 				<MediaUploadCheck>
@@ -124,10 +116,10 @@ export const image = {
 						} }
 					/> }
 					{ isObjectActive &&
-						<PopoverAtImage
-							// Reposition Popover when the selection changes or
-							// when the width changes.
-							dependencies={ [ style, value.start ] }
+						<Popover
+							position="bottom center"
+							focusOnMount={ false }
+							anchorRef={ getRange() }
 						>
 							{ // Disable reason: KeyPress must be suppressed so the block doesn't hide the toolbar
 							/* eslint-disable jsx-a11y/no-noninteractive-element-interactions */ }
@@ -165,7 +157,7 @@ export const image = {
 								<IconButton icon="editor-break" label={ __( 'Apply' ) } type="submit" />
 							</form>
 							{ /* eslint-enable jsx-a11y/no-noninteractive-element-interactions */ }
-						</PopoverAtImage>
+						</Popover>
 					}
 				</MediaUploadCheck>
 			);
diff --git a/packages/format-library/src/link/inline.js b/packages/format-library/src/link/inline.js
index bf1784db2132d..34c05ad432c0c 100644
--- a/packages/format-library/src/link/inline.js
+++ b/packages/format-library/src/link/inline.js
@@ -8,7 +8,6 @@ import {
 	withSpokenMessages,
 } from '@wordpress/components';
 import { LEFT, RIGHT, UP, DOWN, BACKSPACE, ENTER } from '@wordpress/keycodes';
-import { getRectangleFromRange } from '@wordpress/dom';
 import { prependHTTP } from '@wordpress/url';
 import {
 	create,
@@ -32,15 +31,17 @@ function isShowingInput( props, state ) {
 }
 
 const URLPopoverAtLink = ( { isActive, addingLink, value, ...props } ) => {
-	const anchorRect = useMemo( () => {
+	const anchorRef = useMemo( () => {
 		const selection = window.getSelection();
-		const range = selection.rangeCount > 0 ? selection.getRangeAt( 0 ) : null;
-		if ( ! range ) {
+
+		if ( ! selection.rangeCount ) {
 			return;
 		}
 
+		const range = selection.getRangeAt( 0 );
+
 		if ( addingLink ) {
-			return getRectangleFromRange( range );
+			return range;
 		}
 
 		let element = range.startContainer;
@@ -52,17 +53,14 @@ const URLPopoverAtLink = ( { isActive, addingLink, value, ...props } ) => {
 			element = element.parentNode;
 		}
 
-		const closest = element.closest( 'a' );
-		if ( closest ) {
-			return closest.getBoundingClientRect();
-		}
+		return element.closest( 'a' );
 	}, [ isActive, addingLink, value.start, value.end ] );
 
-	if ( ! anchorRect ) {
+	if ( ! anchorRef ) {
 		return null;
 	}
 
-	return <URLPopover anchorRect={ anchorRect } { ...props } />;
+	return <URLPopover anchorRef={ anchorRef } { ...props } />;
 };
 
 class InlineLinkUI extends Component {
diff --git a/packages/nux/src/components/dot-tip/index.js b/packages/nux/src/components/dot-tip/index.js
index ed8119cb0d115..48bad4d0ae0c7 100644
--- a/packages/nux/src/components/dot-tip/index.js
+++ b/packages/nux/src/components/dot-tip/index.js
@@ -7,13 +7,6 @@ import { __ } from '@wordpress/i18n';
 import { withSelect, withDispatch } from '@wordpress/data';
 import { useCallback, useRef } from '@wordpress/element';
 
-function getAnchorRect( anchor ) {
-	// The default getAnchorRect() excludes an element's top and bottom padding
-	// from its calculation. We want tips to point to the outer margin of an
-	// element, so we override getAnchorRect() to include all padding.
-	return anchor.parentNode.getBoundingClientRect();
-}
-
 function onClick( event ) {
 	// Tips are often nested within buttons. We stop propagation so that clicking
 	// on a tip doesn't result in the button being clicked.
@@ -29,13 +22,6 @@ export function DotTip( {
 	onDisable,
 } ) {
 	const anchorParent = useRef( null );
-	const getAnchorRectCallback = useCallback(
-		( anchor ) => {
-			anchorParent.current = anchor.parentNode;
-			return getAnchorRect( anchor );
-		},
-		[ anchorParent ]
-	);
 	const onFocusOutsideCallback = useCallback(
 		( event ) => {
 			if ( ! anchorParent.current ) {
@@ -58,7 +44,7 @@ export function DotTip( {
 			position={ position }
 			noArrow
 			focusOnMount="container"
-			getAnchorRect={ getAnchorRectCallback }
+			anchorIncludePadding
 			role="dialog"
 			aria-label={ __( 'Editor tips' ) }
 			onClick={ onClick }
diff --git a/packages/nux/src/components/dot-tip/test/__snapshots__/index.js.snap b/packages/nux/src/components/dot-tip/test/__snapshots__/index.js.snap
index c7e04b79aaf56..844af24efddb4 100644
--- a/packages/nux/src/components/dot-tip/test/__snapshots__/index.js.snap
+++ b/packages/nux/src/components/dot-tip/test/__snapshots__/index.js.snap
@@ -2,10 +2,10 @@
 
 exports[`DotTip should render correctly 1`] = `
 <Popover
+  anchorIncludePadding={true}
   aria-label="Editor tips"
   className="nux-dot-tip"
   focusOnMount="container"
-  getAnchorRect={[Function]}
   noArrow={true}
   onClick={[Function]}
   onFocusOutside={[Function]}