diff --git a/packages/block-library/src/button/edit.js b/packages/block-library/src/button/edit.js
index b0fcc297d6b9e..4b84458e99dd1 100644
--- a/packages/block-library/src/button/edit.js
+++ b/packages/block-library/src/button/edit.js
@@ -107,11 +107,12 @@ function URLPicker( {
 	onToggleOpenInNewTab,
 	anchorRef,
 } ) {
+	const toggleButtonRef = useRef();
 	const [ isURLPickerOpen, setIsURLPickerOpen ] = useState( false );
 	const urlIsSet = !! url;
 	const urlIsSetandSelected = urlIsSet && isSelected;
-	const openLinkControl = () => {
-		setIsURLPickerOpen( true );
+	const toggleLinkControl = () => {
+		setIsURLPickerOpen( ( value ) => ! value );
 		return false; // prevents default behaviour for event
 	};
 	const unlinkButton = () => {
@@ -125,7 +126,14 @@ function URLPicker( {
 	const linkControl = ( isURLPickerOpen || urlIsSetandSelected ) && (
 		<Popover
 			position="bottom center"
-			onClose={ () => setIsURLPickerOpen( false ) }
+			onClose={ () => {
+				if (
+					toggleButtonRef.current !==
+					toggleButtonRef.current.ownerDocument.activeElement
+				) {
+					setIsURLPickerOpen( false );
+				}
+			} }
 			anchorRef={ anchorRef?.current }
 		>
 			<LinkControl
@@ -153,7 +161,8 @@ function URLPicker( {
 						icon={ link }
 						title={ __( 'Link' ) }
 						shortcut={ displayShortcut.primary( 'k' ) }
-						onClick={ openLinkControl }
+						onClick={ toggleLinkControl }
+						ref={ toggleButtonRef }
 					/>
 				) }
 				{ urlIsSetandSelected && (
@@ -171,7 +180,7 @@ function URLPicker( {
 				<KeyboardShortcuts
 					bindGlobal
 					shortcuts={ {
-						[ rawShortcut.primary( 'k' ) ]: openLinkControl,
+						[ rawShortcut.primary( 'k' ) ]: toggleLinkControl,
 						[ rawShortcut.primaryShift( 'k' ) ]: unlinkButton,
 					} }
 				/>
diff --git a/packages/block-library/src/navigation-link/edit.js b/packages/block-library/src/navigation-link/edit.js
index 96bb68eff378b..5063f49710980 100644
--- a/packages/block-library/src/navigation-link/edit.js
+++ b/packages/block-library/src/navigation-link/edit.js
@@ -160,6 +160,7 @@ export default function NavigationLinkEdit( {
 	const isDraggingWithin = useIsDraggingWithin( listItemRef );
 	const itemLabelPlaceholder = __( 'Add link…' );
 	const ref = useRef();
+	const toggleButtonRef = useRef();
 
 	const {
 		isParentOfSelectedBlock,
@@ -370,7 +371,7 @@ export default function NavigationLinkEdit( {
 						bindGlobal
 						shortcuts={ {
 							[ rawShortcut.primary( 'k' ) ]: () =>
-								setIsLinkOpen( true ),
+								setIsLinkOpen( ( value ) => ! value ),
 						} }
 					/>
 					<ToolbarButton
@@ -378,7 +379,8 @@ export default function NavigationLinkEdit( {
 						icon={ linkIcon }
 						title={ __( 'Link' ) }
 						shortcut={ displayShortcut.primary( 'k' ) }
-						onClick={ () => setIsLinkOpen( true ) }
+						onClick={ () => setIsLinkOpen( ( value ) => ! value ) }
+						ref={ toggleButtonRef }
 					/>
 					<ToolbarButton
 						name="submenu"
@@ -468,7 +470,15 @@ export default function NavigationLinkEdit( {
 					{ isLinkOpen && (
 						<Popover
 							position="bottom center"
-							onClose={ () => setIsLinkOpen( false ) }
+							onClose={ () => {
+								if (
+									toggleButtonRef.current !==
+									toggleButtonRef.current.ownerDocument
+										.activeElement
+								) {
+									setIsLinkOpen( false );
+								}
+							} }
 							anchorRef={ listItemRef.current }
 						>
 							<KeyboardShortcuts
diff --git a/packages/block-library/src/post-date/edit.js b/packages/block-library/src/post-date/edit.js
index 4b75b93a648d3..6421beda2150c 100644
--- a/packages/block-library/src/post-date/edit.js
+++ b/packages/block-library/src/post-date/edit.js
@@ -7,7 +7,7 @@ import classnames from 'classnames';
  * WordPress dependencies
  */
 import { useEntityProp } from '@wordpress/core-data';
-import { useState } from '@wordpress/element';
+import { useState, useRef } from '@wordpress/element';
 import { __experimentalGetSettings, dateI18n } from '@wordpress/date';
 import {
 	AlignmentToolbar,
@@ -37,6 +37,7 @@ export default function PostDateEdit( { attributes, context, setAttributes } ) {
 		'date',
 		postId
 	);
+	const toggleButtonRef = useRef();
 	const [ isPickerOpen, setIsPickerOpen ] = useState( false );
 	const settings = __experimentalGetSettings();
 	// To know if the current time format is a 12 hour time, look for "a".
@@ -77,6 +78,7 @@ export default function PostDateEdit( { attributes, context, setAttributes } ) {
 						<ToolbarButton
 							icon={ edit }
 							title={ __( 'Change Date' ) }
+							dateToggleButton={ toggleButtonRef }
 							onClick={ () =>
 								setIsPickerOpen(
 									( _isPickerOpen ) => ! _isPickerOpen
@@ -112,7 +114,15 @@ export default function PostDateEdit( { attributes, context, setAttributes } ) {
 
 						{ isPickerOpen && (
 							<Popover
-								onClose={ setIsPickerOpen.bind( null, false ) }
+								onClose={ () => {
+									if (
+										toggleButtonRef.current !==
+										toggleButtonRef.current.ownerDocument
+											.activeElement
+									) {
+										setIsPickerOpen( false );
+									}
+								} }
 							>
 								<DateTimePicker
 									currentDate={ date }