Bluetooth: Controller: Introduce prepare deferred feature

subsys/bluetooth/controller/flash/soc_flash_nrf_ticker.c

@@ -26,14 +26,13 @@
 #define FLASH_SYNC_SWITCHING_TIME (FLASH_RADIO_ABORT_DELAY_US +\
 				   FLASH_RADIO_WORK_DELAY_US)
 
-struct ticker_sync_context {
-	uint32_t interval;    /* timeslot interval. */
-	uint32_t slot;        /* timeslot length. */
-	uint32_t ticks_begin; /* timeslot begin timestamp */
+static struct {
+	uint32_t interval;       /* timeslot interval. */
+	uint32_t slot;           /* timeslot length. */
+	uint32_t ticks_begin;    /* timeslot begin timestamp */
+	uint32_t ticks_previous; /* timeslot previous reference */
 	int result;
-};
-
-static struct ticker_sync_context _ticker_sync_context;
+} _ticker_sync_context;
 
 /* semaphore for synchronization of flash operations */
 static struct k_sem sem_sync;
@@ -151,6 +150,7 @@ static void time_slot_callback_abort(uint32_t ticks_at_expire,
 				     void *context)
 {
 	ll_radio_state_abort();
+
 	time_slot_delay(ticks_at_expire,
 			HAL_TICKER_US_TO_TICKS(FLASH_RADIO_WORK_DELAY_US),
 			time_slot_callback_work,
@@ -163,6 +163,18 @@ static void time_slot_callback_prepare(uint32_t ticks_at_expire,
 				       uint16_t lazy, uint8_t force,
 				       void *context)
 {
+	uint32_t ticks_elapsed;
+	uint32_t ticks_now;
+
+	/* Skip flash operation if in the past, this can be when requested in the past due to
+	 * flash operations requested too often.
+	 */
+	ticks_now = ticker_ticks_now_get();
+	ticks_elapsed = ticker_ticks_diff_get(ticks_now, ticks_at_expire);
+	if (ticks_elapsed > HAL_TICKER_US_TO_TICKS(FLASH_RADIO_WORK_DELAY_US)) {
+		return;
+	}
+
 #if defined(CONFIG_BT_CTLR_LOW_LAT)
 	time_slot_callback_abort(ticks_at_expire, ticks_drift, remainder, lazy,
 				 force, context);
@@ -178,6 +190,7 @@ static void time_slot_callback_prepare(uint32_t ticks_at_expire,
 int nrf_flash_sync_init(void)
 {
 	k_sem_init(&sem_sync, 0, 1);
+
 	return 0;
 }
 
@@ -193,10 +206,28 @@ void nrf_flash_sync_set_context(uint32_t duration)
 int nrf_flash_sync_exe(struct flash_op_desc *op_desc)
 {
 	uint8_t instance_index;
+	uint32_t ticks_elapsed;
+	uint32_t ticks_slot;
+	uint32_t ticks_now;
 	uint8_t ticker_id;
 	uint32_t ret;
 	int result;
 
+	/* Check if flash operation request too often that can starve connection events.
+	 * We will request to start ticker in the past so that it does not `force` itself onto
+	 * scheduling by causing connection events to be skipped.
+	 */
+	ticks_now = ticker_ticks_now_get();
+	ticks_elapsed = ticker_ticks_diff_get(ticks_now, _ticker_sync_context.ticks_previous);
+	_ticker_sync_context.ticks_previous = ticks_now;
+	ticks_slot = HAL_TICKER_US_TO_TICKS(_ticker_sync_context.slot);
+	if (ticks_elapsed < ticks_slot) {
+		uint32_t ticks_interval  = HAL_TICKER_US_TO_TICKS(_ticker_sync_context.interval);
+
+		/* Set ticker start reference one flash slot interval in the past */
+		ticks_now = ticker_ticks_diff_get(ticks_now, ticks_interval);
+	}
+
 	/* Get the ticker instance and ticker id for flash operations */
 	ll_timeslice_ticker_id_get(&instance_index, &ticker_id);
 
@@ -205,7 +236,7 @@ int nrf_flash_sync_exe(struct flash_op_desc *op_desc)
 			   3, /* user id for thread mode */
 			      /* (MAYFLY_CALL_ID_PROGRAM) */
 			   ticker_id, /* flash ticker id */
-			   ticker_ticks_now_get(), /* current tick */
+			   ticks_now, /* current tick */
 			   0, /* first int. immediately */
 			   /* period */
 			   HAL_TICKER_US_TO_TICKS(

subsys/bluetooth/controller/ll_sw/lll.h

@@ -236,7 +236,8 @@ struct lll_prepare_param {
 #if defined(CONFIG_BT_CTLR_JIT_SCHEDULING)
 	int8_t  prio;
 #endif /* CONFIG_BT_CTLR_JIT_SCHEDULING */
-	uint8_t force;
+	uint8_t force:1;
+	uint8_t defer:1;
 	void *param;
 };
 

subsys/bluetooth/controller/ll_sw/lll_common.c

@@ -62,6 +62,8 @@ int lll_prepare(lll_is_abort_cb_t is_abort_cb, lll_abort_cb_t abort_cb,
 	prepare_param->prio = prio;
 #endif /* CONFIG_BT_CTLR_JIT_SCHEDULING */
 
+	prepare_param->defer = 0U;
+
 	err = lll_prepare_resolve(is_abort_cb, abort_cb, prepare_cb, prepare_param, 0U, 0U);
 
 	return err;

subsys/bluetooth/controller/ll_sw/nordic/lll/lll.c

@@ -807,6 +807,8 @@ void lll_isr_early_abort(void *param)
 {
 	int err;
 
+	radio_status_reset();
+
 	radio_isr_set(isr_race, param);
 	if (!radio_is_idle()) {
 		radio_disable();
@@ -1316,7 +1318,9 @@ static void preempt(void *param)
 		/* Returns -EBUSY when same curr and next state/role, do not
 		 * abort same curr and next event.
 		 */
-		if (err != -EBUSY) {
+		if (err == -EBUSY) {
+			ready->prepare_param.defer = 1U;
+		} else {
 			/* Let preemptor LLL know about the cancelled prepare */
 			ready->is_aborted = 1;
 			ready->abort_cb(&ready->prepare_param, ready->prepare_param.param);

subsys/bluetooth/controller/ll_sw/nordic/lll/lll_central.c

@@ -247,12 +247,21 @@ static int prepare_cb(struct lll_prepare_param *p)
 	overhead = lll_preempt_calc(ull, (TICKER_ID_CONN_BASE + lll->handle), ticks_at_event);
 	/* check if preempt to start has changed */
 	if (overhead) {
-		LL_ASSERT_OVERHEAD(overhead);
+		int err;
+
+		if (p->defer == 1U) {
+			/* We accept the overlap as previous event elected to continue */
+			err = 0;
+		} else {
+			LL_ASSERT_OVERHEAD(overhead);
+
+			err = -ECANCELED;
+		}
 
 		radio_isr_set(lll_isr_abort, lll);
 		radio_disable();
 
-		return -ECANCELED;
+		return err;
 	}
 #endif /* !CONFIG_BT_CTLR_XTAL_ADVANCED */
 

subsys/bluetooth/controller/ll_sw/nordic/lll/lll_clock.c

@@ -127,7 +127,7 @@ int lll_hfclock_on_wait(void)
 
 int lll_hfclock_off(void)
 {
-	if (hf_refcnt < 1) {
+	if (atomic_get(&hf_refcnt) < 1) {
 		return -EALREADY;
 	}
 

subsys/bluetooth/controller/ll_sw/nordic/lll/lll_conn.c

@@ -69,6 +69,7 @@ static uint8_t crc_valid;
 static uint8_t is_aborted;
 static uint16_t tx_cnt;
 static uint16_t trx_cnt;
+static uint8_t trx_busy_iteration;
 
 #if defined(CONFIG_BT_CTLR_LE_ENC)
 static uint8_t mic_state;
@@ -153,27 +154,38 @@ void lll_conn_prepare_reset(void)
 	crc_valid = 0U;
 	crc_expire = 0U;
 	is_aborted = 0U;
+	trx_busy_iteration = 0U;
 
 #if defined(CONFIG_BT_CTLR_LE_ENC)
 	mic_state = LLL_CONN_MIC_NONE;
 #endif /* CONFIG_BT_CTLR_LE_ENC */
 }
 
 #if defined(CONFIG_BT_CENTRAL)
+/* Number of times central event being aborted by same event instance be skipped */
+/* FIXME: Increasing this causes event pipeline overflow assertion, add LLL implementation to
+ *        gracefully abort the deferred next event when -EBUSY is returned in this is_abort_cb
+ *        interface.
+ */
+#define CENTRAL_TRX_BUSY_ITERATION_MAX 1
+
 int lll_conn_central_is_abort_cb(void *next, void *curr,
 				 lll_prepare_cb_t *resume_cb)
 {
 	struct lll_conn *lll = curr;
 
-	/* Do not abort if near supervision timeout */
-	if (lll->forced) {
-		return 0;
-	}
+	if (next != curr) {
+		/* Do not be aborted by a different event if near supervision timeout */
+		if ((lll->forced == 1U) && (trx_cnt < 1U)) {
+			return 0;
+		}
 
-	/* Do not be aborted by same event if a single central trx has not been
-	 * exchanged.
-	 */
-	if ((next == curr) && (trx_cnt < 1U)) {
+	} else if ((trx_cnt < 1U) && (trx_busy_iteration < CENTRAL_TRX_BUSY_ITERATION_MAX)) {
+		trx_busy_iteration++;
+
+		/* Do not be aborted by same event if a single central's Rx has not completed.
+		 * Cases where single trx duration can be greater than connection interval.
+		 */
 		return -EBUSY;
 	}
 
@@ -182,20 +194,30 @@ int lll_conn_central_is_abort_cb(void *next, void *curr,
 #endif /* CONFIG_BT_CENTRAL */
 
 #if defined(CONFIG_BT_PERIPHERAL)
+/* Number of times peripheral event being aborted by same event instance be skipped */
+/* FIXME: Increasing this causes event pipeline overflow assertion, add LLL implementation to
+ *        gracefully abort the deferred next event when -EBUSY is returned in this is_abort_cb
+ *        interface.
+ */
+#define PERIPHERAL_TRX_BUSY_ITERATION_MAX 1
+
 int lll_conn_peripheral_is_abort_cb(void *next, void *curr,
 				    lll_prepare_cb_t *resume_cb)
 {
 	struct lll_conn *lll = curr;
 
-	/* Do not abort if near supervision timeout */
-	if (lll->forced) {
-		return 0;
-	}
+	if (next != curr) {
+		/* Do not be aborted by a different event if near supervision timeout */
+		if ((lll->forced == 1U) && (tx_cnt < 1U)) {
+			return 0;
+		}
 
-	/* Do not be aborted by same event if a single peripheral trx has not
-	 * been exchanged.
-	 */
-	if ((next == curr) && (tx_cnt < 1U)) {
+	} else if ((tx_cnt < 1U) && (trx_busy_iteration < PERIPHERAL_TRX_BUSY_ITERATION_MAX)) {
+		trx_busy_iteration++;
+
+		/* Do not be aborted by same event if a single peripheral's Tx has not completed.
+		 * Cases where single trx duration can be greater than connection interval.
+		 */
 		return -EBUSY;
 	}
 

subsys/bluetooth/controller/ll_sw/nordic/lll/lll_peripheral.c

@@ -339,12 +339,21 @@ static int prepare_cb(struct lll_prepare_param *p)
 	overhead = lll_preempt_calc(ull, (TICKER_ID_CONN_BASE + lll->handle), ticks_at_event);
 	/* check if preempt to start has changed */
 	if (overhead) {
-		LL_ASSERT_OVERHEAD(overhead);
+		int err;
+
+		if (p->defer == 1U) {
+			/* We accept the overlap as previous event elected to continue */
+			err = 0;
+		} else {
+			LL_ASSERT_OVERHEAD(overhead);
+
+			err = -ECANCELED;
+		}
 
 		radio_isr_set(lll_isr_abort, lll);
 		radio_disable();
 
-		return -ECANCELED;
+		return err;
 	}
 #endif /* CONFIG_BT_CTLR_XTAL_ADVANCED */
 

subsys/bluetooth/controller/ll_sw/ull.c

@@ -2578,24 +2578,32 @@ static void rx_demux(void *param)
 	do {
 #endif /* CONFIG_BT_CTLR_LOW_LAT_ULL */
 		struct node_rx_hdr *rx;
-
-		link = memq_peek(memq_ull_rx.head, memq_ull_rx.tail,
-				 (void **)&rx);
-		if (link) {
 #if defined(CONFIG_BT_CONN)
-			struct node_tx *node_tx;
-			memq_link_t *link_tx;
-			uint16_t handle; /* Handle to Ack TX */
+		struct node_tx *tx;
+		memq_link_t *link_tx;
+		uint8_t ack_last;
+		uint16_t handle; /* Handle to Ack TX */
+
+		/* Save the ack_last, Tx Ack FIFO's last index to avoid the value being changed if
+		 * there were no Rx PDUs and we were pre-empted before calling
+		 * `rx_demux_conn_tx_ack()` in the `else` clause.
+		 */
+		link_tx = ull_conn_ack_peek(&ack_last, &handle, &tx);
+
+		/* Ensure that the value is fetched before call to memq_peek, i.e. compiler shall
+		 * not reorder memory write before above read.
+		 */
+		cpu_dmb();
 #endif /* CONFIG_BT_CONN */
 
+		link = memq_peek(memq_ull_rx.head, memq_ull_rx.tail, (void **)&rx);
+		if (link) {
 			LL_ASSERT(rx);
 
 #if defined(CONFIG_BT_CONN)
-			link_tx = ull_conn_ack_by_last_peek(rx->ack_last,
-							    &handle, &node_tx);
+			link_tx = ull_conn_ack_by_last_peek(rx->ack_last, &handle, &tx);
 			if (link_tx) {
-				rx_demux_conn_tx_ack(rx->ack_last, handle,
-						     link_tx, node_tx);
+				rx_demux_conn_tx_ack(rx->ack_last, handle, link_tx, tx);
 			} else
 #endif /* CONFIG_BT_CONN */
 			{
@@ -2607,21 +2615,14 @@ static void rx_demux(void *param)
 #endif /* !CONFIG_BT_CTLR_LOW_LAT_ULL */
 
 #if defined(CONFIG_BT_CONN)
-		} else {
-			struct node_tx *node_tx;
-			uint8_t ack_last;
-			uint16_t handle;
-
-			link = ull_conn_ack_peek(&ack_last, &handle, &node_tx);
-			if (link) {
-				rx_demux_conn_tx_ack(ack_last, handle,
-						      link, node_tx);
+		} else if (link_tx) {
+			rx_demux_conn_tx_ack(ack_last, handle, link_tx, tx);
 
 #if defined(CONFIG_BT_CTLR_LOW_LAT_ULL)
-				rx_demux_yield();
-#endif /* CONFIG_BT_CTLR_LOW_LAT_ULL */
-
-			}
+			rx_demux_yield();
+#else /* !CONFIG_BT_CTLR_LOW_LAT_ULL */
+			link = link_tx;
+#endif /* !CONFIG_BT_CTLR_LOW_LAT_ULL */
 #endif /* CONFIG_BT_CONN */
 		}