diff --git a/subsys/bluetooth/host/hci_core.c b/subsys/bluetooth/host/hci_core.c
index aa8b6b71510..7ca50e3da4b 100644
--- a/subsys/bluetooth/host/hci_core.c
+++ b/subsys/bluetooth/host/hci_core.c
@@ -1339,7 +1339,12 @@ static void enh_conn_complete(struct bt_hci_evt_le_enh_conn_complete *evt)
 
 #if defined(CONFIG_BT_SMP)
 	if (atomic_test_and_clear_bit(bt_dev.flags, BT_DEV_ID_PENDING)) {
-		bt_keys_foreach(BT_KEYS_IRK, update_pending_id, NULL);
+		if (IS_ENABLED(CONFIG_BT_CENTRAL) &&
+		    IS_ENABLED(CONFIG_BT_PRIVACY)) {
+			bt_keys_foreach(BT_KEYS_ALL, update_pending_id, NULL);
+		} else {
+			bt_keys_foreach(BT_KEYS_IRK, update_pending_id, NULL);
+		}
 	}
 #endif
 
@@ -3178,7 +3183,9 @@ done:
 
 static void keys_add_id(struct bt_keys *keys, void *data)
 {
-	hci_id_add(keys->id, &keys->addr, keys->irk.val);
+	if (keys != (struct bt_keys *)data) {
+		hci_id_add(keys->id, &keys->addr, keys->irk.val);
+	}
 }
 
 void bt_id_del(struct bt_keys *keys)
@@ -3229,8 +3236,12 @@ void bt_id_del(struct bt_keys *keys)
 	/* We checked size + 1 earlier, so here we know we can fit again */
 	if (bt_dev.le.rl_entries > bt_dev.le.rl_size) {
 		bt_dev.le.rl_entries--;
-		keys->keys &= ~BT_KEYS_IRK;
-		bt_keys_foreach(BT_KEYS_IRK, keys_add_id, NULL);
+		if (IS_ENABLED(CONFIG_BT_CENTRAL) &&
+		    IS_ENABLED(CONFIG_BT_PRIVACY)) {
+			bt_keys_foreach(BT_KEYS_ALL, keys_add_id, keys);
+		} else {
+			bt_keys_foreach(BT_KEYS_IRK, keys_add_id, keys);
+		}
 		goto done;
 	}
 
diff --git a/subsys/bluetooth/host/keys.c b/subsys/bluetooth/host/keys.c
index d3e4c9ce342..288e5e8823b 100644
--- a/subsys/bluetooth/host/keys.c
+++ b/subsys/bluetooth/host/keys.c
@@ -236,7 +236,8 @@ void bt_keys_clear(struct bt_keys *keys)
 {
 	BT_DBG("%s (keys 0x%04x)", bt_addr_le_str(&keys->addr), keys->keys);
 
-	if (keys->keys & BT_KEYS_IRK) {
+	if ((IS_ENABLED(CONFIG_BT_CENTRAL) && IS_ENABLED(CONFIG_BT_PRIVACY)) ||
+	    keys->keys & BT_KEYS_IRK) {
 		bt_id_del(keys);
 	}
 
@@ -393,7 +394,11 @@ static int keys_commit(void)
 	 * called multiple times for the same address, especially if
 	 * the keys were already removed.
 	 */
-	bt_keys_foreach(BT_KEYS_IRK, id_add, NULL);
+	if (IS_ENABLED(CONFIG_BT_CENTRAL) && IS_ENABLED(CONFIG_BT_PRIVACY)) {
+		bt_keys_foreach(BT_KEYS_ALL, id_add, NULL);
+	} else {
+		bt_keys_foreach(BT_KEYS_IRK, id_add, NULL);
+	}
 
 	return 0;
 }
diff --git a/subsys/bluetooth/host/smp.c b/subsys/bluetooth/host/smp.c
index 1c8faafe766..3ddec039f90 100644
--- a/subsys/bluetooth/host/smp.c
+++ b/subsys/bluetooth/host/smp.c
@@ -4405,6 +4405,16 @@ static void bt_smp_encrypt_change(struct bt_l2cap_chan *chan,
 		atomic_set_bit(&smp->allowed_cmds, BT_SMP_CMD_SIGNING_INFO);
 	}
 
+	if (IS_ENABLED(CONFIG_BT_CENTRAL) &&
+	    IS_ENABLED(CONFIG_BT_PRIVACY) &&
+	    !(smp->remote_dist & BT_SMP_DIST_ID_KEY)) {
+		/* To resolve directed advertising we need our local IRK
+		 * in the controllers resolving list, add it now since the
+		 * peer has no identity key.
+		 */
+		bt_id_add(conn->le.keys);
+	}
+
 	atomic_set_bit(smp->flags, SMP_FLAG_KEYS_DISTR);
 
 	/* Slave distributes it's keys first */