diff --git a/subsys/bluetooth/audio/ascs.c b/subsys/bluetooth/audio/ascs.c
index 23ed2bb85ce..ef6d7f7853b 100644
--- a/subsys/bluetooth/audio/ascs.c
+++ b/subsys/bluetooth/audio/ascs.c
@@ -1012,7 +1012,9 @@ static void ase_stream_add(struct bt_ascs *ascs, struct bt_ascs_ase *ase,
 {
 	LOG_DBG("ase %p stream %p", ase, stream);
 	ase->ep.stream = stream;
-	stream->conn = ascs->conn;
+	if (stream->conn == NULL) {
+		stream->conn = bt_conn_ref(ascs->conn);
+	}
 	stream->ep = &ase->ep;
 }
 
diff --git a/subsys/bluetooth/audio/stream.c b/subsys/bluetooth/audio/stream.c
index 1c29b12a1ba..b22b46d3c11 100644
--- a/subsys/bluetooth/audio/stream.c
+++ b/subsys/bluetooth/audio/stream.c
@@ -81,7 +81,9 @@ void bt_audio_stream_attach(struct bt_conn *conn,
 	if (conn != NULL) {
 		__ASSERT(stream->conn == NULL || stream->conn == conn,
 			 "stream->conn %p already attached", stream->conn);
-		stream->conn = bt_conn_ref(conn);
+		if (stream->conn == NULL) {
+			stream->conn = bt_conn_ref(conn);
+		}
 	}
 	stream->codec = codec;
 	stream->ep = ep;