diff --git a/doc/connectivity/usb/device/usb_device.rst b/doc/connectivity/usb/device/usb_device.rst index e86f7ecc082..ea4ad88d7a1 100644 --- a/doc/connectivity/usb/device/usb_device.rst +++ b/doc/connectivity/usb/device/usb_device.rst @@ -603,6 +603,8 @@ The following Product IDs are currently used: +----------------------------------------------------+--------+ | :zephyr:code-sample:`uac2-explicit-feedback` | 0x000E | +----------------------------------------------------+--------+ +| :zephyr:code-sample:`uac2-implicit-feedback` | 0x000F | ++----------------------------------------------------+--------+ | :zephyr:code-sample:`usb-dfu` (DFU Mode) | 0xFFFF | +----------------------------------------------------+--------+ diff --git a/doc/connectivity/usb/device_next/usb_device.rst b/doc/connectivity/usb/device_next/usb_device.rst index fcb6b8cde7a..b73f7c0f6a8 100644 --- a/doc/connectivity/usb/device_next/usb_device.rst +++ b/doc/connectivity/usb/device_next/usb_device.rst @@ -55,6 +55,8 @@ Samples * :zephyr:code-sample:`uac2-explicit-feedback` +* :zephyr:code-sample:`uac2-implicit-feedback` + Samples ported to new USB device support ---------------------------------------- diff --git a/samples/subsys/usb/uac2_implicit_feedback/CMakeLists.txt b/samples/subsys/usb/uac2_implicit_feedback/CMakeLists.txt new file mode 100644 index 00000000000..26ce264c421 --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/CMakeLists.txt @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20.0) +find_package(Zephyr REQUIRED HINTS $ENV{ZEPHYR_BASE}) +project(usb_audio_async_i2s) + +include(${ZEPHYR_BASE}/samples/subsys/usb/common/common.cmake) +target_sources(app PRIVATE src/main.c) + +if (CONFIG_SOC_COMPATIBLE_NRF5340_CPUAPP) + target_sources(app PRIVATE src/feedback_nrf53.c) +else() + target_sources(app PRIVATE src/feedback_dummy.c) +endif() diff --git a/samples/subsys/usb/uac2_implicit_feedback/Kconfig b/samples/subsys/usb/uac2_implicit_feedback/Kconfig new file mode 100644 index 00000000000..24f8d41f7a1 --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/Kconfig @@ -0,0 +1,9 @@ +# Copyright (c) 2023-2024 Nordic Semiconductor ASA +# SPDX-License-Identifier: Apache-2.0 + +# Source common USB sample options used to initialize new experimental USB +# device stack. The scope of these options is limited to USB samples in project +# tree, you cannot use them in your own application. +source "samples/subsys/usb/common/Kconfig.sample_usbd" + +source "Kconfig.zephyr" diff --git a/samples/subsys/usb/uac2_implicit_feedback/README.rst b/samples/subsys/usb/uac2_implicit_feedback/README.rst new file mode 100644 index 00000000000..99e4bd72aa6 --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/README.rst @@ -0,0 +1,41 @@ +.. zephyr:code-sample:: uac2-implicit-feedback + :name: USB Audio asynchronous implicit feedback sample + :relevant-api: usbd_api uac2_device i2s_interface + + USB Audio 2 implicit feedback sample playing stereo and recording mono audio + on I2S interface. + +Overview +******** + +This sample demonstrates how to implement USB asynchronous bidirectional audio +with implicit feedback. The host adjusts number of stereo samples sent for +headphones playback based on the number of mono microphone samples received. + +Requirements +************ + +Target must be able to measure I2S block start (i.e. first sample from output +buffer gets out) relative to USB SOF. The relative offset must be reported with +single sample accuracy. + +This sample has been tested on :ref:`nrf5340dk_nrf5340`. While for actual audio +experience it is necessary to connect external I2S ADC and I2S DAC, simple echo +can be accomplished by shorting I2S data output with I2S data input. + +Theoretically it should be possible to obtain the timing information based on +I2S and USB interrupts, but currently neither subsystem currently provides +necessary timestamp information. + +Building and Running +******************** + +The code can be found in :zephyr_file:`samples/subsys/usb/uac2_implicit_feedback`. + +To build and flash the application: + +.. zephyr-app-commands:: + :zephyr-app: samples/subsys/usb/uac2_implicit_feedback + :board: nrf5340dk/nrf5340/cpuapp + :goals: build flash + :compact: diff --git a/samples/subsys/usb/uac2_implicit_feedback/app.overlay b/samples/subsys/usb/uac2_implicit_feedback/app.overlay new file mode 100644 index 00000000000..4e3409860c7 --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/app.overlay @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 Nordic Semiconductor ASA + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +/ { + uac2_headset: usb_audio2 { + compatible = "zephyr,uac2"; + status = "okay"; + audio-function = ; + + uac_aclk: aclk { + compatible = "zephyr,uac2-clock-source"; + clock-type = "internal-programmable"; + frequency-control = "host-programmable"; + sampling-frequencies = <48000>; + }; + + out_terminal: out_terminal { + compatible = "zephyr,uac2-input-terminal"; + clock-source = <&uac_aclk>; + terminal-type = ; + front-left; + front-right; + }; + + headphones_output: headphones { + compatible = "zephyr,uac2-output-terminal"; + data-source = <&out_terminal>; + clock-source = <&uac_aclk>; + terminal-type = ; + assoc-terminal = <&mic_input>; + }; + + mic_input: microphone { + compatible = "zephyr,uac2-input-terminal"; + clock-source = <&uac_aclk>; + terminal-type = ; + /* Circular reference, macros will figure it out and + * provide correct associated terminal ID because the + * terminals associations are always 1-to-1. + * + * assoc-terminal = <&headphones_output>; + */ + front-left; + }; + + in_terminal: in_terminal { + compatible = "zephyr,uac2-output-terminal"; + data-source = <&mic_input>; + clock-source = <&uac_aclk>; + terminal-type = ; + }; + + as_iso_out: out_interface { + compatible = "zephyr,uac2-audio-streaming"; + linked-terminal = <&out_terminal>; + implicit-feedback; + subslot-size = <2>; + bit-resolution = <16>; + }; + + as_iso_in: in_interface { + compatible = "zephyr,uac2-audio-streaming"; + linked-terminal = <&in_terminal>; + implicit-feedback; + subslot-size = <2>; + bit-resolution = <16>; + }; + }; +}; diff --git a/samples/subsys/usb/uac2_implicit_feedback/boards/nrf5340dk_nrf5340_cpuapp.conf b/samples/subsys/usb/uac2_implicit_feedback/boards/nrf5340dk_nrf5340_cpuapp.conf new file mode 100644 index 00000000000..ce67033381a --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/boards/nrf5340dk_nrf5340_cpuapp.conf @@ -0,0 +1,2 @@ +#Enable timer for asynchronous feedback +CONFIG_NRFX_TIMER2=y diff --git a/samples/subsys/usb/uac2_implicit_feedback/boards/nrf5340dk_nrf5340_cpuapp.overlay b/samples/subsys/usb/uac2_implicit_feedback/boards/nrf5340dk_nrf5340_cpuapp.overlay new file mode 100644 index 00000000000..4aeb4195356 --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/boards/nrf5340dk_nrf5340_cpuapp.overlay @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Nordic Semiconductor ASA + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "../app.overlay" + +&pinctrl { + i2s0_default_alt: i2s0_default_alt { + group1 { + psels = , + , + , + ; + }; + }; +}; + +&clock { + hfclkaudio-frequency = <12288000>; +}; + +i2s_rxtx: &i2s0 { + status = "okay"; + pinctrl-0 = <&i2s0_default_alt>; + pinctrl-names = "default"; + clock-source = "ACLK"; +}; diff --git a/samples/subsys/usb/uac2_implicit_feedback/prj.conf b/samples/subsys/usb/uac2_implicit_feedback/prj.conf new file mode 100644 index 00000000000..61d654a3b3a --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/prj.conf @@ -0,0 +1,12 @@ +CONFIG_I2S=y + +#USB related configs +CONFIG_USB_DEVICE_STACK_NEXT=y +CONFIG_USBD_AUDIO2_CLASS=y +CONFIG_SAMPLE_USBD_PID=0x000F +CONFIG_SAMPLE_USBD_PRODUCT="UAC2 implicit feedback sample" + +#LOG subsystem related configs +CONFIG_LOG=y +CONFIG_USBD_LOG_LEVEL_WRN=y +CONFIG_UDC_DRIVER_LOG_LEVEL_WRN=y diff --git a/samples/subsys/usb/uac2_implicit_feedback/sample.yaml b/samples/subsys/usb/uac2_implicit_feedback/sample.yaml new file mode 100644 index 00000000000..c82f1739692 --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/sample.yaml @@ -0,0 +1,10 @@ +sample: + name: USB Audio 2 asynchronous implicit feedback sample +tests: + sample.subsys.usb.uac2_implicit_feedback: + depends_on: + - usbd + - i2s + tags: usb i2s + platform_allow: nrf5340dk/nrf5340/cpuapp + harness: TBD diff --git a/samples/subsys/usb/uac2_implicit_feedback/src/feedback.h b/samples/subsys/usb/uac2_implicit_feedback/src/feedback.h new file mode 100644 index 00000000000..3fff2425d8b --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/src/feedback.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024 Nordic Semiconductor ASA + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef FEEDBACK_H_ +#define FEEDBACK_H_ + +#include + +/* Nominal number of samples received on each SOF. This sample is currently + * supporting only 48 kHz sample rate. + */ +#define SAMPLES_PER_SOF 48 + +struct feedback_ctx *feedback_init(void); +void feedback_reset_ctx(struct feedback_ctx *ctx); +void feedback_process(struct feedback_ctx *ctx); +void feedback_start(struct feedback_ctx *ctx, int i2s_blocks_queued); + +/* Return offset between I2S block start and USB SOF in samples. + * + * Positive offset means that I2S block started at least 1 sample after SOF and + * to correct the situation, shorter than nominal buffers are needed. + * + * Negative offset means that I2S block started at least 1 sample before SOF and + * to correct the situation, larger than nominal buffers are needed. + * + * Offset 0 means that I2S block started within 1 sample around SOF. This is the + * dominant value expected during normal operation. + */ +int feedback_samples_offset(struct feedback_ctx *ctx); + +#endif /* FEEDBACK_H_ */ diff --git a/samples/subsys/usb/uac2_implicit_feedback/src/feedback_dummy.c b/samples/subsys/usb/uac2_implicit_feedback/src/feedback_dummy.c new file mode 100644 index 00000000000..3b91deafe5c --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/src/feedback_dummy.c @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024 Nordic Semiconductor ASA + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "feedback.h" + +#warning "No target specific feedback code, overruns/underruns will occur" + +struct feedback_ctx *feedback_init(void) +{ + return NULL; +} + +void feedback_process(struct feedback_ctx *ctx) +{ + ARG_UNUSED(ctx); +} + +void feedback_reset_ctx(struct feedback_ctx *ctx) +{ + ARG_UNUSED(ctx); +} + +void feedback_start(struct feedback_ctx *ctx, int i2s_blocks_queued) +{ + ARG_UNUSED(ctx); + ARG_UNUSED(i2s_blocks_queued); +} + +int feedback_samples_offset(struct feedback_ctx *ctx) +{ + ARG_UNUSED(ctx); + + /* Always send nominal number of samples */ + return 0; +} diff --git a/samples/subsys/usb/uac2_implicit_feedback/src/feedback_nrf53.c b/samples/subsys/usb/uac2_implicit_feedback/src/feedback_nrf53.c new file mode 100644 index 00000000000..ecc59f1a35a --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/src/feedback_nrf53.c @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2024 Nordic Semiconductor ASA + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include "feedback.h" + +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(feedback, LOG_LEVEL_INF); + +#define FEEDBACK_TIMER_INSTANCE_NUMBER 2 +#define FEEDBACK_TIMER_USBD_SOF_CAPTURE 0 +#define FEEDBACK_TIMER_I2S_FRAMESTART_CAPTURE 1 + +static const nrfx_timer_t feedback_timer_instance = + NRFX_TIMER_INSTANCE(FEEDBACK_TIMER_INSTANCE_NUMBER); + +/* While it might be possible to determine I2S FRAMESTART to USB SOF offset + * entirely in software, the I2S API lacks appropriate timestamping. Therefore + * this sample uses target-specific code to perform the measurements. Note that + * the use of dedicated target-specific peripheral essentially eliminates + * software scheduling jitter and it is likely that a pure software only + * solution would require additional filtering in indirect offset measurements. + * + * Use timer clock (independent from both Audio clock and USB host SOF clock) + * values directly to determine samples offset. This works fine because the + * regulator cares only about error (SOF offset is both error and regulator + * input) and achieves its goal by sending nominal + 1 or nominal - 1 samples. + * SOF offset is around 0 when regulated and therefore the relative clock + * frequency discrepancies are essentially negligible. + */ +#define CLKS_PER_SAMPLE (16000000 / (SAMPLES_PER_SOF * 1000)) + +static struct feedback_ctx { + int32_t rel_sof_offset; + int32_t base_sof_offset; +} fb_ctx; + +struct feedback_ctx *feedback_init(void) +{ + nrfx_err_t err; + uint8_t usbd_sof_gppi_channel; + uint8_t i2s_framestart_gppi_channel; + const nrfx_timer_config_t cfg = { + .frequency = NRFX_MHZ_TO_HZ(16UL), + .mode = NRF_TIMER_MODE_TIMER, + .bit_width = NRF_TIMER_BIT_WIDTH_32, + .interrupt_priority = NRFX_TIMER_DEFAULT_CONFIG_IRQ_PRIORITY, + .p_context = NULL, + }; + + feedback_reset_ctx(&fb_ctx); + + err = nrfx_timer_init(&feedback_timer_instance, &cfg, NULL); + if (err != NRFX_SUCCESS) { + LOG_ERR("nrfx timer init error - Return value: %d", err); + return &fb_ctx; + } + + /* Subscribe TIMER CAPTURE task to USBD SOF event */ + err = nrfx_gppi_channel_alloc(&usbd_sof_gppi_channel); + if (err != NRFX_SUCCESS) { + LOG_ERR("gppi_channel_alloc failed with: %d\n", err); + return &fb_ctx; + } + + nrfx_gppi_channel_endpoints_setup(usbd_sof_gppi_channel, + nrf_usbd_event_address_get(NRF_USBD, NRF_USBD_EVENT_SOF), + nrfx_timer_capture_task_address_get(&feedback_timer_instance, + FEEDBACK_TIMER_USBD_SOF_CAPTURE)); + nrfx_gppi_fork_endpoint_setup(usbd_sof_gppi_channel, + nrfx_timer_task_address_get(&feedback_timer_instance, + NRF_TIMER_TASK_CLEAR)); + + nrfx_gppi_channels_enable(BIT(usbd_sof_gppi_channel)); + + /* Subscribe TIMER CAPTURE task to I2S FRAMESTART event */ + err = nrfx_gppi_channel_alloc(&i2s_framestart_gppi_channel); + if (err != NRFX_SUCCESS) { + LOG_ERR("gppi_channel_alloc failed with: %d\n", err); + return &fb_ctx; + } + + nrfx_gppi_channel_endpoints_setup(i2s_framestart_gppi_channel, + nrf_i2s_event_address_get(NRF_I2S0, NRF_I2S_EVENT_FRAMESTART), + nrfx_timer_capture_task_address_get(&feedback_timer_instance, + FEEDBACK_TIMER_I2S_FRAMESTART_CAPTURE)); + + nrfx_gppi_channels_enable(BIT(i2s_framestart_gppi_channel)); + + /* Enable feedback timer */ + nrfx_timer_enable(&feedback_timer_instance); + + return &fb_ctx; +} + +static void update_sof_offset(struct feedback_ctx *ctx, uint32_t sof_cc, + uint32_t framestart_cc) +{ + int sof_offset; + + /* /2 because we treat the middle as a turning point from being + * "too late" to "too early". + */ + if (framestart_cc > (SAMPLES_PER_SOF * CLKS_PER_SAMPLE)/2) { + sof_offset = framestart_cc - SAMPLES_PER_SOF * CLKS_PER_SAMPLE; + } else { + sof_offset = framestart_cc; + } + + /* The heuristic above is not enough when the offset gets too large. + * If the sign of the simple heuristic changes, check whether the offset + * crossed through the zero or the outer bound. + */ + if ((ctx->rel_sof_offset >= 0) != (sof_offset >= 0)) { + uint32_t abs_diff; + int32_t base_change; + + if (sof_offset >= 0) { + abs_diff = sof_offset - ctx->rel_sof_offset; + base_change = -(SAMPLES_PER_SOF * CLKS_PER_SAMPLE); + } else { + abs_diff = ctx->rel_sof_offset - sof_offset; + base_change = SAMPLES_PER_SOF * CLKS_PER_SAMPLE; + } + + /* Adjust base offset only if the change happened through the + * outer bound. The actual changes should be significantly lower + * than the threshold here. + */ + if (abs_diff > (SAMPLES_PER_SOF * CLKS_PER_SAMPLE)/2) { + ctx->base_sof_offset += base_change; + } + } + + ctx->rel_sof_offset = sof_offset; +} + +void feedback_process(struct feedback_ctx *ctx) +{ + uint32_t sof_cc; + uint32_t framestart_cc; + + sof_cc = nrfx_timer_capture_get(&feedback_timer_instance, + FEEDBACK_TIMER_USBD_SOF_CAPTURE); + framestart_cc = nrfx_timer_capture_get(&feedback_timer_instance, + FEEDBACK_TIMER_I2S_FRAMESTART_CAPTURE); + + update_sof_offset(ctx, sof_cc, framestart_cc); +} + +void feedback_reset_ctx(struct feedback_ctx *ctx) +{ + ARG_UNUSED(ctx); +} + +void feedback_start(struct feedback_ctx *ctx, int i2s_blocks_queued) +{ + /* I2S data was supposed to go out at SOF, but it is inevitably + * delayed due to triggering I2S start by software. Set relative + * SOF offset value in a way that ensures that values past "half + * frame" are treated as "too late" instead of "too early" + */ + ctx->rel_sof_offset = (SAMPLES_PER_SOF * CLKS_PER_SAMPLE) / 2; + /* If there are more than 2 I2S TX blocks queued, use feedback regulator + * to correct the situation. + */ + ctx->base_sof_offset = (i2s_blocks_queued - 2) * + (SAMPLES_PER_SOF * CLKS_PER_SAMPLE); +} + +int feedback_samples_offset(struct feedback_ctx *ctx) +{ + int32_t offset = ctx->rel_sof_offset + ctx->base_sof_offset; + + return offset / CLKS_PER_SAMPLE; +} diff --git a/samples/subsys/usb/uac2_implicit_feedback/src/main.c b/samples/subsys/usb/uac2_implicit_feedback/src/main.c new file mode 100644 index 00000000000..f52d1227102 --- /dev/null +++ b/samples/subsys/usb/uac2_implicit_feedback/src/main.c @@ -0,0 +1,570 @@ +/* + * Copyright (c) 2023-2024 Nordic Semiconductor ASA + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include +#include "feedback.h" + +#include +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(uac2_sample, LOG_LEVEL_INF); + +#define HEADPHONES_OUT_TERMINAL_ID UAC2_ENTITY_ID(DT_NODELABEL(out_terminal)) +#define MICROPHONE_IN_TERMINAL_ID UAC2_ENTITY_ID(DT_NODELABEL(in_terminal)) + +#define SAMPLES_PER_SOF 48 +#define SAMPLE_FREQUENCY (SAMPLES_PER_SOF * 1000) +#define SAMPLE_BIT_WIDTH 16 +#define NUMBER_OF_CHANNELS 2 +#define BYTES_PER_SAMPLE DIV_ROUND_UP(SAMPLE_BIT_WIDTH, 8) +#define BYTES_PER_SLOT (BYTES_PER_SAMPLE * NUMBER_OF_CHANNELS) +#define MIN_BLOCK_SIZE ((SAMPLES_PER_SOF - 1) * BYTES_PER_SLOT) +#define BLOCK_SIZE (SAMPLES_PER_SOF * BYTES_PER_SLOT) +#define MAX_BLOCK_SIZE ((SAMPLES_PER_SOF + 1) * BYTES_PER_SLOT) + +/* Absolute minimum is 5 TX buffers (1 actively consumed by I2S, 2nd queued as + * next buffer, 3rd acquired by USB stack to receive data to, and 2 to handle + * SOF/I2S offset errors), but add 2 additional buffers to prevent out of memory + * errors when USB host decides to perform rapid terminal enable/disable cycles. + */ +#define I2S_BLOCKS 7 +K_MEM_SLAB_DEFINE_STATIC(i2s_tx_slab, MAX_BLOCK_SIZE, I2S_BLOCKS, 4); +K_MEM_SLAB_DEFINE_STATIC(i2s_rx_slab, MAX_BLOCK_SIZE, I2S_BLOCKS, 4); + +struct usb_i2s_ctx { + const struct device *i2s_dev; + bool headphones_enabled; + bool microphone_enabled; + bool i2s_started; + bool rx_started; + bool usb_data_received; + /* Counter used to determine when to start I2S and then when to start + * sending RX packets to host. Overflows are not a problem because this + * variable is not necessary after both I2S and RX is started. + */ + uint8_t i2s_counter; + struct feedback_ctx *fb; + + /* Leftover samples from I2S receive buffer, already compacted to mono, + * that were not sent to host. The buffer, if not NULL, is allocated + * from I2S RX slab. + */ + uint8_t *pending_mic_buf; + uint8_t pending_mic_samples; + + /* Rolling bit buffers for tracking nominal + 1 and nominal - 1 samples + * sent. Bits are mutually exclusive, i.e.: + * plus_ones | minus_ones = plus_ones ^ minus_ones + * + * Used to avoid overcompensation in feedback regulator. LSBs indicate + * latest write size. + */ + uint8_t plus_ones; + uint8_t minus_ones; +}; + +static void uac2_terminal_update_cb(const struct device *dev, uint8_t terminal, + bool enabled, bool microframes, + void *user_data) +{ + struct usb_i2s_ctx *ctx = user_data; + + /* This sample is for Full-Speed only devices. */ + __ASSERT_NO_MSG(microframes == false); + + if (terminal == HEADPHONES_OUT_TERMINAL_ID) { + ctx->headphones_enabled = enabled; + } else if (terminal == MICROPHONE_IN_TERMINAL_ID) { + ctx->microphone_enabled = enabled; + } + + if (ctx->i2s_started && !ctx->headphones_enabled && + !ctx->microphone_enabled) { + i2s_trigger(ctx->i2s_dev, I2S_DIR_BOTH, I2S_TRIGGER_DROP); + ctx->i2s_started = false; + ctx->i2s_counter = 0; + ctx->plus_ones = ctx->minus_ones = 0; + if (ctx->pending_mic_samples) { + k_mem_slab_free(&i2s_rx_slab, ctx->pending_mic_buf); + ctx->pending_mic_buf = NULL; + ctx->pending_mic_samples = 0; + } + } +} + +static void *uac2_get_recv_buf(const struct device *dev, uint8_t terminal, + uint16_t size, void *user_data) +{ + ARG_UNUSED(dev); + struct usb_i2s_ctx *ctx = user_data; + void *buf = NULL; + int ret; + + if (terminal == HEADPHONES_OUT_TERMINAL_ID) { + __ASSERT_NO_MSG(size <= MAX_BLOCK_SIZE); + + if (!ctx->headphones_enabled) { + LOG_ERR("Buffer request on disabled terminal"); + return NULL; + } + + ret = k_mem_slab_alloc(&i2s_tx_slab, &buf, K_NO_WAIT); + if (ret != 0) { + buf = NULL; + } + } + + return buf; +} + +static void uac2_data_recv_cb(const struct device *dev, uint8_t terminal, + void *buf, uint16_t size, void *user_data) +{ + struct usb_i2s_ctx *ctx = user_data; + int ret; + + ctx->usb_data_received = true; + + if (!ctx->headphones_enabled && !ctx->microphone_enabled) { + k_mem_slab_free(&i2s_tx_slab, buf); + return; + } + + if (!size) { + /* This code path is expected when host only records microphone + * data and is not streaming any audio to the headphones. Simply + * transmit as many zero-filled samples were last sent to allow + * the feedback regulator to work. + * + * When host is streaming audio, this can be a transient error. + * While the "feedback regulator delay" is likely to differ, + * it is still probably best to just zero-fill last sent number + * of samples. If we overcompensate as a result, the situation + * will stabilize after a while anyway. + * + * In either case, we have to keep I2S going and the only way + * we can control the SOF to I2S offset is by varying the number + * of samples sent. + */ + if (ctx->plus_ones & 1) { + size = (SAMPLES_PER_SOF + 1) * BYTES_PER_SLOT; + } else if (ctx->minus_ones & 1) { + size = (SAMPLES_PER_SOF - 1) * BYTES_PER_SLOT; + } else { + size = SAMPLES_PER_SOF * BYTES_PER_SLOT; + } + memset(buf, 0, size); + sys_cache_data_flush_range(buf, size); + } + + LOG_DBG("Received %d data to input terminal %d", size, terminal); + + ret = i2s_write(ctx->i2s_dev, buf, size); + if (ret < 0) { + ctx->i2s_started = false; + ctx->i2s_counter = 0; + ctx->plus_ones = ctx->minus_ones = 0; + if (ctx->pending_mic_samples) { + k_mem_slab_free(&i2s_rx_slab, ctx->pending_mic_buf); + ctx->pending_mic_buf = NULL; + ctx->pending_mic_samples = 0; + } + + /* Most likely underrun occurred, prepare I2S restart */ + i2s_trigger(ctx->i2s_dev, I2S_DIR_BOTH, I2S_TRIGGER_PREPARE); + + ret = i2s_write(ctx->i2s_dev, buf, size); + if (ret < 0) { + /* Drop data block, will try again on next frame */ + k_mem_slab_free(&i2s_tx_slab, buf); + } + } + + if (ret == 0) { + ctx->i2s_counter++; + } +} + +static void uac2_buf_release_cb(const struct device *dev, uint8_t terminal, + void *buf, void *user_data) +{ + if (terminal == MICROPHONE_IN_TERMINAL_ID) { + k_mem_slab_free(&i2s_rx_slab, buf); + } +} + +/* Determine next number of samples to send, called at most once every SOF */ +static int next_mic_num_samples(struct usb_i2s_ctx *ctx) +{ + int offset = feedback_samples_offset(ctx->fb); + + /* The rolling buffers essentially handle controller dead time, i.e. + * the buffers are used to prevent overcompensating on feedback offset. + * Remove the oldest entry by shifting the values by one bit. + */ + ctx->plus_ones <<= 1; + ctx->minus_ones <<= 1; + + if ((offset < 0) && (POPCOUNT(ctx->plus_ones) < -offset)) { + /* I2S buffer starts at least 1 sample before SOF, send nominal + * + 1 samples to host in order to shift offset towards 0. + */ + ctx->plus_ones |= 1; + return SAMPLES_PER_SOF + 1; + } + + if ((offset > 0) && (POPCOUNT(ctx->minus_ones) < offset)) { + /* I2S buffer starts at least 1 sample after SOF, send nominal + * - 1 samples to host in order to shift offset towards 0 + */ + ctx->minus_ones |= 1; + return SAMPLES_PER_SOF - 1; + } + + /* I2S is either spot on, or the offset is expected to correct soon */ + return SAMPLES_PER_SOF; +} + +static void process_mic_data(const struct device *dev, struct usb_i2s_ctx *ctx) +{ + size_t num_bytes; + uint8_t *dst, *src; + uint8_t *mic_buf; + void *rx_block; + int ret; + int samples_to_send, mic_samples, rx_samples, leftover_samples; + + samples_to_send = next_mic_num_samples(ctx); + + if (ctx->pending_mic_samples >= samples_to_send) { + /* No need to fetch new I2S samples, this happens shortly after + * we have "borrowed" samples from next buffer. This is expected + * and means that the streams have synchronized. + */ + rx_block = NULL; + rx_samples = 0; + } else { + ret = i2s_read(ctx->i2s_dev, &rx_block, &num_bytes); + if (ret) { + /* No data available, I2S will restart soon */ + return; + } + sys_cache_data_invd_range(rx_block, num_bytes); + + /* I2S operates on 2 channels (stereo) */ + rx_samples = num_bytes / (BYTES_PER_SAMPLE * 2); + } + + /* Prepare microphone data to send, use pending samples if any */ + src = rx_block; + if (ctx->pending_mic_buf) { + mic_buf = ctx->pending_mic_buf; + mic_samples = ctx->pending_mic_samples; + dst = &ctx->pending_mic_buf[mic_samples * BYTES_PER_SAMPLE]; + } else if (rx_samples >= 1) { + /* First sample is already in place */ + mic_buf = rx_block; + dst = &mic_buf[BYTES_PER_SAMPLE]; + src += 2 * BYTES_PER_SAMPLE; + mic_samples = 1; + rx_samples--; + } else { + /* Something went horribly wrong, free the buffer and leave */ + k_mem_slab_free(&i2s_rx_slab, rx_block); + return; + } + + /* Copy as many samples as possible, stop if mic buffer is ready */ + while ((mic_samples < samples_to_send) && (rx_samples > 0)) { + memcpy(dst, src, BYTES_PER_SAMPLE); + + dst += BYTES_PER_SAMPLE; + src += 2 * BYTES_PER_SAMPLE; + + mic_samples++; + rx_samples--; + } + + /* Is mic buffer ready to go? */ + if (mic_samples < samples_to_send) { + /* No, we have to borrow sample from next buffer. This can only + * happen if we fully drained current receive buffer. + */ + __ASSERT_NO_MSG(rx_samples == 0); + + if (rx_block != mic_buf) { + /* RX buffer no longer needed, samples are in mic_buf */ + k_mem_slab_free(&i2s_rx_slab, rx_block); + } + + ret = i2s_read(ctx->i2s_dev, &rx_block, &num_bytes); + if (ret) { + /* No data, I2S will likely restart due to error soon */ + ctx->pending_mic_buf = mic_buf; + ctx->pending_mic_samples = mic_samples; + return; + } + sys_cache_data_invd_range(rx_block, num_bytes); + + src = rx_block; + rx_samples = num_bytes / (BYTES_PER_SAMPLE * 2); + } + + /* Copy remaining sample, under normal conditions (i.e. connected to + * non-malicious host) this is guaranteed to fully fill mic_buf. + */ + while ((mic_samples < samples_to_send) && (rx_samples > 0)) { + memcpy(dst, src, BYTES_PER_SAMPLE); + + dst += BYTES_PER_SAMPLE; + src += 2 * BYTES_PER_SAMPLE; + + mic_samples++; + rx_samples--; + } + + /* Are we still short on samples? */ + if (mic_samples < samples_to_send) { + /* The only possibility for this code to execute is that we were + * short on samples and the next block (pointed to by rx_block) + * did not contain enough samples to fill the gap. + */ + __ASSERT_NO_MSG(rx_block != mic_buf); + + /* Bailing out at this point likely leads to faster recovery. + * Note that this should never happen during normal operation. + */ + ctx->pending_mic_buf = mic_buf; + ctx->pending_mic_samples = mic_samples; + + /* RX buffer is no longer needed */ + k_mem_slab_free(&i2s_rx_slab, rx_block); + return; + } + + /* Handle any potential leftover, start by sanitizing length */ + leftover_samples = mic_samples - samples_to_send + rx_samples; + if (leftover_samples > (MAX_BLOCK_SIZE / BYTES_PER_SAMPLE)) { + size_t dropped_samples = + leftover_samples - (MAX_BLOCK_SIZE / BYTES_PER_SAMPLE); + + LOG_WRN("Too many leftover samples, dropping %d samples", + dropped_samples); + if (rx_samples >= dropped_samples) { + rx_samples -= dropped_samples; + } else { + mic_samples -= (dropped_samples - rx_samples); + rx_samples = 0; + } + + leftover_samples = (MAX_BLOCK_SIZE / BYTES_PER_SAMPLE); + } + + if (leftover_samples == 0) { + /* No leftover samples */ + if ((rx_block != NULL) && (rx_block != mic_buf)) { + /* All samples were copied, free source buffer */ + k_mem_slab_free(&i2s_rx_slab, rx_block); + } + rx_block = NULL; + } else if ((mic_samples > samples_to_send) || + ((rx_samples > 0) && (rx_block == mic_buf))) { + /* Leftover samples have to be copied to new buffer */ + ret = k_mem_slab_alloc(&i2s_rx_slab, &rx_block, K_NO_WAIT); + if (ret != 0) { + LOG_WRN("Out of memory dropping %d samples", + leftover_samples); + mic_samples = samples_to_send; + rx_samples = 0; + rx_block = NULL; + } + } + + /* At this point rx_block is either + * * NULL if there are no leftover samples, OR + * * src buffer if leftover data can be copied from back to front, OR + * * brand new buffer if there is leftover data in mic buffer. + */ + ctx->pending_mic_buf = rx_block; + ctx->pending_mic_samples = 0; + + /* Copy excess samples from pending mic buf, if any */ + if (mic_samples > samples_to_send) { + size_t bytes; + + /* Samples in mic buffer are already compacted */ + bytes = (mic_samples - samples_to_send) * BYTES_PER_SAMPLE; + memcpy(ctx->pending_mic_buf, &mic_buf[mic_samples], bytes); + + ctx->pending_mic_samples = mic_samples - samples_to_send; + dst = &ctx->pending_mic_buf[bytes]; + } else { + dst = ctx->pending_mic_buf; + } + + /* Copy excess samples from src buffer, so we don't lose any */ + while (rx_samples > 0) { + memcpy(dst, src, BYTES_PER_SAMPLE); + + dst += BYTES_PER_SAMPLE; + src += 2 * BYTES_PER_SAMPLE; + + ctx->pending_mic_samples++; + rx_samples--; + } + + /* Finally send the microphone samples to host */ + sys_cache_data_flush_range(mic_buf, mic_samples * BYTES_PER_SAMPLE); + if (usbd_uac2_send(dev, MICROPHONE_IN_TERMINAL_ID, + mic_buf, mic_samples * BYTES_PER_SAMPLE) < 0) { + k_mem_slab_free(&i2s_rx_slab, mic_buf); + } +} + +static void uac2_sof(const struct device *dev, void *user_data) +{ + ARG_UNUSED(dev); + struct usb_i2s_ctx *ctx = user_data; + + if (ctx->i2s_started) { + feedback_process(ctx->fb); + } + + /* If we didn't receive data since last SOF but either terminal is + * enabled, then we have to come up with the buffer ourself to keep + * I2S going. + */ + if (!ctx->usb_data_received && + (ctx->microphone_enabled || ctx->headphones_enabled)) { + /* No data received since last SOF but we have to keep going */ + void *buf; + int ret; + + ret = k_mem_slab_alloc(&i2s_tx_slab, &buf, K_NO_WAIT); + if (ret != 0) { + buf = NULL; + } + + if (buf) { + /* Use size 0 to utilize zero-fill functionality */ + uac2_data_recv_cb(dev, HEADPHONES_OUT_TERMINAL_ID, + buf, 0, user_data); + } + } + ctx->usb_data_received = false; + + /* We want to maintain 3 SOFs delay, i.e. samples received from host + * during SOF n should be transmitted on I2S during SOF n+3. This + * provides enough wiggle room for software scheduling that effectively + * eliminates "buffers not provided in time" problem. + * + * ">= 2" translates into 3 SOFs delay because the timeline is: + * USB SOF n + * OUT DATA0 n received from host + * USB SOF n+1 + * DATA0 n is available to UDC driver (See Universal Serial Bus + * Specification Revision 2.0 5.12.5 Data Prebuffering) and copied + * to I2S buffer before SOF n+2; i2s_counter = 1 + * OUT DATA0 n+1 received from host + * USB SOF n+2 + * DATA0 n+1 is copied; i2s_counter = 2 + * OUT DATA0 n+2 received from host + * USB SOF n+3 + * This function triggers I2S start + * DATA0 n+2 is copied; i2s_counter is no longer relevant + * OUT DATA0 n+3 received from host + */ + if (!ctx->i2s_started && + (ctx->headphones_enabled || ctx->microphone_enabled) && + ctx->i2s_counter >= 2) { + i2s_trigger(ctx->i2s_dev, I2S_DIR_BOTH, I2S_TRIGGER_START); + ctx->i2s_started = true; + feedback_start(ctx->fb, ctx->i2s_counter); + ctx->i2s_counter = 0; + } + + /* Start sending I2S RX data only when there are at least 3 buffers + * ready with data. This guarantees that there'll always be a buffer + * available from which sample can be borrowed. + */ + if (!ctx->rx_started && ctx->i2s_started && ctx->i2s_counter >= 3) { + ctx->rx_started = true; + } + + if (ctx->rx_started) { + process_mic_data(dev, ctx); + } +} + +static struct uac2_ops usb_audio_ops = { + .sof_cb = uac2_sof, + .terminal_update_cb = uac2_terminal_update_cb, + .get_recv_buf = uac2_get_recv_buf, + .data_recv_cb = uac2_data_recv_cb, + .buf_release_cb = uac2_buf_release_cb, +}; + +static struct usb_i2s_ctx main_ctx; + +int main(void) +{ + const struct device *dev = DEVICE_DT_GET(DT_NODELABEL(uac2_headset)); + struct usbd_context *sample_usbd; + struct i2s_config config; + int ret; + + main_ctx.i2s_dev = DEVICE_DT_GET(DT_NODELABEL(i2s_rxtx)); + + if (!device_is_ready(main_ctx.i2s_dev)) { + printk("%s is not ready\n", main_ctx.i2s_dev->name); + return 0; + } + + config.word_size = SAMPLE_BIT_WIDTH; + config.channels = NUMBER_OF_CHANNELS; + config.format = I2S_FMT_DATA_FORMAT_I2S; + config.options = I2S_OPT_BIT_CLK_MASTER | I2S_OPT_FRAME_CLK_MASTER; + config.frame_clk_freq = SAMPLE_FREQUENCY; + config.mem_slab = &i2s_tx_slab; + config.block_size = MAX_BLOCK_SIZE; + config.timeout = 0; + + ret = i2s_configure(main_ctx.i2s_dev, I2S_DIR_TX, &config); + if (ret < 0) { + printk("Failed to configure TX stream: %d\n", ret); + return 0; + } + + config.mem_slab = &i2s_rx_slab; + ret = i2s_configure(main_ctx.i2s_dev, I2S_DIR_RX, &config); + if (ret < 0) { + printk("Failed to configure RX stream: %d\n", ret); + return 0; + } + + main_ctx.fb = feedback_init(); + + usbd_uac2_set_ops(dev, &usb_audio_ops, &main_ctx); + + sample_usbd = sample_usbd_init_device(NULL); + if (sample_usbd == NULL) { + return -ENODEV; + } + + ret = usbd_enable(sample_usbd); + if (ret) { + return ret; + } + + return 0; +}