esp_hal/spi/master/
dma.rs

1use core::{
2    cell::{Cell, UnsafeCell},
3    cmp::min,
4    mem::{ManuallyDrop, MaybeUninit},
5    pin::Pin,
6    ptr::NonNull,
7    sync::atomic::{Ordering, fence},
8    task::{Context, Poll},
9};
10
11#[cfg(feature = "unstable")]
12use embedded_hal::spi::{ErrorType, SpiBus};
13use enumset::EnumSet;
14#[cfg(place_spi_master_driver_in_ram)]
15use procmacros::ram;
16
17use super::*;
18use crate::{
19    RegisterToggle,
20    dma::{
21        CHUNK_SIZE,
22        Channel,
23        DmaDescriptor,
24        DmaEligiblePeripheral,
25        DmaRxBuf,
26        DmaRxBuffer,
27        DmaTxBuf,
28        DmaTxBuffer,
29        NoBuffer,
30        ScopedDmaRxBuf,
31        ScopedDmaTxBuf,
32        TransferDirection,
33        aligned::{DmaAlignedMut, InternalMemory},
34        asynch::DmaRxFuture,
35        prepare_for_rx,
36        prepare_for_tx,
37    },
38    pac::spi2::RegisterBlock,
39    private::DropGuard,
40    soc::is_slice_in_dram,
41    spi::{DmaError, master::low_level::SpiClockGuard},
42};
43#[cfg(dma_can_access_psram)]
44use crate::{dma::ManualWritebackBuffer, soc::is_slice_in_psram};
45
46const MAX_DMA_SIZE: usize = 32736;
47
48impl<'d> Spi<'d, Blocking> {
49    #[doc_replace(
50        "dma_channel" => {
51            cfg(spi_master_dma_engine = "SPI_DMA") => "DMA_SPI2",
52            cfg(spi_master_dma_engine = "AHB_GDMA") => "DMA_CH0",
53            cfg(spi_master_dma_engine = "AXI_GDMA") => "DMA_AXI_CH0",
54        }
55    )]
56    /// Converts the driver into an [`SpiDma`] driver that uses the specified DMA channel.
57    ///
58    /// ```rust, no_run
59    /// # {before_snippet}
60    /// use esp_hal::spi::{
61    ///     Mode,
62    ///     master::{Config, Spi},
63    /// };
64    ///
65    /// let mut spi_dma = Spi::new(
66    ///     peripherals.SPI2,
67    ///     Config::default()
68    ///         .with_frequency(Rate::from_khz(100))
69    ///         .with_mode(Mode::_0),
70    /// )?
71    /// .with_dma(peripherals.__dma_channel__);
72    /// # {after_snippet}
73    /// ```
74    #[instability::unstable]
75    pub fn with_dma(
76        self,
77        channel: impl SpiMasterDmaChannel<'d, AnySpi<'d>>,
78    ) -> SpiDma<'d, crate::Blocking> {
79        SpiDma::new_from_spi(self, channel.into())
80    }
81}
82
83#[doc_replace(
84    "dma_channel" => {
85        cfg(spi_master_dma_engine = "SPI_DMA") => "DMA_SPI2",
86        cfg(spi_master_dma_engine = "AHB_GDMA") => "DMA_CH0",
87        cfg(spi_master_dma_engine = "AXI_GDMA") => "DMA_AXI_CH0",
88    }
89)]
90/// DMA-controlled SPI driver.
91///
92/// This driver uses DMA to transfer data, allowing the CPU to continue working while the SPI
93/// transfer is in progress.
94///
95/// The driver provides two separate approaches to transferring data:
96///
97/// - The slice-based API allows transferring data from/to slices of memory. The data may be copied
98///   into an internal buffer before the transfer begins. A pair of copy buffers can be set up by
99///   passing them to [`with_buffers`](SpiDma::with_buffers) before the first transfer begins. For
100///   more details on when copying is necessary, see the documentation of the
101///   [`with_buffers`](SpiDma::with_buffers) method.
102/// - The buffer API allows transferring externally managed buffers. In this mode, you provide the
103///   buffers to be transferred. The buffer objects ensure that data is located in appropriate
104///   memory regions. The buffers and the driver object are moved into transfer objects for the
105///   duration of the transfer. These functions take [`DmaRxBuf`] and [`DmaTxBuf`] objects as
106///   arguments as well as the number of bytes to transfer, and their names end with `_buffer`.
107///
108/// These approaches provide different trade-offs between memory usage / CPU overhead and ease of
109/// use. `embedded-hal` traits are implemented by the slice-based API's functions.
110///
111/// ## Examples
112///
113/// ```rust, no_run
114/// # {before_snippet}
115/// use esp_hal::{
116///     dma::{DmaRxBuf, DmaTxBuf},
117///     dma_rx_buffer,
118///     dma_tx_buffer,
119///     spi::{
120///         Mode,
121///         master::{Config, Spi},
122///     },
123/// };
124///
125/// // Optional: create and set up copy buffers.
126/// let dma_rx_buf = dma_rx_buffer!(32000)?;
127/// let dma_tx_buf = dma_tx_buffer!(32000)?;
128///
129/// let mut spi = Spi::new(
130///     peripherals.SPI2,
131///     Config::default()
132///         .with_frequency(Rate::from_khz(100))
133///         .with_mode(Mode::_0),
134/// )?
135/// .with_dma(peripherals.__dma_channel__)
136/// .with_buffers(dma_rx_buf, dma_tx_buf);
137/// #
138/// # {after_snippet}
139/// ```
140#[cfg_attr(feature = "defmt", derive(defmt::Format))]
141pub struct SpiDma<'d, Dm>
142where
143    Dm: DriverMode,
144{
145    spi: SpiWrapper<'d>,
146    pub(crate) channel: Channel<Dm, SpiMasterErased<'d>>,
147}
148
149impl<Dm> crate::private::Sealed for SpiDma<'_, Dm> where Dm: DriverMode {}
150
151impl<Dm> core::fmt::Debug for SpiDma<'_, Dm>
152where
153    Dm: DriverMode + core::fmt::Debug,
154{
155    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
156        f.debug_struct("SpiDma").field("spi", &self.spi).finish()
157    }
158}
159
160#[instability::unstable]
161impl crate::interrupt::InterruptConfigurable for SpiDma<'_, Blocking> {
162    fn set_interrupt_handler(&mut self, handler: InterruptHandler) {
163        self.set_interrupt_handler(handler);
164    }
165}
166
167#[instability::unstable]
168impl<Dm> embassy_embedded_hal::SetConfig for SpiDma<'_, Dm>
169where
170    Dm: DriverMode,
171{
172    type Config = Config;
173    type ConfigError = ConfigError;
174
175    fn set_config(&mut self, config: &Self::Config) -> Result<(), Self::ConfigError> {
176        self.apply_config(config)
177    }
178}
179
180#[instability::unstable]
181impl<Dm> ErrorType for SpiDma<'_, Dm>
182where
183    Dm: DriverMode,
184{
185    type Error = Error;
186}
187
188#[instability::unstable]
189impl<Dm> SpiBus for SpiDma<'_, Dm>
190where
191    Dm: DriverMode,
192{
193    fn read(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
194        self.read(words)
195    }
196
197    fn write(&mut self, words: &[u8]) -> Result<(), Self::Error> {
198        self.write(words)
199    }
200
201    fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Self::Error> {
202        self.transfer(read, write)
203    }
204
205    fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
206        self.transfer_in_place(words)
207    }
208
209    fn flush(&mut self) -> Result<(), Self::Error> {
210        // DMA limitation - we must ensure the transfers complete before returning
211        // to user code, otherwise the user might access the buffers while the transfer
212        // is still in progress. Therefore, there is no such thing as "flushing".
213        Ok(())
214    }
215}
216
217#[instability::unstable]
218impl embedded_hal_async::spi::SpiBus for SpiDma<'_, Async> {
219    async fn read(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
220        self.read_async(words).await
221    }
222
223    async fn write(&mut self, words: &[u8]) -> Result<(), Self::Error> {
224        self.write_async(words).await
225    }
226
227    async fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Self::Error> {
228        self.transfer_async(read, write).await
229    }
230
231    async fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
232        self.transfer_in_place_async(words).await
233    }
234
235    async fn flush(&mut self) -> Result<(), Self::Error> {
236        // DMA limitation - we must ensure the transfers complete before returning
237        // to user code, otherwise the user might access the buffers while the transfer
238        // is still in progress. Therefore, there is no such thing as "flushing".
239        Ok(())
240    }
241}
242
243impl<'d> SpiDma<'d, Blocking> {
244    /// Converts the SPI driver into async mode.
245    #[instability::unstable]
246    pub fn into_async(self) -> SpiDma<'d, Async> {
247        self.spi
248            .set_interrupt_handler(self.spi.info().async_handler);
249        SpiDma {
250            spi: self.spi,
251            channel: self.channel.into_async(),
252        }
253    }
254
255    fn new_inner(spi: SpiWrapper<'d>, channel: SpiMasterErased<'d>) -> Self {
256        let channel = Channel::new(channel);
257        channel.runtime_ensure_compatible(spi.spi.dma_peripheral());
258
259        let state = spi.spi.dma_state();
260
261        state.tx_transfer_in_progress.set(false);
262        state.rx_transfer_in_progress.set(false);
263
264        // Safety: The descriptors occupy their own shared cache line and are updated in a
265        // synchronised fashion.
266        let (tx_descriptors, rx_descriptors) = unsafe {
267            let descriptors = (&mut *state.descriptors.get()).get_mut().into_inner();
268            descriptors.fill(DmaDescriptor::EMPTY);
269            let (tx_descriptors, rx_descriptors) = descriptors.split_at_mut(1);
270            (
271                DmaAlignedMut::new_unchecked(tx_descriptors),
272                DmaAlignedMut::new_unchecked(rx_descriptors),
273            )
274        };
275
276        let tx_buffer = cfg_select! {
277            all(spi_master_version = "1", spi_address_workaround) => unsafe {
278                (&mut *state.default_tx_buffer.get()).get_mut().unsize()
279            },
280            _ => unsafe { DmaAlignedMut::new_unchecked(&mut [][..]) },
281        };
282
283        let rx_buffer = unwrap!(DmaRxBuf::new(rx_descriptors, unsafe {
284            DmaAlignedMut::new_unchecked(&mut [])
285        }));
286        let tx_buffer = unwrap!(DmaTxBuf::new(tx_descriptors, tx_buffer));
287
288        // The buffers must be set up when creating the driver.
289        unsafe { (&mut *state.tx_buffer.get()).write(tx_buffer.into_scoped()) };
290        unsafe { (&mut *state.rx_buffer.get()).write(rx_buffer.into_scoped()) };
291
292        Self { spi, channel }
293    }
294
295    pub(super) fn new_from_spi(
296        spi_driver: Spi<'d, Blocking>,
297        channel: SpiMasterErased<'d>,
298    ) -> Self {
299        let spi = spi_driver.spi;
300
301        Self::new_inner(spi, channel)
302    }
303
304    /// Listen for the given interrupts
305    #[instability::unstable]
306    pub fn listen(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {
307        self.driver().enable_listen(interrupts.into(), true);
308    }
309
310    /// Unlisten the given interrupts
311    #[instability::unstable]
312    pub fn unlisten(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {
313        self.driver().enable_listen(interrupts.into(), false);
314    }
315
316    /// Gets asserted interrupts
317    #[instability::unstable]
318    pub fn interrupts(&mut self) -> EnumSet<SpiInterrupt> {
319        self.driver().interrupts()
320    }
321
322    /// Resets asserted interrupts
323    #[instability::unstable]
324    pub fn clear_interrupts(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {
325        self.driver().clear_interrupts(interrupts.into());
326    }
327
328    #[cfg_attr(
329        not(multi_core),
330        doc = "Registers an interrupt handler for the peripheral."
331    )]
332    #[cfg_attr(
333        multi_core,
334        doc = "Registers an interrupt handler for the peripheral on the current core."
335    )]
336    #[doc = ""]
337    /// Note that this will replace any previously registered interrupt
338    /// handlers.
339    ///
340    /// You can restore the default/unhandled interrupt handler by using
341    /// [crate::interrupt::DEFAULT_INTERRUPT_HANDLER]
342    ///
343    /// # Panics
344    ///
345    /// Panics if passed interrupt handler is invalid (e.g. has priority
346    /// `None`)
347    #[instability::unstable]
348    pub fn set_interrupt_handler(&mut self, handler: InterruptHandler) {
349        self.spi.set_interrupt_handler(handler);
350    }
351}
352
353impl<'d> SpiDma<'d, Async> {
354    /// Converts the SPI instance into blocking mode.
355    #[instability::unstable]
356    pub fn into_blocking(self) -> SpiDma<'d, Blocking> {
357        self.spi.disable_peri_interrupt_on_all_cores();
358        SpiDma {
359            spi: self.spi,
360            channel: self.channel.into_blocking(),
361        }
362    }
363
364    async fn wait_for_idle_async(&mut self) {
365        if self.dma_driver().state.rx_transfer_in_progress.get() {
366            _ = DmaRxFuture::new(&mut self.channel.rx).await;
367            self.dma_driver().state.rx_transfer_in_progress.set(false);
368        }
369
370        struct Fut(Driver);
371        impl Fut {
372            const DONE_EVENTS: EnumSet<SpiInterrupt> =
373                enumset::enum_set!(SpiInterrupt::TransferDone);
374        }
375        impl Future for Fut {
376            type Output = ();
377
378            fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
379                if !self.0.interrupts().is_disjoint(Self::DONE_EVENTS) {
380                    #[cfg(any(spi_master_version = "1", spi_master_version = "2"))]
381                    // Need to poll for done-ness even after interrupt fires.
382                    if self.0.busy() {
383                        cx.waker().wake_by_ref();
384                        return Poll::Pending;
385                    }
386
387                    self.0.clear_interrupts(Self::DONE_EVENTS);
388                    return Poll::Ready(());
389                }
390
391                self.0.state.waker.register(cx.waker());
392                self.0.enable_listen(Self::DONE_EVENTS, true);
393                Poll::Pending
394            }
395        }
396        impl Drop for Fut {
397            fn drop(&mut self) {
398                self.0.enable_listen(Self::DONE_EVENTS, false);
399            }
400        }
401
402        if !self.is_done() {
403            Fut(self.driver()).await;
404        }
405
406        if self.dma_driver().state.tx_transfer_in_progress.get() {
407            // In case DMA TX buffer is bigger than what the SPI consumes, stop the DMA.
408            if !self.channel.tx.is_done() {
409                self.channel.tx.stop_transfer();
410            }
411            self.dma_driver().state.tx_transfer_in_progress.set(false);
412        }
413    }
414
415    /// Fill the given buffer with data from the bus.
416    #[instability::unstable]
417    pub async fn read_async(&mut self, words: &mut [u8]) -> Result<(), Error> {
418        if words.is_empty() {
419            return Ok(());
420        }
421
422        let _clock = SpiClockGuard::new(self.spi.info());
423
424        self.driver().setup_full_duplex()?;
425
426        if self.use_blocking_transfer(words.len()) {
427            self.dma_driver().disable_dma();
428            return self.driver().read(words);
429        }
430
431        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
432        let mut maybe_copy_buffer = match DmaOperationKind::for_read(words) {
433            DmaOperationKind::Copied => {
434                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
435            }
436            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
437                descriptors: &mut descriptors,
438                #[cfg(dma_can_access_psram)]
439                align_buffer: [const { None }; 2],
440            },
441        };
442
443        if maybe_copy_buffer.chunk_size() == 0 {
444            return Err(Error::from(DmaError::BufferTooSmall));
445        }
446
447        for chunk in words.chunks_mut(maybe_copy_buffer.chunk_size()) {
448            let read_bytes = chunk.len();
449            let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *chunk)) };
450            let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };
451
452            self.transfer_buffers_dma_async(read_bytes, 0, rx_buffer, tx_buffer)
453                .await?;
454
455            maybe_copy_buffer.finish(chunk);
456        }
457
458        Ok(())
459    }
460
461    /// Transmit the given buffer to the bus.
462    #[instability::unstable]
463    pub async fn write_async(&mut self, words: &[u8]) -> Result<(), Error> {
464        if words.is_empty() {
465            return Ok(());
466        }
467
468        let _clock = SpiClockGuard::new(self.spi.info());
469
470        self.driver().setup_full_duplex()?;
471
472        if self.use_blocking_transfer(words.len()) {
473            self.dma_driver().disable_dma();
474            return self.driver().write(words);
475        }
476
477        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
478        let mut maybe_copy_buffer = match DmaOperationKind::for_write(words) {
479            DmaOperationKind::Copied => {
480                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
481            }
482            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),
483        };
484
485        if maybe_copy_buffer.chunk_size() == 0 {
486            return Err(Error::from(DmaError::BufferTooSmall));
487        }
488
489        for chunk in words.chunks(maybe_copy_buffer.chunk_size()) {
490            let write_bytes = chunk.len();
491            let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) };
492            let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(chunk)) };
493
494            self.transfer_buffers_dma_async(0, write_bytes, rx_buffer, tx_buffer)
495                .await?;
496        }
497
498        Ok(())
499    }
500
501    /// Transfer by writing out a buffer and reading the response from
502    /// the bus into another buffer.
503    #[instability::unstable]
504    pub async fn transfer_async(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Error> {
505        if read.is_empty() && write.is_empty() {
506            return Ok(());
507        }
508
509        let _clock = SpiClockGuard::new(self.spi.info());
510
511        self.driver().setup_full_duplex()?;
512
513        if self.use_blocking_transfer(read.len().max(write.len())) {
514            self.dma_driver().disable_dma();
515            return if read.is_empty() {
516                self.driver().write(write)
517            } else if write.is_empty() {
518                self.driver().read(read)
519            } else {
520                self.driver().transfer(read, write)
521            };
522        }
523
524        let common_length = min(read.len(), write.len());
525        let (read_common, read_remainder) = read.split_at_mut(common_length);
526        let (write_common, write_remainder) = write.split_at(common_length);
527
528        // DmaOperationKind must be determined on the sub-slices actually passed to DMA.
529        let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
530        let mut maybe_copy_rx_buffer = match DmaOperationKind::for_read(read_common) {
531            DmaOperationKind::Copied => {
532                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
533            }
534            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
535                descriptors: &mut rx_descriptors,
536                #[cfg(dma_can_access_psram)]
537                align_buffer: [const { None }; 2],
538            },
539        };
540
541        let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
542        let mut maybe_copy_tx_buffer = match DmaOperationKind::for_write(write_common) {
543            DmaOperationKind::Copied => {
544                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
545            }
546            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut tx_descriptors),
547        };
548
549        let chunk_size = min(
550            maybe_copy_rx_buffer.chunk_size(),
551            maybe_copy_tx_buffer.chunk_size(),
552        );
553
554        if chunk_size == 0 {
555            return Err(Error::from(DmaError::BufferTooSmall));
556        }
557
558        for (read_chunk, write_chunk) in read_common
559            .chunks_mut(chunk_size)
560            .zip(write_common.chunks(chunk_size))
561        {
562            let read_bytes = read_chunk.len();
563            let write_bytes = write_chunk.len();
564            let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(NonNull::from(write_chunk)) };
565            let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(NonNull::from(&mut *read_chunk)) };
566
567            self.transfer_buffers_dma_async(read_bytes, write_bytes, rx_buffer, tx_buffer)
568                .await?;
569
570            maybe_copy_rx_buffer.finish(read_chunk);
571        }
572
573        if !read_remainder.is_empty() {
574            self.read_async(read_remainder).await
575        } else if !write_remainder.is_empty() {
576            self.write_async(write_remainder).await
577        } else {
578            Ok(())
579        }
580    }
581
582    /// Transfer by writing out a buffer and reading the response from
583    /// the bus into the same buffer.
584    #[instability::unstable]
585    pub async fn transfer_in_place_async(&mut self, words: &mut [u8]) -> Result<(), Error> {
586        if words.is_empty() {
587            return Ok(());
588        }
589
590        let _clock = SpiClockGuard::new(self.spi.info());
591        self.driver().setup_full_duplex()?;
592
593        if self.use_blocking_transfer(words.len()) {
594            self.dma_driver().disable_dma();
595            return self.driver().transfer_in_place(words);
596        }
597
598        let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
599        let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
600        let (mut maybe_copy_rx_buffer, mut maybe_copy_tx_buffer) =
601            match DmaOperationKind::for_write(words) {
602                DmaOperationKind::Copied => (
603                    MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }),
604                    MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }),
605                ),
606                DmaOperationKind::InPlace => (
607                    MaybeCopyRxBuf::Direct {
608                        descriptors: &mut rx_descriptors,
609                        #[cfg(dma_can_access_psram)]
610                        align_buffer: [const { None }; 2],
611                    },
612                    MaybeCopyTxBuf::Direct(&mut tx_descriptors),
613                ),
614            };
615
616        let chunk_size = min(
617            maybe_copy_rx_buffer.chunk_size(),
618            maybe_copy_tx_buffer.chunk_size(),
619        );
620
621        if chunk_size == 0 {
622            return Err(Error::from(DmaError::BufferTooSmall));
623        }
624
625        for chunk in words.chunks_mut(chunk_size) {
626            let bytes = chunk.len();
627            let ptr = NonNull::from(&mut *chunk);
628            let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(ptr) };
629            let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(ptr) };
630
631            self.transfer_buffers_dma_async(bytes, bytes, rx_buffer, tx_buffer)
632                .await?;
633
634            maybe_copy_rx_buffer.finish(chunk);
635        }
636
637        Ok(())
638    }
639
640    /// Half-duplex read.
641    ///
642    /// This performs the command, address, dummy, and data phases as a single
643    /// SPI transaction. Because command and address phases cannot be split
644    /// across multiple DMA transfers, `buffer` must fit in one DMA transfer or
645    /// in the configured internal RX copy buffer.
646    #[instability::unstable]
647    pub async fn half_duplex_read_async(
648        &mut self,
649        data_mode: DataMode,
650        cmd: Command,
651        address: Address,
652        dummy: u8,
653        buffer: &mut [u8],
654    ) -> Result<(), Error> {
655        let _clock = SpiClockGuard::new(self.spi.info());
656
657        if buffer.is_empty() {
658            let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) };
659            self.half_duplex_read_dma_async(data_mode, cmd, address, dummy, 0, rx_buffer)
660                .await?;
661            return Ok(());
662        }
663
664        // Transfers below the configured threshold can skip the DMA setup cost entirely.
665        if self.use_blocking_transfer(buffer.len()) {
666            self.dma_driver().disable_dma();
667            return self
668                .driver()
669                .half_duplex_read(data_mode, cmd, address, dummy, buffer);
670        }
671
672        let operation = DmaOperationKind::for_read(buffer);
673        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
674        let mut maybe_copy_buffer = match operation {
675            DmaOperationKind::Copied => {
676                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
677            }
678            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
679                descriptors: &mut descriptors,
680                #[cfg(dma_can_access_psram)]
681                align_buffer: [const { None }; 2],
682            },
683        };
684
685        let chunk_size = maybe_copy_buffer.chunk_size();
686        if chunk_size == 0 {
687            return Err(Error::from(DmaError::BufferTooSmall));
688        }
689        if buffer.len() > chunk_size {
690            return match operation {
691                DmaOperationKind::Copied => Err(Error::from(DmaError::Overflow)),
692                DmaOperationKind::InPlace => Err(Error::MaxDmaTransferSizeExceeded),
693            };
694        }
695
696        let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *buffer)) };
697        self.half_duplex_read_dma_async(data_mode, cmd, address, dummy, buffer.len(), rx_buffer)
698            .await?;
699        maybe_copy_buffer.finish(buffer);
700
701        Ok(())
702    }
703
704    /// Half-duplex write.
705    ///
706    /// This performs the command, address, dummy, and data phases as a single
707    /// SPI transaction. Because command and address phases cannot be split
708    /// across multiple DMA transfers, `buffer` must fit in one DMA transfer or
709    /// in the configured internal TX copy buffer.
710    #[instability::unstable]
711    pub async fn half_duplex_write_async(
712        &mut self,
713        data_mode: DataMode,
714        cmd: Command,
715        address: Address,
716        dummy: u8,
717        buffer: &[u8],
718    ) -> Result<(), Error> {
719        let _clock = SpiClockGuard::new(self.spi.info());
720
721        if buffer.is_empty() {
722            let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };
723            self.half_duplex_write_dma_async(data_mode, cmd, address, dummy, 0, tx_buffer)
724                .await?;
725            return Ok(());
726        }
727
728        // Transfers below the configured threshold can skip the DMA setup cost entirely.
729        if self.use_blocking_transfer(buffer.len()) {
730            self.dma_driver().disable_dma();
731            return self
732                .driver()
733                .half_duplex_write(data_mode, cmd, address, dummy, buffer);
734        }
735
736        let operation = DmaOperationKind::for_write(buffer);
737        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
738        let mut maybe_copy_buffer = match operation {
739            DmaOperationKind::Copied => {
740                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
741            }
742            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),
743        };
744
745        let chunk_size = maybe_copy_buffer.chunk_size();
746        if chunk_size == 0 {
747            return Err(Error::from(DmaError::BufferTooSmall));
748        }
749        if buffer.len() > chunk_size {
750            return match operation {
751                DmaOperationKind::Copied => Err(Error::from(DmaError::Overflow)),
752                DmaOperationKind::InPlace => Err(Error::MaxDmaTransferSizeExceeded),
753            };
754        }
755
756        let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(buffer)) };
757        self.half_duplex_write_dma_async(data_mode, cmd, address, dummy, buffer.len(), tx_buffer)
758            .await
759    }
760
761    async fn transfer_buffers_dma_async(
762        &mut self,
763        read_bytes: usize,
764        write_bytes: usize,
765        mut rx_buffer: impl DmaRxBuffer,
766        mut tx_buffer: impl DmaTxBuffer,
767    ) -> Result<(), Error> {
768        let _clock = SpiClockGuard::new(self.spi.info());
769
770        let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());
771        unsafe {
772            spi.start_dma_transfer(read_bytes, write_bytes, &mut rx_buffer, &mut tx_buffer)?;
773        }
774        spi.wait_for_idle_async().await;
775        spi.defuse();
776        Ok(())
777    }
778
779    async fn half_duplex_read_dma_async(
780        &mut self,
781        data_mode: DataMode,
782        cmd: Command,
783        address: Address,
784        dummy: u8,
785        bytes_to_read: usize,
786        mut rx_buffer: impl DmaRxBuffer,
787    ) -> Result<(), Error> {
788        let _clock = SpiClockGuard::new(self.spi.info());
789
790        let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());
791        unsafe {
792            spi.start_half_duplex_read(
793                data_mode,
794                cmd,
795                address,
796                dummy,
797                bytes_to_read,
798                &mut rx_buffer,
799            )?;
800        }
801        spi.wait_for_idle_async().await;
802        spi.defuse();
803        Ok(())
804    }
805
806    async fn half_duplex_write_dma_async(
807        &mut self,
808        data_mode: DataMode,
809        cmd: Command,
810        address: Address,
811        dummy: u8,
812        bytes_to_write: usize,
813        mut tx_buffer: impl DmaTxBuffer,
814    ) -> Result<(), Error> {
815        let _clock = SpiClockGuard::new(self.spi.info());
816
817        let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());
818        unsafe {
819            spi.start_half_duplex_write(
820                data_mode,
821                cmd,
822                address,
823                dummy,
824                bytes_to_write,
825                &mut tx_buffer,
826            )?;
827        }
828        spi.wait_for_idle_async().await;
829        spi.defuse();
830        Ok(())
831    }
832}
833
834// +1 to make sure we have enough descriptors to satisfy strict alignment requirements
835const LINK_DESCRIPTOR_COUNT: usize = MAX_DMA_SIZE.div_ceil(CHUNK_SIZE) + 2 + 1;
836
837enum MaybeCopyTxBuf<'a> {
838    Copy(&'a mut ScopedDmaTxBuf<'static>),
839    Direct(&'a mut [DmaDescriptor; LINK_DESCRIPTOR_COUNT]),
840}
841
842impl<'a> MaybeCopyTxBuf<'a> {
843    unsafe fn setup(&mut self, data: NonNull<[u8]>) -> NoBuffer {
844        match self {
845            MaybeCopyTxBuf::Copy(tx_buffer) => {
846                tx_buffer.as_mut_slice()[..data.len()].copy_from_slice(unsafe { data.as_ref() });
847                NoBuffer(tx_buffer.prepare())
848            }
849            MaybeCopyTxBuf::Direct(descriptors) => {
850                let (buffer, _) = unsafe { unwrap!(prepare_for_tx(&mut **descriptors, data, 1)) };
851                buffer
852            }
853        }
854    }
855
856    fn chunk_size(&self) -> usize {
857        match self {
858            MaybeCopyTxBuf::Copy(buffer) => buffer.capacity().min(MAX_DMA_SIZE),
859            MaybeCopyTxBuf::Direct(_) => MAX_DMA_SIZE,
860        }
861    }
862}
863
864#[allow(clippy::large_enum_variant)]
865enum MaybeCopyRxBuf<'a> {
866    Copy(&'a mut ScopedDmaRxBuf<'static>),
867    Direct {
868        descriptors: &'a mut [DmaDescriptor; LINK_DESCRIPTOR_COUNT],
869        #[cfg(dma_can_access_psram)]
870        align_buffer: [Option<ManualWritebackBuffer>; 2],
871    },
872}
873
874impl<'a> MaybeCopyRxBuf<'a> {
875    unsafe fn setup(&mut self, data: NonNull<[u8]>) -> NoBuffer {
876        match self {
877            MaybeCopyRxBuf::Copy(rx_buffer) => NoBuffer(rx_buffer.prepare()),
878            MaybeCopyRxBuf::Direct {
879                descriptors,
880                #[cfg(dma_can_access_psram)]
881                align_buffer,
882            } => {
883                let (buffer, _) = unsafe {
884                    prepare_for_rx(
885                        &mut **descriptors,
886                        #[cfg(dma_can_access_psram)]
887                        align_buffer,
888                        data,
889                    )
890                };
891                buffer
892            }
893        }
894    }
895
896    fn chunk_size(&self) -> usize {
897        match self {
898            MaybeCopyRxBuf::Copy(buffer) => buffer.capacity().min(MAX_DMA_SIZE),
899            MaybeCopyRxBuf::Direct { .. } => MAX_DMA_SIZE,
900        }
901    }
902
903    fn finish(&mut self, chunk: &mut [u8]) {
904        match self {
905            MaybeCopyRxBuf::Copy(buffer) => {
906                chunk.copy_from_slice(&buffer.as_slice()[..chunk.len()]);
907            }
908            MaybeCopyRxBuf::Direct {
909                #[cfg(dma_can_access_psram)]
910                align_buffer,
911                ..
912            } => {
913                #[cfg(soc_internal_memory_cached)]
914                unsafe {
915                    crate::soc::cache_invalidate_addr(chunk.as_ptr() as u32, chunk.len() as u32);
916                }
917
918                #[cfg(dma_can_access_psram)]
919                for buffer in align_buffer.iter_mut() {
920                    if let Some(buffer) = buffer.as_mut() {
921                        buffer.write_back();
922                    }
923                    *buffer = None;
924                }
925            }
926        }
927    }
928}
929
930#[derive(Clone, Copy)]
931enum DmaOperationKind {
932    /// The entire slice must be copied into the internal buffer first
933    Copied,
934
935    /// The slice can be transferred directly, with minimal copying done for alignment
936    InPlace,
937}
938
939impl DmaOperationKind {
940    fn compute(buffer: &[u8], direction: TransferDirection) -> Self {
941        fn is_dma_compatible(buffer: &[u8], _direction: TransferDirection) -> bool {
942            // FIXME: lazy workaround for ESP32 TX DMA alignment requirements.
943            // `prepare_for_tx` and `prepare_for_rx` should be updated to handle ESP32.
944            #[cfg(spi_master_version = "1")]
945            if !((buffer.as_ptr() as usize).is_multiple_of(4) && buffer.len().is_multiple_of(4)) {
946                return false;
947            }
948
949            if is_slice_in_dram(buffer) {
950                return true;
951            }
952            #[cfg(dma_can_access_psram)]
953            if is_slice_in_psram(buffer) {
954                #[cfg(spi_master_version = "2")]
955                if _direction == TransferDirection::In {
956                    // For some reason, having tail bytes in internal RAM causes issues, so we
957                    // force copying if the end of the PSRAM buffer is not aligned.
958                    let tail_bytes = (buffer.as_ptr() as usize + buffer.len()).wrapping_neg() & 15;
959                    if tail_bytes > 0 {
960                        return false;
961                    }
962                }
963
964                return true;
965            }
966
967            // TODO: C5+ DMA can read from flash
968
969            false
970        }
971
972        if is_dma_compatible(buffer, direction) {
973            Self::InPlace
974        } else {
975            Self::Copied
976        }
977    }
978
979    fn for_read(buffer: &mut [u8]) -> Self {
980        Self::compute(buffer, TransferDirection::In)
981    }
982
983    fn for_write(buffer: &[u8]) -> Self {
984        Self::compute(buffer, TransferDirection::Out)
985    }
986}
987
988impl<'d, Dm> SpiDma<'d, Dm>
989where
990    Dm: DriverMode,
991{
992    fn use_blocking_transfer(&self, transfer_size: usize) -> bool {
993        let threshold = self
994            .spi
995            .state()
996            .min_async_transfer_size
997            .load(Ordering::Relaxed);
998        threshold > 0 && transfer_size < threshold
999    }
1000
1001    fn spi(&self) -> &SpiWrapper<'_> {
1002        &self.spi
1003    }
1004
1005    fn driver(&self) -> Driver {
1006        Driver {
1007            info: self.spi.info(),
1008            state: self.spi.state(),
1009        }
1010    }
1011
1012    fn dma_driver(&self) -> DmaDriver {
1013        DmaDriver {
1014            driver: self.driver(),
1015            state: self.spi().dma_state(),
1016            dma_peripheral: self.spi.spi.dma_peripheral(),
1017        }
1018    }
1019
1020    fn is_done(&self) -> bool {
1021        if self.driver().busy() {
1022            return false;
1023        }
1024        if self.dma_driver().state.rx_transfer_in_progress.get() {
1025            // If this is an asymmetric transfer and the RX side is smaller, the RX channel
1026            // will never be "done" as it won't have enough descriptors/buffer to receive
1027            // the EOF bit from the SPI. So instead the RX channel will hit
1028            // a "descriptor empty" which means the DMA is written as much
1029            // of the received data as possible into the buffer and
1030            // discarded the rest. The user doesn't care about this discarded data.
1031
1032            if !self.channel.rx.is_done() && !self.channel.rx.has_dscr_empty_error() {
1033                return false;
1034            }
1035        }
1036        true
1037    }
1038
1039    fn wait_for_idle(&mut self) {
1040        while !self.is_done() {
1041            // Wait for the SPI to become idle
1042        }
1043        self.dma_driver().state.rx_transfer_in_progress.set(false);
1044        self.dma_driver().state.tx_transfer_in_progress.set(false);
1045        fence(Ordering::Acquire);
1046    }
1047
1048    /// # Safety:
1049    ///
1050    /// The caller must ensure to not access the buffer contents while the
1051    /// transfer is in progress. Moving the buffer itself is allowed.
1052    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1053    unsafe fn start_transfer_dma<RX: DmaRxBuffer, TX: DmaTxBuffer>(
1054        &mut self,
1055        full_duplex: bool,
1056        bytes_to_read: usize,
1057        bytes_to_write: usize,
1058        rx_buffer: &mut RX,
1059        tx_buffer: &mut TX,
1060    ) -> Result<(), Error> {
1061        if bytes_to_read > MAX_DMA_SIZE || bytes_to_write > MAX_DMA_SIZE {
1062            return Err(Error::MaxDmaTransferSizeExceeded);
1063        }
1064
1065        self.dma_driver()
1066            .state
1067            .rx_transfer_in_progress
1068            .set(bytes_to_read > 0);
1069        self.dma_driver()
1070            .state
1071            .tx_transfer_in_progress
1072            .set(bytes_to_write > 0);
1073        unsafe {
1074            self.dma_driver().start_transfer_dma(
1075                full_duplex,
1076                bytes_to_read,
1077                bytes_to_write,
1078                rx_buffer,
1079                tx_buffer,
1080                &mut self.channel,
1081            )
1082        }
1083    }
1084
1085    /// # Safety:
1086    ///
1087    /// The caller must ensure that the buffers are not accessed while the
1088    /// transfer is in progress. Moving the buffers is allowed.
1089    #[cfg(all(spi_master_version = "1", spi_address_workaround))]
1090    unsafe fn set_up_address_workaround(
1091        &mut self,
1092        cmd: Command,
1093        address: Address,
1094        dummy: u8,
1095    ) -> Result<(), Error> {
1096        if dummy > 0 {
1097            // FIXME: https://github.com/esp-rs/esp-hal/issues/2240
1098            error!("Dummy bits are not supported when there is no data to write");
1099            return Err(Error::Unsupported);
1100        }
1101
1102        let buffer = unsafe { self.dma_driver().tx_buffer() };
1103
1104        let bytes_to_write = address.width().div_ceil(8);
1105        // The address register is read in big-endian order,
1106        // we have to prepare the emulated write in the same way.
1107        let addr_bytes = address.value().to_be_bytes();
1108        let addr_bytes = &addr_bytes[4 - bytes_to_write..][..bytes_to_write];
1109        buffer.fill(addr_bytes);
1110
1111        self.driver().setup_half_duplex(
1112            true,
1113            cmd,
1114            Address::None,
1115            false,
1116            dummy,
1117            bytes_to_write == 0,
1118            address.mode(),
1119        )?;
1120
1121        let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1122
1123        unsafe { self.start_transfer_dma(false, 0, bytes_to_write, rx_buffer, buffer) }
1124    }
1125
1126    fn cancel_transfer(&mut self) {
1127        let state = self.dma_driver().state;
1128        if state.tx_transfer_in_progress.get() || state.rx_transfer_in_progress.get() {
1129            self.dma_driver().abort_transfer();
1130
1131            // We need to stop the DMA transfer, too.
1132            if state.tx_transfer_in_progress.get() {
1133                self.channel.tx.stop_transfer();
1134                state.tx_transfer_in_progress.set(false);
1135            }
1136            if state.rx_transfer_in_progress.get() {
1137                self.channel.rx.stop_transfer();
1138                state.rx_transfer_in_progress.set(false);
1139            }
1140        }
1141    }
1142
1143    /// # Safety:
1144    ///
1145    /// The caller must ensure that the buffers are not accessed while the
1146    /// transfer is in progress. Moving the buffers is allowed.
1147    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1148    unsafe fn start_dma_write(
1149        &mut self,
1150        bytes_to_write: usize,
1151        buffer: &mut impl DmaTxBuffer,
1152    ) -> Result<(), Error> {
1153        let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1154
1155        unsafe { self.start_dma_transfer(0, bytes_to_write, rx_buffer, buffer) }
1156    }
1157
1158    /// Assigns copy buffers to the SPI driver.
1159    ///
1160    /// These buffers will be used to copy data when using the slice-based transfer functions.
1161    ///
1162    /// Data is copied in two cases:
1163    ///   - When the buffer is not located in a memory region that can be accessed by the DMA.
1164    #[cfg_attr(
1165        not(spi_master_dma_can_access_flash),
1166        doc = "The DMA cannot read flash memory."
1167    )]
1168    ///   - When the alignment of the buffer does not meet the DMA's requirements, the unaligned
1169    ///     parts of the buffer are copied.
1170    #[cfg_attr(
1171        spi_master_version = "1",
1172        doc = "On ESP32, transferring from internal SRAM requires copying the entire buffer if it is
1173not 4-byte aligned. This is a limitation of the current implementation."
1174    )]
1175    #[cfg_attr(
1176        spi_master_version = "2",
1177        doc = "On ESP32-S2, receiving into PSRAM requires the buffer's _end_ to be 16-byte
1178aligned, otherwise the driver requires copying the entire buffer."
1179    )]
1180    #[doc = ""]
1181    /// The maximum useful size for these buffers is 32736 bytes, any additional memory will
1182    /// be wasted.
1183    ///
1184    /// For an example of how to create these buffers, see the [`SpiDma`] documentation.
1185    #[instability::unstable]
1186    pub fn with_buffers(self, dma_rx_buf: DmaRxBuf, dma_tx_buf: DmaTxBuf) -> SpiDma<'d, Dm> {
1187        unsafe {
1188            (&mut *self.spi.dma_state().rx_buffer.get()).write(dma_rx_buf.into_scoped());
1189            (&mut *self.spi.dma_state().tx_buffer.get()).write(dma_tx_buf.into_scoped());
1190        }
1191        self
1192    }
1193
1194    /// Perform a DMA write.
1195    ///
1196    /// This will return a [SpiDmaTransfer] owning the buffer and the
1197    /// SPI instance. The maximum amount of data to be sent is 32736
1198    /// bytes.
1199    #[allow(clippy::type_complexity)]
1200    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1201    #[instability::unstable]
1202    pub fn write_buffer<TX: DmaTxBuffer>(
1203        mut self,
1204        bytes_to_write: usize,
1205        mut buffer: TX,
1206    ) -> Result<SpiDmaTransfer<'d, Dm, TX>, (Error, Self, TX)> {
1207        let clock = SpiClockGuard::new(self.spi.info());
1208
1209        if let Err(e) = self.driver().setup_full_duplex() {
1210            return Err((e, self, buffer));
1211        };
1212        match unsafe { self.start_dma_write(bytes_to_write, &mut buffer) } {
1213            Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1214            Err(e) => Err((e, self, buffer)),
1215        }
1216    }
1217
1218    /// # Safety:
1219    ///
1220    /// The caller must ensure that the buffers are not accessed while the
1221    /// transfer is in progress. Moving the buffers is allowed.
1222    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1223    unsafe fn start_dma_read(
1224        &mut self,
1225        bytes_to_read: usize,
1226        buffer: &mut impl DmaRxBuffer,
1227    ) -> Result<(), Error> {
1228        let tx_buffer = unsafe { self.dma_driver().tx_buffer() };
1229
1230        unsafe { self.start_dma_transfer(bytes_to_read, 0, buffer, tx_buffer) }
1231    }
1232
1233    /// Perform a DMA read.
1234    ///
1235    /// This will return a [SpiDmaTransfer] owning the buffer and
1236    /// the SPI instance. The maximum amount of data to be
1237    /// received is 32736 bytes.
1238    #[allow(clippy::type_complexity)]
1239    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1240    #[instability::unstable]
1241    pub fn read_buffer<RX: DmaRxBuffer>(
1242        mut self,
1243        bytes_to_read: usize,
1244        mut buffer: RX,
1245    ) -> Result<SpiDmaTransfer<'d, Dm, RX>, (Error, Self, RX)> {
1246        let clock = SpiClockGuard::new(self.spi.info());
1247
1248        if let Err(e) = self.driver().setup_full_duplex() {
1249            return Err((e, self, buffer));
1250        };
1251        match unsafe { self.start_dma_read(bytes_to_read, &mut buffer) } {
1252            Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1253            Err(e) => Err((e, self, buffer)),
1254        }
1255    }
1256
1257    /// # Safety:
1258    ///
1259    /// The caller must ensure that the buffers are not accessed while the
1260    /// transfer is in progress. Moving the buffers is allowed.
1261    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1262    unsafe fn start_dma_transfer(
1263        &mut self,
1264        bytes_to_read: usize,
1265        bytes_to_write: usize,
1266        rx_buffer: &mut impl DmaRxBuffer,
1267        tx_buffer: &mut impl DmaTxBuffer,
1268    ) -> Result<(), Error> {
1269        unsafe {
1270            self.start_transfer_dma(true, bytes_to_read, bytes_to_write, rx_buffer, tx_buffer)
1271        }
1272    }
1273
1274    /// Perform a DMA transfer
1275    ///
1276    /// This will return a [SpiDmaTransfer] owning the buffers and
1277    /// the SPI instance. The maximum amount of data to be
1278    /// sent/received is 32736 bytes.
1279    #[allow(clippy::type_complexity)]
1280    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1281    #[instability::unstable]
1282    pub fn transfer_buffers<RX: DmaRxBuffer, TX: DmaTxBuffer>(
1283        mut self,
1284        bytes_to_read: usize,
1285        mut rx_buffer: RX,
1286        bytes_to_write: usize,
1287        mut tx_buffer: TX,
1288    ) -> Result<SpiDmaTransfer<'d, Dm, (RX, TX)>, (Error, Self, RX, TX)> {
1289        let clock = SpiClockGuard::new(self.spi.info());
1290
1291        if let Err(e) = self.driver().setup_full_duplex() {
1292            return Err((e, self, rx_buffer, tx_buffer));
1293        };
1294        match unsafe {
1295            self.start_dma_transfer(
1296                bytes_to_read,
1297                bytes_to_write,
1298                &mut rx_buffer,
1299                &mut tx_buffer,
1300            )
1301        } {
1302            Ok(_) => Ok(SpiDmaTransfer::new(self, (rx_buffer, tx_buffer), clock)),
1303            Err(e) => Err((e, self, rx_buffer, tx_buffer)),
1304        }
1305    }
1306
1307    /// # Safety:
1308    ///
1309    /// The caller must ensure that the buffers are not accessed while the
1310    /// transfer is in progress. Moving the buffers is allowed.
1311    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1312    unsafe fn start_half_duplex_read(
1313        &mut self,
1314        data_mode: DataMode,
1315        cmd: Command,
1316        address: Address,
1317        dummy: u8,
1318        bytes_to_read: usize,
1319        buffer: &mut impl DmaRxBuffer,
1320    ) -> Result<(), Error> {
1321        self.driver().setup_half_duplex(
1322            false,
1323            cmd,
1324            address,
1325            false,
1326            dummy,
1327            bytes_to_read == 0,
1328            data_mode,
1329        )?;
1330
1331        let tx_buffer = unsafe { self.dma_driver().tx_buffer() };
1332
1333        unsafe { self.start_transfer_dma(false, bytes_to_read, 0, buffer, tx_buffer) }
1334    }
1335
1336    /// Perform a half-duplex read operation using DMA.
1337    #[allow(clippy::type_complexity)]
1338    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1339    #[instability::unstable]
1340    pub fn half_duplex_read_buffer<RX: DmaRxBuffer>(
1341        mut self,
1342        data_mode: DataMode,
1343        cmd: Command,
1344        address: Address,
1345        dummy: u8,
1346        bytes_to_read: usize,
1347        mut buffer: RX,
1348    ) -> Result<SpiDmaTransfer<'d, Dm, RX>, (Error, Self, RX)> {
1349        let clock = SpiClockGuard::new(self.spi.info());
1350
1351        match unsafe {
1352            self.start_half_duplex_read(data_mode, cmd, address, dummy, bytes_to_read, &mut buffer)
1353        } {
1354            Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1355            Err(e) => Err((e, self, buffer)),
1356        }
1357    }
1358
1359    /// # Safety:
1360    ///
1361    /// The caller must ensure that the buffers are not accessed while the
1362    /// transfer is in progress. Moving the buffers is allowed.
1363    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1364    unsafe fn start_half_duplex_write(
1365        &mut self,
1366        data_mode: DataMode,
1367        cmd: Command,
1368        address: Address,
1369        dummy: u8,
1370        bytes_to_write: usize,
1371        buffer: &mut impl DmaTxBuffer,
1372    ) -> Result<(), Error> {
1373        #[cfg(all(spi_master_version = "1", spi_address_workaround))]
1374        {
1375            // On the ESP32, if we don't have data, the address is always sent
1376            // on a single line, regardless of its data mode.
1377            if bytes_to_write == 0 && address.mode() != DataMode::SingleTwoDataLines {
1378                return unsafe { self.set_up_address_workaround(cmd, address, dummy) };
1379            }
1380        }
1381
1382        self.driver().setup_half_duplex(
1383            true,
1384            cmd,
1385            address,
1386            false,
1387            dummy,
1388            bytes_to_write == 0,
1389            data_mode,
1390        )?;
1391
1392        let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1393
1394        unsafe { self.start_transfer_dma(false, 0, bytes_to_write, rx_buffer, buffer) }
1395    }
1396
1397    /// Perform a half-duplex write operation using DMA.
1398    #[allow(clippy::type_complexity)]
1399    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1400    #[instability::unstable]
1401    pub fn half_duplex_write_buffer<TX: DmaTxBuffer>(
1402        mut self,
1403        data_mode: DataMode,
1404        cmd: Command,
1405        address: Address,
1406        dummy: u8,
1407        bytes_to_write: usize,
1408        mut buffer: TX,
1409    ) -> Result<SpiDmaTransfer<'d, Dm, TX>, (Error, Self, TX)> {
1410        let clock = SpiClockGuard::new(self.spi.info());
1411
1412        match unsafe {
1413            self.start_half_duplex_write(
1414                data_mode,
1415                cmd,
1416                address,
1417                dummy,
1418                bytes_to_write,
1419                &mut buffer,
1420            )
1421        } {
1422            Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1423            Err(e) => Err((e, self, buffer)),
1424        }
1425    }
1426
1427    #[doc_replace(
1428        "max_frequency" => {
1429            cfg(esp32h2) => "48MHz",
1430            _ => "80MHz",
1431        }
1432    )]
1433    /// Change the bus configuration.
1434    ///
1435    /// # Errors
1436    ///
1437    /// If frequency passed in config exceeds __max_frequency__ or is below 70kHz,
1438    /// [`ConfigError::UnsupportedFrequency`] error will be returned.
1439    #[instability::unstable]
1440    pub fn apply_config(&mut self, config: &Config) -> Result<(), ConfigError> {
1441        self.driver().apply_config(config)
1442    }
1443
1444    fn transfer_buffers_dma(
1445        &mut self,
1446        read_bytes: usize,
1447        write_bytes: usize,
1448        mut rx_buffer: impl DmaRxBuffer,
1449        mut tx_buffer: impl DmaTxBuffer,
1450    ) -> Result<(), Error> {
1451        unsafe {
1452            self.start_dma_transfer(read_bytes, write_bytes, &mut rx_buffer, &mut tx_buffer)?;
1453        }
1454        self.wait_for_idle();
1455        Ok(())
1456    }
1457
1458    /// Reads data from the SPI bus using DMA.
1459    #[instability::unstable]
1460    pub fn read(&mut self, words: &mut [u8]) -> Result<(), Error> {
1461        let _clock = SpiClockGuard::new(self.spi.info());
1462
1463        self.driver().setup_full_duplex()?;
1464
1465        if self.use_blocking_transfer(words.len()) {
1466            self.dma_driver().disable_dma();
1467            return self.driver().read(words);
1468        }
1469
1470        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1471        let mut maybe_copy_buffer = match DmaOperationKind::for_read(words) {
1472            DmaOperationKind::Copied => {
1473                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
1474            }
1475            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
1476                descriptors: &mut descriptors,
1477                #[cfg(dma_can_access_psram)]
1478                align_buffer: [const { None }; 2],
1479            },
1480        };
1481
1482        if maybe_copy_buffer.chunk_size() == 0 {
1483            return Err(Error::from(DmaError::BufferTooSmall));
1484        }
1485
1486        for chunk in words.chunks_mut(maybe_copy_buffer.chunk_size()) {
1487            let read_bytes = chunk.len();
1488            let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *chunk)) };
1489            let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };
1490
1491            self.transfer_buffers_dma(read_bytes, 0, rx_buffer, tx_buffer)?;
1492
1493            maybe_copy_buffer.finish(chunk);
1494        }
1495
1496        Ok(())
1497    }
1498
1499    /// Writes data to the SPI bus using DMA.
1500    #[instability::unstable]
1501    pub fn write(&mut self, words: &[u8]) -> Result<(), Error> {
1502        let _clock = SpiClockGuard::new(self.spi.info());
1503
1504        self.driver().setup_full_duplex()?;
1505
1506        if self.use_blocking_transfer(words.len()) {
1507            self.dma_driver().disable_dma();
1508            return self.driver().write(words);
1509        }
1510
1511        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1512        let mut maybe_copy_buffer = match DmaOperationKind::for_write(words) {
1513            DmaOperationKind::Copied => {
1514                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
1515            }
1516            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),
1517        };
1518
1519        if maybe_copy_buffer.chunk_size() == 0 {
1520            return Err(Error::from(DmaError::BufferTooSmall));
1521        }
1522
1523        for chunk in words.chunks(maybe_copy_buffer.chunk_size()) {
1524            let write_bytes = chunk.len();
1525            let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) };
1526            let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(chunk)) };
1527
1528            self.transfer_buffers_dma(0, write_bytes, rx_buffer, tx_buffer)?;
1529        }
1530
1531        Ok(())
1532    }
1533
1534    /// Transfers data to and from the SPI bus simultaneously using DMA.
1535    #[instability::unstable]
1536    pub fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Error> {
1537        let _clock = SpiClockGuard::new(self.spi.info());
1538
1539        self.driver().setup_full_duplex()?;
1540
1541        if self.use_blocking_transfer(read.len().max(write.len())) {
1542            self.dma_driver().disable_dma();
1543            if read.is_empty() {
1544                return self.driver().write(write);
1545            } else if write.is_empty() {
1546                return self.driver().read(read);
1547            } else {
1548                return self.driver().transfer(read, write);
1549            }
1550        }
1551
1552        let common_length = min(read.len(), write.len());
1553        let (read_common, read_remainder) = read.split_at_mut(common_length);
1554        let (write_common, write_remainder) = write.split_at(common_length);
1555
1556        // DmaOperationKind must be determined on the sub-slices actually passed to DMA.
1557        let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1558        let mut maybe_copy_rx_buffer = match DmaOperationKind::for_read(read_common) {
1559            DmaOperationKind::Copied => {
1560                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
1561            }
1562            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
1563                descriptors: &mut rx_descriptors,
1564                #[cfg(dma_can_access_psram)]
1565                align_buffer: [const { None }; 2],
1566            },
1567        };
1568
1569        let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1570        let mut maybe_copy_tx_buffer = match DmaOperationKind::for_write(write_common) {
1571            DmaOperationKind::Copied => {
1572                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
1573            }
1574            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut tx_descriptors),
1575        };
1576
1577        let chunk_size = min(
1578            maybe_copy_rx_buffer.chunk_size(),
1579            maybe_copy_tx_buffer.chunk_size(),
1580        );
1581
1582        if chunk_size == 0 {
1583            return Err(Error::from(DmaError::BufferTooSmall));
1584        }
1585
1586        for (read_chunk, write_chunk) in read_common
1587            .chunks_mut(chunk_size)
1588            .zip(write_common.chunks(chunk_size))
1589        {
1590            let read_bytes = read_chunk.len();
1591            let write_bytes = write_chunk.len();
1592            let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(NonNull::from(write_chunk)) };
1593            let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(NonNull::from(&mut *read_chunk)) };
1594
1595            self.transfer_buffers_dma(read_bytes, write_bytes, rx_buffer, tx_buffer)?;
1596
1597            maybe_copy_rx_buffer.finish(read_chunk);
1598        }
1599
1600        if !read_remainder.is_empty() {
1601            self.read(read_remainder)
1602        } else if !write_remainder.is_empty() {
1603            self.write(write_remainder)
1604        } else {
1605            Ok(())
1606        }
1607    }
1608
1609    /// Transfers data in place on the SPI bus using DMA.
1610    #[instability::unstable]
1611    pub fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Error> {
1612        let _clock = SpiClockGuard::new(self.spi.info());
1613
1614        self.driver().setup_full_duplex()?;
1615
1616        if self.use_blocking_transfer(words.len()) {
1617            self.dma_driver().disable_dma();
1618            return self.driver().transfer_in_place(words);
1619        }
1620
1621        let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1622        let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1623        let (mut maybe_copy_rx_buffer, mut maybe_copy_tx_buffer) =
1624            match DmaOperationKind::for_write(words) {
1625                DmaOperationKind::Copied => (
1626                    MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }),
1627                    MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }),
1628                ),
1629                DmaOperationKind::InPlace => (
1630                    MaybeCopyRxBuf::Direct {
1631                        descriptors: &mut rx_descriptors,
1632                        #[cfg(dma_can_access_psram)]
1633                        align_buffer: [const { None }; 2],
1634                    },
1635                    MaybeCopyTxBuf::Direct(&mut tx_descriptors),
1636                ),
1637            };
1638
1639        let chunk_size = min(
1640            maybe_copy_rx_buffer.chunk_size(),
1641            maybe_copy_tx_buffer.chunk_size(),
1642        );
1643
1644        if chunk_size == 0 {
1645            return Err(Error::from(DmaError::BufferTooSmall));
1646        }
1647
1648        for chunk in words.chunks_mut(chunk_size) {
1649            let bytes = chunk.len();
1650            let ptr = NonNull::from(&mut *chunk);
1651            let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(ptr) };
1652            let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(ptr) };
1653
1654            self.transfer_buffers_dma(bytes, bytes, rx_buffer, tx_buffer)?;
1655
1656            maybe_copy_rx_buffer.finish(chunk);
1657        }
1658
1659        Ok(())
1660    }
1661
1662    /// Half-duplex read.
1663    #[instability::unstable]
1664    pub fn half_duplex_read(
1665        &mut self,
1666        data_mode: DataMode,
1667        cmd: Command,
1668        address: Address,
1669        dummy: u8,
1670        buffer: &mut [u8],
1671    ) -> Result<(), Error> {
1672        let _clock = SpiClockGuard::new(self.spi.info());
1673
1674        let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1675        if rx_buffer.capacity() == 0 {
1676            return Err(Error::from(DmaError::BufferTooSmall));
1677        }
1678        if buffer.len() > rx_buffer.capacity() {
1679            return Err(Error::from(DmaError::Overflow));
1680        }
1681
1682        unsafe {
1683            self.start_half_duplex_read(data_mode, cmd, address, dummy, buffer.len(), rx_buffer)?;
1684        }
1685
1686        self.wait_for_idle();
1687
1688        buffer.copy_from_slice(&rx_buffer.as_slice()[..buffer.len()]);
1689
1690        Ok(())
1691    }
1692
1693    /// Half-duplex write.
1694    #[instability::unstable]
1695    pub fn half_duplex_write(
1696        &mut self,
1697        data_mode: DataMode,
1698        cmd: Command,
1699        address: Address,
1700        dummy: u8,
1701        buffer: &[u8],
1702    ) -> Result<(), Error> {
1703        let _clock = SpiClockGuard::new(self.spi.info());
1704
1705        let tx_buffer = unsafe { self.dma_driver().tx_buffer() };
1706        if tx_buffer.capacity() == 0 {
1707            return Err(Error::from(DmaError::BufferTooSmall));
1708        }
1709        if buffer.len() > tx_buffer.capacity() {
1710            return Err(Error::from(DmaError::Overflow));
1711        }
1712
1713        tx_buffer.as_mut_slice()[..buffer.len()].copy_from_slice(buffer);
1714
1715        unsafe {
1716            self.start_half_duplex_write(data_mode, cmd, address, dummy, buffer.len(), tx_buffer)?;
1717        }
1718
1719        self.wait_for_idle();
1720
1721        Ok(())
1722    }
1723}
1724
1725/// A structure representing a DMA transfer for SPI.
1726///
1727/// This structure holds references to the SPI instance, DMA buffers, and
1728/// transfer status.
1729#[instability::unstable]
1730pub struct SpiDmaTransfer<'d, Dm, Buf>
1731where
1732    Dm: DriverMode,
1733{
1734    spi_dma: ManuallyDrop<SpiDma<'d, Dm>>,
1735    dma_buf: ManuallyDrop<Buf>,
1736    clock: ManuallyDrop<SpiClockGuard>,
1737}
1738
1739impl<Buf> SpiDmaTransfer<'_, Async, Buf> {
1740    /// Waits for the DMA transfer to complete asynchronously.
1741    ///
1742    /// This method awaits the completion of both RX and TX operations.
1743    #[instability::unstable]
1744    pub async fn wait_for_done(&mut self) {
1745        self.spi_dma.wait_for_idle_async().await;
1746    }
1747}
1748
1749impl<'d, Dm, Buf> SpiDmaTransfer<'d, Dm, Buf>
1750where
1751    Dm: DriverMode,
1752{
1753    fn new(spi_dma: SpiDma<'d, Dm>, dma_buf: Buf, clock: SpiClockGuard) -> Self {
1754        Self {
1755            spi_dma: ManuallyDrop::new(spi_dma),
1756            dma_buf: ManuallyDrop::new(dma_buf),
1757            clock: ManuallyDrop::new(clock),
1758        }
1759    }
1760
1761    /// Checks if the transfer is complete.
1762    ///
1763    /// This method returns `true` if both RX and TX operations are done,
1764    /// and the SPI instance is no longer busy.
1765    #[instability::unstable]
1766    pub fn is_done(&self) -> bool {
1767        self.spi_dma.is_done()
1768    }
1769
1770    /// Waits for the DMA transfer to complete.
1771    ///
1772    /// This method blocks until the transfer is finished and returns the
1773    /// `SpiDma` instance and the associated buffer.
1774    #[instability::unstable]
1775    pub fn wait(mut self) -> (SpiDma<'d, Dm>, Buf) {
1776        self.spi_dma.wait_for_idle();
1777        let retval = unsafe {
1778            (
1779                ManuallyDrop::take(&mut self.spi_dma),
1780                ManuallyDrop::take(&mut self.dma_buf),
1781            )
1782        };
1783        let _ = unsafe { ManuallyDrop::take(&mut self.clock) };
1784        core::mem::forget(self);
1785        retval
1786    }
1787
1788    /// Cancels the DMA transfer.
1789    #[instability::unstable]
1790    pub fn cancel(&mut self) {
1791        if !self.spi_dma.is_done() {
1792            self.spi_dma.cancel_transfer();
1793        }
1794    }
1795}
1796
1797impl<Dm, Buf> Drop for SpiDmaTransfer<'_, Dm, Buf>
1798where
1799    Dm: DriverMode,
1800{
1801    fn drop(&mut self) {
1802        if !self.is_done() {
1803            self.spi_dma.cancel_transfer();
1804            self.spi_dma.wait_for_idle();
1805        }
1806
1807        unsafe {
1808            ManuallyDrop::drop(&mut self.spi_dma);
1809            ManuallyDrop::drop(&mut self.dma_buf);
1810        }
1811        let _ = unsafe { ManuallyDrop::take(&mut self.clock) };
1812    }
1813}
1814
1815pub(super) struct DmaDriver {
1816    driver: Driver,
1817    dma_peripheral: crate::dma::DmaPeripheral,
1818    state: &'static DmaState,
1819}
1820
1821impl DmaDriver {
1822    unsafe fn rx_buffer(&self) -> &'static mut ScopedDmaRxBuf<'static> {
1823        unsafe { self.state.rx_buffer() }
1824    }
1825
1826    unsafe fn tx_buffer(&self) -> &'static mut ScopedDmaTxBuf<'static> {
1827        unsafe { self.state.tx_buffer() }
1828    }
1829
1830    fn abort_transfer(&self) {
1831        // The SPI peripheral is controlling how much data we transfer, so let's
1832        // update its counter.
1833        // 0 doesn't take effect on ESP32 and cuts the currently transmitted byte
1834        // immediately.
1835        // 1 seems to stop after transmitting the current byte which is somewhat less
1836        // impolite.
1837        self.driver.configure_datalen(1, 1);
1838        self.driver.update();
1839    }
1840
1841    fn disable_dma(&self) {
1842        #[cfg(not(any(spi_master_version = "1", spi_master_version = "2")))]
1843        self.regs().dma_conf().modify(|_, w| {
1844            w.dma_tx_ena().clear_bit();
1845            w.dma_rx_ena().clear_bit()
1846        });
1847
1848        // PDMA: nothing to do
1849    }
1850
1851    fn regs(&self) -> &RegisterBlock {
1852        self.driver.regs()
1853    }
1854
1855    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1856    unsafe fn start_transfer_dma<Dm: DriverMode>(
1857        &self,
1858        _full_duplex: bool,
1859        rx_len: usize,
1860        tx_len: usize,
1861        rx_buffer: &mut impl DmaRxBuffer,
1862        tx_buffer: &mut impl DmaTxBuffer,
1863        channel: &mut Channel<Dm, SpiMasterErased<'_>>,
1864    ) -> Result<(), Error> {
1865        #[cfg(spi_master_version = "2")]
1866        {
1867            // without this a transfer after a write will fail
1868            self.regs().dma_out_link().write(|w| unsafe { w.bits(0) });
1869            self.regs().dma_in_link().write(|w| unsafe { w.bits(0) });
1870        }
1871
1872        self.driver.configure_datalen(rx_len, tx_len);
1873
1874        // enable the MISO and MOSI if needed
1875        self.regs()
1876            .user()
1877            .modify(|_, w| w.usr_miso().bit(rx_len > 0).usr_mosi().bit(tx_len > 0));
1878
1879        self.enable_dma();
1880
1881        if rx_len > 0 {
1882            unsafe {
1883                channel
1884                    .rx
1885                    .prepare_transfer(self.dma_peripheral, rx_buffer)
1886                    .and_then(|_| channel.rx.start_transfer())?;
1887            }
1888        } else {
1889            #[cfg(spi_master_version = "1")]
1890            {
1891                // see https://github.com/espressif/esp-idf/commit/366e4397e9dae9d93fe69ea9d389b5743295886f
1892                // see https://github.com/espressif/esp-idf/commit/0c3653b1fd7151001143451d4aa95dbf15ee8506
1893                if _full_duplex {
1894                    self.regs()
1895                        .dma_in_link()
1896                        .modify(|_, w| unsafe { w.inlink_addr().bits(0) });
1897                    self.regs()
1898                        .dma_in_link()
1899                        .modify(|_, w| w.inlink_start().set_bit());
1900                }
1901            }
1902        }
1903        if tx_len > 0 {
1904            unsafe {
1905                channel
1906                    .tx
1907                    .prepare_transfer(self.dma_peripheral, tx_buffer)
1908                    .and_then(|_| channel.tx.start_transfer())?;
1909            }
1910        }
1911
1912        #[cfg(not(any(spi_master_version = "1", spi_master_version = "2")))]
1913        self.reset_dma();
1914
1915        self.driver.start_operation();
1916
1917        Ok(())
1918    }
1919
1920    fn enable_dma(&self) {
1921        cfg_select! {
1922            any(spi_master_version = "1", spi_master_version = "2") => {
1923                self.reset_dma();
1924            }
1925            _ => {
1926                self.regs().dma_conf().modify(|_, w| {
1927                    w.dma_tx_ena().set_bit();
1928                    w.dma_rx_ena().set_bit()
1929                });
1930            }
1931        }
1932    }
1933
1934    fn reset_dma(&self) {
1935        self.regs().dma_conf().toggle(|w, bit| {
1936            cfg_select! {
1937                any(spi_master_version = "1", spi_master_version = "2") => {
1938                    w.out_rst().bit(bit);
1939                    w.in_rst().bit(bit);
1940                    w.ahbm_fifo_rst().bit(bit);
1941                    w.ahbm_rst().bit(bit)
1942                }
1943                _ => {
1944                    w.rx_afifo_rst().bit(bit);
1945                    w.buf_afifo_rst().bit(bit);
1946                    w.dma_afifo_rst().bit(bit)
1947                }
1948            }
1949        });
1950
1951        self.clear_dma_interrupts();
1952    }
1953
1954    fn clear_dma_interrupts(&self) {
1955        self.regs().dma_int_clr().write(|w| {
1956            cfg_select! {
1957                any(spi_master_version = "1", spi_master_version = "2") => {
1958                    w.inlink_dscr_empty().clear_bit_by_one();
1959                    w.outlink_dscr_error().clear_bit_by_one();
1960                    w.inlink_dscr_error().clear_bit_by_one();
1961                    w.in_done().clear_bit_by_one();
1962                    w.in_err_eof().clear_bit_by_one();
1963                    w.in_suc_eof().clear_bit_by_one();
1964                    w.out_done().clear_bit_by_one();
1965                    w.out_eof().clear_bit_by_one();
1966                    w.out_total_eof().clear_bit_by_one()
1967                }
1968                _ => {
1969                    w.dma_infifo_full_err().clear_bit_by_one();
1970                    w.dma_outfifo_empty_err().clear_bit_by_one();
1971                    w.trans_done().clear_bit_by_one();
1972                    w.mst_rx_afifo_wfull_err().clear_bit_by_one();
1973                    w.mst_tx_afifo_rempty_err().clear_bit_by_one()
1974                }
1975            }
1976        });
1977    }
1978}
1979
1980struct DmaState {
1981    tx_transfer_in_progress: Cell<bool>,
1982    rx_transfer_in_progress: Cell<bool>,
1983
1984    rx_buffer: UnsafeCell<MaybeUninit<ScopedDmaRxBuf<'static>>>,
1985    tx_buffer: UnsafeCell<MaybeUninit<ScopedDmaTxBuf<'static>>>,
1986
1987    descriptors: UnsafeCell<InternalMemory<[DmaDescriptor; 2]>>,
1988
1989    #[cfg(all(spi_master_version = "1", spi_address_workaround))]
1990    default_tx_buffer: UnsafeCell<InternalMemory<[u8; 4]>>,
1991}
1992
1993impl DmaState {
1994    // Syntactic helper to get a mutable reference to the "empty" RX DMA buffer.
1995    //
1996    // # Safety
1997    //
1998    // The caller must ensure that Rust's aliasing rules are upheld.
1999    #[allow(
2000        clippy::mut_from_ref,
2001        reason = "Safety requirements ensure this is okay"
2002    )]
2003    unsafe fn rx_buffer(&self) -> &mut ScopedDmaRxBuf<'static> {
2004        unsafe { (&mut *self.rx_buffer.get()).assume_init_mut() }
2005    }
2006
2007    // Syntactic helper to get a mutable reference to the "empty" TX DMA buffer.
2008    //
2009    // # Safety
2010    //
2011    // The caller must ensure that Rust's aliasing rules are upheld.
2012    #[allow(
2013        clippy::mut_from_ref,
2014        reason = "Safety requirements ensure this is okay"
2015    )]
2016    unsafe fn tx_buffer(&self) -> &mut ScopedDmaTxBuf<'static> {
2017        unsafe { (&mut *self.tx_buffer.get()).assume_init_mut() }
2018    }
2019}
2020
2021// SAFETY: State belongs to the currently constructed driver instance. As such, it'll not be
2022// accessed concurrently in multiple threads.
2023unsafe impl Sync for DmaState {}
2024
2025for_each_spi_master!(
2026    (all $( ($peri:ident, $sys:ident, $sclk:ident $_cs:tt $_sio:tt $(, $is_qspi:tt)?)),* ) => {
2027        impl AnySpi<'_> {
2028            #[inline(always)]
2029            fn dma_state(&self) -> &'static DmaState {
2030                match &self.0 {
2031                    $(
2032                        super::any::Inner::$sys(_spi) => {
2033                            static DMA_STATE: DmaState = DmaState {
2034                                tx_transfer_in_progress: Cell::new(false),
2035                                rx_transfer_in_progress: Cell::new(false),
2036
2037                                rx_buffer: UnsafeCell::new(MaybeUninit::uninit()),
2038                                tx_buffer: UnsafeCell::new(MaybeUninit::uninit()),
2039
2040                                descriptors: UnsafeCell::new(InternalMemory::new([DmaDescriptor::EMPTY; 2])),
2041                                #[cfg(all(spi_master_version = "1", spi_address_workaround))]
2042                                default_tx_buffer: UnsafeCell::new(InternalMemory::new([0; 4])),
2043                            };
2044
2045                            &DMA_STATE
2046                        }
2047                    )*
2048                }
2049            }
2050        }
2051    };
2052);
2053
2054impl SpiWrapper<'_> {
2055    fn dma_state(&self) -> &'static DmaState {
2056        self.spi.dma_state()
2057    }
2058}
2059
2060with_spi_master_dma_engine! {
2061    ($engine:tt, $any_channel:ident) => {
2062        /// DMA channel trait for SPI peripherals.
2063        ///
2064        /// Implemented for each channel type that can serve a particular SPI instance `S`.
2065        #[instability::unstable]
2066        #[diagnostic::on_unimplemented(
2067            message = "The DMA channel cannot be used with this SPI peripheral",
2068            label = "This DMA channel",
2069            note = "Use a channel that matches the SPI instance."
2070        )]
2071        pub trait SpiMasterDmaChannel<'d, S>: crate::private::Sealed + Into<crate::dma::$any_channel<'d>> {}
2072
2073        crate::macros::impl_dma_channel_trait! {
2074            $engine,
2075            any_peri = AnySpi<'d>,
2076            peris = for_each_spi_master,
2077            ($peri:path, $ch:path) => {
2078                impl<'d> SpiMasterDmaChannel<'d, $peri> for $ch {}
2079            }
2080        }
2081
2082        // Proxy type so that the type-erased DMA channel can be named in the driver, regardless of the DMA engine.
2083        type SpiMasterErased<'d> = crate::dma::$any_channel<'d>;
2084    };
2085}
esp_hal/spi/master/dma.rs

esp_hal/spi/master/
dma.rs
