Skip to main content

esp_hal/spi/master/
dma.rs

1use core::{
2    cell::{Cell, UnsafeCell},
3    cmp::min,
4    mem::{ManuallyDrop, MaybeUninit},
5    pin::Pin,
6    ptr::NonNull,
7    sync::atomic::{Ordering, fence},
8    task::{Context, Poll},
9};
10
11#[cfg(feature = "unstable")]
12use embedded_hal::spi::{ErrorType, SpiBus};
13use enumset::EnumSet;
14#[cfg(place_spi_master_driver_in_ram)]
15use procmacros::ram;
16
17use super::*;
18use crate::{
19    RegisterToggle,
20    dma::{
21        CHUNK_SIZE,
22        Channel,
23        DmaDescriptor,
24        DmaEligiblePeripheral,
25        DmaRxBuf,
26        DmaRxBuffer,
27        DmaTxBuf,
28        DmaTxBuffer,
29        NoBuffer,
30        ScopedDmaRxBuf,
31        ScopedDmaTxBuf,
32        TransferDirection,
33        aligned::{DmaAlignedMut, InternalMemory},
34        asynch::DmaRxFuture,
35        prepare_for_rx,
36        prepare_for_tx,
37    },
38    pac::spi2::RegisterBlock,
39    private::DropGuard,
40    soc::is_slice_in_dram,
41    spi::{DmaError, master::low_level::SpiClockGuard},
42};
43#[cfg(dma_can_access_psram)]
44use crate::{dma::ManualWritebackBuffer, soc::is_slice_in_psram};
45
46const MAX_DMA_SIZE: usize = 32736;
47
48impl<'d> Spi<'d, Blocking> {
49    #[doc_replace(
50        "dma_channel" => {
51            cfg(spi_master_dma_engine = "SPI_DMA") => "DMA_SPI2",
52            cfg(spi_master_dma_engine = "AHB_GDMA") => "DMA_CH0",
53            cfg(spi_master_dma_engine = "AXI_GDMA") => "DMA_AXI_CH0",
54        }
55    )]
56    /// Converts the driver into an [`SpiDma`] driver that uses the specified DMA channel.
57    ///
58    /// ```rust, no_run
59    /// # {before_snippet}
60    /// use esp_hal::spi::{
61    ///     Mode,
62    ///     master::{Config, Spi},
63    /// };
64    ///
65    /// let mut spi_dma = Spi::new(
66    ///     peripherals.SPI2,
67    ///     Config::default()
68    ///         .with_frequency(Rate::from_khz(100))
69    ///         .with_mode(Mode::_0),
70    /// )?
71    /// .with_dma(peripherals.__dma_channel__);
72    /// # {after_snippet}
73    /// ```
74    #[instability::unstable]
75    pub fn with_dma(
76        self,
77        channel: impl SpiMasterDmaChannel<'d, AnySpi<'d>>,
78    ) -> SpiDma<'d, crate::Blocking> {
79        SpiDma::new_from_spi(self, channel.into())
80    }
81}
82
83#[doc_replace(
84    "dma_channel" => {
85        cfg(spi_master_dma_engine = "SPI_DMA") => "DMA_SPI2",
86        cfg(spi_master_dma_engine = "AHB_GDMA") => "DMA_CH0",
87        cfg(spi_master_dma_engine = "AXI_GDMA") => "DMA_AXI_CH0",
88    }
89)]
90/// DMA-controlled SPI driver.
91///
92/// This driver uses DMA to transfer data, allowing the CPU to continue working while the SPI
93/// transfer is in progress.
94///
95/// The driver provides two separate approaches to transferring data:
96///
97/// - The slice-based API allows transferring data from/to slices of memory. The data may be copied
98///   into an internal buffer before the transfer begins. A pair of copy buffers can be set up by
99///   passing them to [`with_buffers`](SpiDma::with_buffers) before the first transfer begins. For
100///   more details on when copying is necessary, see the documentation of the
101///   [`with_buffers`](SpiDma::with_buffers) method.
102/// - The buffer API allows transferring externally managed buffers. In this mode, the buffers to be
103///   transferred are provided by the caller. The buffer objects ensure that data is located in
104///   appropriate memory regions. The buffers and the driver object are moved into transfer objects
105///   for the duration of the transfer. These functions take [`DmaRxBuf`] and [`DmaTxBuf`] objects
106///   as arguments as well as the number of bytes to transfer, and their names end with `_buffer`
107///
108/// These approaches provide different trade-offs between memory usage / CPU overhead and ease of
109/// use. `embedded-hal` traits are implemented by the slice-based API's functions.
110///
111/// # Examples
112///
113/// ```rust, no_run
114/// # {before_snippet}
115/// use esp_hal::{
116///     dma::{DmaRxBuf, DmaTxBuf},
117///     dma_rx_buffer,
118///     dma_tx_buffer,
119///     spi::{
120///         Mode,
121///         master::{Config, Spi},
122///     },
123/// };
124///
125/// // Optional: create and set up copy buffers.
126/// let dma_rx_buf = dma_rx_buffer!(32000)?;
127/// let dma_tx_buf = dma_tx_buffer!(32000)?;
128///
129/// let mut spi = Spi::new(
130///     peripherals.SPI2,
131///     Config::default()
132///         .with_frequency(Rate::from_khz(100))
133///         .with_mode(Mode::_0),
134/// )?
135/// .with_dma(peripherals.__dma_channel__)
136/// .with_buffers(dma_rx_buf, dma_tx_buf);
137/// #
138/// # {after_snippet}
139/// ```
140#[cfg_attr(feature = "defmt", derive(defmt::Format))]
141pub struct SpiDma<'d, Dm>
142where
143    Dm: DriverMode,
144{
145    spi: SpiWrapper<'d>,
146    pub(crate) channel: Channel<Dm, SpiMasterErased<'d>>,
147}
148
149impl<Dm> crate::private::Sealed for SpiDma<'_, Dm> where Dm: DriverMode {}
150
151impl<Dm> core::fmt::Debug for SpiDma<'_, Dm>
152where
153    Dm: DriverMode + core::fmt::Debug,
154{
155    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
156        f.debug_struct("SpiDma").field("spi", &self.spi).finish()
157    }
158}
159
160#[instability::unstable]
161impl crate::interrupt::InterruptConfigurable for SpiDma<'_, Blocking> {
162    fn set_interrupt_handler(&mut self, handler: InterruptHandler) {
163        self.set_interrupt_handler(handler);
164    }
165}
166
167#[instability::unstable]
168impl<Dm> embassy_embedded_hal::SetConfig for SpiDma<'_, Dm>
169where
170    Dm: DriverMode,
171{
172    type Config = Config;
173    type ConfigError = ConfigError;
174
175    fn set_config(&mut self, config: &Self::Config) -> Result<(), Self::ConfigError> {
176        self.apply_config(config)
177    }
178}
179
180#[instability::unstable]
181impl<Dm> ErrorType for SpiDma<'_, Dm>
182where
183    Dm: DriverMode,
184{
185    type Error = Error;
186}
187
188#[instability::unstable]
189impl<Dm> SpiBus for SpiDma<'_, Dm>
190where
191    Dm: DriverMode,
192{
193    fn read(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
194        self.read(words)
195    }
196
197    fn write(&mut self, words: &[u8]) -> Result<(), Self::Error> {
198        self.write(words)
199    }
200
201    fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Self::Error> {
202        self.transfer(read, write)
203    }
204
205    fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
206        self.transfer_in_place(words)
207    }
208
209    fn flush(&mut self) -> Result<(), Self::Error> {
210        // DMA limitation - we must ensure the transfers complete before returning
211        // to user code, otherwise the user might access the buffers while the transfer
212        // is still in progress. Therefore, there is no such thing as "flushing".
213        Ok(())
214    }
215}
216
217#[instability::unstable]
218impl embedded_hal_async::spi::SpiBus for SpiDma<'_, Async> {
219    async fn read(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
220        self.read_async(words).await
221    }
222
223    async fn write(&mut self, words: &[u8]) -> Result<(), Self::Error> {
224        self.write_async(words).await
225    }
226
227    async fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Self::Error> {
228        self.transfer_async(read, write).await
229    }
230
231    async fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
232        self.transfer_in_place_async(words).await
233    }
234
235    async fn flush(&mut self) -> Result<(), Self::Error> {
236        // DMA limitation - we must ensure the transfers complete before returning
237        // to user code, otherwise the user might access the buffers while the transfer
238        // is still in progress. Therefore, there is no such thing as "flushing".
239        Ok(())
240    }
241}
242
243impl<'d> SpiDma<'d, Blocking> {
244    /// Converts the SPI driver into async mode.
245    #[instability::unstable]
246    pub fn into_async(self) -> SpiDma<'d, Async> {
247        self.spi
248            .set_interrupt_handler(self.spi.info().async_handler);
249        SpiDma {
250            spi: self.spi,
251            channel: self.channel.into_async(),
252        }
253    }
254
255    fn new_inner(spi: SpiWrapper<'d>, channel: SpiMasterErased<'d>) -> Self {
256        let channel = Channel::new(channel);
257        channel.runtime_ensure_compatible(spi.spi.dma_peripheral());
258
259        let state = spi.spi.dma_state();
260
261        state.tx_transfer_in_progress.set(false);
262        state.rx_transfer_in_progress.set(false);
263
264        // Safety: The descriptors occupy their own shared cache line and are updated in a
265        // synchronised fashion.
266        let (tx_descriptors, rx_descriptors) = unsafe {
267            let descriptors = (&mut *state.descriptors.get()).get_mut().into_inner();
268            descriptors.fill(DmaDescriptor::EMPTY);
269            let (tx_descriptors, rx_descriptors) = descriptors.split_at_mut(1);
270            (
271                DmaAlignedMut::new_unchecked(tx_descriptors),
272                DmaAlignedMut::new_unchecked(rx_descriptors),
273            )
274        };
275
276        let tx_buffer = cfg_select! {
277            all(spi_master_version = "1", spi_address_workaround) => unsafe {
278                (&mut *state.default_tx_buffer.get()).get_mut().unsize()
279            },
280            _ => unsafe { DmaAlignedMut::new_unchecked(&mut [][..]) },
281        };
282
283        let rx_buffer = unwrap!(DmaRxBuf::new(rx_descriptors, unsafe {
284            DmaAlignedMut::new_unchecked(&mut [])
285        }));
286        let tx_buffer = unwrap!(DmaTxBuf::new(tx_descriptors, tx_buffer));
287
288        // The buffers must be set up when creating the driver.
289        unsafe { (&mut *state.tx_buffer.get()).write(tx_buffer.into_scoped()) };
290        unsafe { (&mut *state.rx_buffer.get()).write(rx_buffer.into_scoped()) };
291
292        Self { spi, channel }
293    }
294
295    pub(super) fn new_from_spi(
296        spi_driver: Spi<'d, Blocking>,
297        channel: SpiMasterErased<'d>,
298    ) -> Self {
299        let spi = spi_driver.spi;
300
301        Self::new_inner(spi, channel)
302    }
303
304    /// Listens for the given interrupts.
305    #[instability::unstable]
306    pub fn listen(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {
307        self.driver().enable_listen(interrupts.into(), true);
308    }
309
310    /// Unlistens from the given interrupts.
311    #[instability::unstable]
312    pub fn unlisten(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {
313        self.driver().enable_listen(interrupts.into(), false);
314    }
315
316    /// Returns the asserted interrupts.
317    #[instability::unstable]
318    pub fn interrupts(&mut self) -> EnumSet<SpiInterrupt> {
319        self.driver().interrupts()
320    }
321
322    /// Resets asserted interrupts.
323    #[instability::unstable]
324    pub fn clear_interrupts(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {
325        self.driver().clear_interrupts(interrupts.into());
326    }
327
328    #[cfg_attr(
329        not(multi_core),
330        doc = "Registers an interrupt handler for the peripheral."
331    )]
332    #[cfg_attr(
333        multi_core,
334        doc = "Registers an interrupt handler for the peripheral on the current core."
335    )]
336    #[doc = ""]
337    /// Replaces any previously registered interrupt handlers.
338    ///
339    /// The default/unhandled interrupt handler can be restored with
340    /// [crate::interrupt::DEFAULT_INTERRUPT_HANDLER]
341    ///
342    /// # Panics
343    ///
344    /// Panics if passed interrupt handler is invalid (e.g. has priority
345    /// `None`)
346    #[instability::unstable]
347    pub fn set_interrupt_handler(&mut self, handler: InterruptHandler) {
348        self.spi.set_interrupt_handler(handler);
349    }
350}
351
352impl<'d> SpiDma<'d, Async> {
353    /// Converts the SPI instance into blocking mode.
354    #[instability::unstable]
355    pub fn into_blocking(self) -> SpiDma<'d, Blocking> {
356        self.spi.disable_peri_interrupt_on_all_cores();
357        SpiDma {
358            spi: self.spi,
359            channel: self.channel.into_blocking(),
360        }
361    }
362
363    async fn wait_for_idle_async(&mut self) {
364        if self.dma_driver().state.rx_transfer_in_progress.get() {
365            _ = DmaRxFuture::new(&mut self.channel.rx).await;
366            self.dma_driver().state.rx_transfer_in_progress.set(false);
367        }
368
369        struct Fut(Driver);
370        impl Fut {
371            const DONE_EVENTS: EnumSet<SpiInterrupt> =
372                enumset::enum_set!(SpiInterrupt::TransferDone);
373        }
374        impl Future for Fut {
375            type Output = ();
376
377            fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
378                if !self.0.interrupts().is_disjoint(Self::DONE_EVENTS) {
379                    #[cfg(any(spi_master_version = "1", spi_master_version = "2"))]
380                    // Need to poll for done-ness even after interrupt fires.
381                    if self.0.busy() {
382                        cx.waker().wake_by_ref();
383                        return Poll::Pending;
384                    }
385
386                    self.0.clear_interrupts(Self::DONE_EVENTS);
387                    return Poll::Ready(());
388                }
389
390                self.0.state.waker.register(cx.waker());
391                self.0.enable_listen(Self::DONE_EVENTS, true);
392                Poll::Pending
393            }
394        }
395        impl Drop for Fut {
396            fn drop(&mut self) {
397                self.0.enable_listen(Self::DONE_EVENTS, false);
398            }
399        }
400
401        if !self.is_done() {
402            Fut(self.driver()).await;
403        }
404
405        if self.dma_driver().state.tx_transfer_in_progress.get() {
406            // In case DMA TX buffer is bigger than what the SPI consumes, stop the DMA.
407            if !self.channel.tx.is_done() {
408                self.channel.tx.stop_transfer();
409            }
410            self.dma_driver().state.tx_transfer_in_progress.set(false);
411        }
412        self.channel.rx.end_transfer();
413        self.channel.tx.end_transfer();
414    }
415
416    /// Fills the given buffer with data from the bus.
417    #[instability::unstable]
418    pub async fn read_async(&mut self, words: &mut [u8]) -> Result<(), Error> {
419        if words.is_empty() {
420            return Ok(());
421        }
422
423        let _clock = SpiClockGuard::new(self.spi.info());
424
425        self.driver().setup_full_duplex()?;
426
427        if self.use_blocking_transfer(words.len()) {
428            self.dma_driver().disable_dma();
429            return self.driver().read(words);
430        }
431
432        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
433        let mut maybe_copy_buffer = match DmaOperationKind::for_read(words) {
434            DmaOperationKind::Copied => {
435                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
436            }
437            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
438                descriptors: &mut descriptors,
439                #[cfg(dma_can_access_psram)]
440                align_buffer: [const { None }; 2],
441            },
442        };
443
444        if maybe_copy_buffer.chunk_size() == 0 {
445            return Err(Error::from(DmaError::BufferTooSmall));
446        }
447
448        for chunk in words.chunks_mut(maybe_copy_buffer.chunk_size()) {
449            let read_bytes = chunk.len();
450            let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *chunk)) };
451            let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };
452
453            self.transfer_buffers_dma_async(read_bytes, 0, rx_buffer, tx_buffer)
454                .await?;
455
456            maybe_copy_buffer.finish(chunk);
457        }
458
459        Ok(())
460    }
461
462    /// Transmits the given buffer to the bus.
463    #[instability::unstable]
464    pub async fn write_async(&mut self, words: &[u8]) -> Result<(), Error> {
465        if words.is_empty() {
466            return Ok(());
467        }
468
469        let _clock = SpiClockGuard::new(self.spi.info());
470
471        self.driver().setup_full_duplex()?;
472
473        if self.use_blocking_transfer(words.len()) {
474            self.dma_driver().disable_dma();
475            return self.driver().write(words);
476        }
477
478        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
479        let mut maybe_copy_buffer = match DmaOperationKind::for_write(words) {
480            DmaOperationKind::Copied => {
481                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
482            }
483            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),
484        };
485
486        if maybe_copy_buffer.chunk_size() == 0 {
487            return Err(Error::from(DmaError::BufferTooSmall));
488        }
489
490        for chunk in words.chunks(maybe_copy_buffer.chunk_size()) {
491            let write_bytes = chunk.len();
492            let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) };
493            let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(chunk)) };
494
495            self.transfer_buffers_dma_async(0, write_bytes, rx_buffer, tx_buffer)
496                .await?;
497        }
498
499        Ok(())
500    }
501
502    /// Transfers by writing out a buffer and reading the response from
503    /// the bus into another buffer.
504    #[instability::unstable]
505    pub async fn transfer_async(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Error> {
506        if read.is_empty() && write.is_empty() {
507            return Ok(());
508        }
509
510        let _clock = SpiClockGuard::new(self.spi.info());
511
512        self.driver().setup_full_duplex()?;
513
514        if self.use_blocking_transfer(read.len().max(write.len())) {
515            self.dma_driver().disable_dma();
516            return if read.is_empty() {
517                self.driver().write(write)
518            } else if write.is_empty() {
519                self.driver().read(read)
520            } else {
521                self.driver().transfer(read, write)
522            };
523        }
524
525        let common_length = min(read.len(), write.len());
526        let (read_common, read_remainder) = read.split_at_mut(common_length);
527        let (write_common, write_remainder) = write.split_at(common_length);
528
529        // DmaOperationKind must be determined on the sub-slices actually passed to DMA.
530        let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
531        let mut maybe_copy_rx_buffer = match DmaOperationKind::for_read(read_common) {
532            DmaOperationKind::Copied => {
533                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
534            }
535            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
536                descriptors: &mut rx_descriptors,
537                #[cfg(dma_can_access_psram)]
538                align_buffer: [const { None }; 2],
539            },
540        };
541
542        let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
543        let mut maybe_copy_tx_buffer = match DmaOperationKind::for_write(write_common) {
544            DmaOperationKind::Copied => {
545                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
546            }
547            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut tx_descriptors),
548        };
549
550        let chunk_size = min(
551            maybe_copy_rx_buffer.chunk_size(),
552            maybe_copy_tx_buffer.chunk_size(),
553        );
554
555        if chunk_size == 0 {
556            return Err(Error::from(DmaError::BufferTooSmall));
557        }
558
559        for (read_chunk, write_chunk) in read_common
560            .chunks_mut(chunk_size)
561            .zip(write_common.chunks(chunk_size))
562        {
563            let read_bytes = read_chunk.len();
564            let write_bytes = write_chunk.len();
565            let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(NonNull::from(write_chunk)) };
566            let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(NonNull::from(&mut *read_chunk)) };
567
568            self.transfer_buffers_dma_async(read_bytes, write_bytes, rx_buffer, tx_buffer)
569                .await?;
570
571            maybe_copy_rx_buffer.finish(read_chunk);
572        }
573
574        if !read_remainder.is_empty() {
575            self.read_async(read_remainder).await
576        } else if !write_remainder.is_empty() {
577            self.write_async(write_remainder).await
578        } else {
579            Ok(())
580        }
581    }
582
583    /// Transfers by writing out a buffer and reading the response from
584    /// the bus into the same buffer.
585    #[instability::unstable]
586    pub async fn transfer_in_place_async(&mut self, words: &mut [u8]) -> Result<(), Error> {
587        if words.is_empty() {
588            return Ok(());
589        }
590
591        let _clock = SpiClockGuard::new(self.spi.info());
592        self.driver().setup_full_duplex()?;
593
594        if self.use_blocking_transfer(words.len()) {
595            self.dma_driver().disable_dma();
596            return self.driver().transfer_in_place(words);
597        }
598
599        let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
600        let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
601        let (mut maybe_copy_rx_buffer, mut maybe_copy_tx_buffer) =
602            match DmaOperationKind::for_write(words) {
603                DmaOperationKind::Copied => (
604                    MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }),
605                    MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }),
606                ),
607                DmaOperationKind::InPlace => (
608                    MaybeCopyRxBuf::Direct {
609                        descriptors: &mut rx_descriptors,
610                        #[cfg(dma_can_access_psram)]
611                        align_buffer: [const { None }; 2],
612                    },
613                    MaybeCopyTxBuf::Direct(&mut tx_descriptors),
614                ),
615            };
616
617        let chunk_size = min(
618            maybe_copy_rx_buffer.chunk_size(),
619            maybe_copy_tx_buffer.chunk_size(),
620        );
621
622        if chunk_size == 0 {
623            return Err(Error::from(DmaError::BufferTooSmall));
624        }
625
626        for chunk in words.chunks_mut(chunk_size) {
627            let bytes = chunk.len();
628            let ptr = NonNull::from(&mut *chunk);
629            let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(ptr) };
630            let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(ptr) };
631
632            self.transfer_buffers_dma_async(bytes, bytes, rx_buffer, tx_buffer)
633                .await?;
634
635            maybe_copy_rx_buffer.finish(chunk);
636        }
637
638        Ok(())
639    }
640
641    /// Half-duplex read.
642    ///
643    /// This performs the command, address, dummy, and data phases as a single
644    /// SPI transaction. Because command and address phases cannot be split
645    /// across multiple DMA transfers, `buffer` must fit in one DMA transfer or
646    /// in the configured internal RX copy buffer.
647    #[instability::unstable]
648    pub async fn half_duplex_read_async(
649        &mut self,
650        data_mode: DataMode,
651        cmd: Command,
652        address: Address,
653        dummy: u8,
654        buffer: &mut [u8],
655    ) -> Result<(), Error> {
656        let _clock = SpiClockGuard::new(self.spi.info());
657
658        if buffer.is_empty() {
659            let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) };
660            self.half_duplex_read_dma_async(data_mode, cmd, address, dummy, 0, rx_buffer)
661                .await?;
662            return Ok(());
663        }
664
665        // Transfers below the configured threshold can skip the DMA setup cost entirely.
666        if self.use_blocking_transfer(buffer.len()) {
667            self.dma_driver().disable_dma();
668            return self
669                .driver()
670                .half_duplex_read(data_mode, cmd, address, dummy, buffer);
671        }
672
673        let operation = DmaOperationKind::for_read(buffer);
674        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
675        let mut maybe_copy_buffer = match operation {
676            DmaOperationKind::Copied => {
677                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
678            }
679            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
680                descriptors: &mut descriptors,
681                #[cfg(dma_can_access_psram)]
682                align_buffer: [const { None }; 2],
683            },
684        };
685
686        let chunk_size = maybe_copy_buffer.chunk_size();
687        if chunk_size == 0 {
688            return Err(Error::from(DmaError::BufferTooSmall));
689        }
690        if buffer.len() > chunk_size {
691            return match operation {
692                DmaOperationKind::Copied => Err(Error::from(DmaError::Overflow)),
693                DmaOperationKind::InPlace => Err(Error::MaxDmaTransferSizeExceeded),
694            };
695        }
696
697        let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *buffer)) };
698        self.half_duplex_read_dma_async(data_mode, cmd, address, dummy, buffer.len(), rx_buffer)
699            .await?;
700        maybe_copy_buffer.finish(buffer);
701
702        Ok(())
703    }
704
705    /// Half-duplex write.
706    ///
707    /// This performs the command, address, dummy, and data phases as a single
708    /// SPI transaction. Because command and address phases cannot be split
709    /// across multiple DMA transfers, `buffer` must fit in one DMA transfer or
710    /// in the configured internal TX copy buffer.
711    #[instability::unstable]
712    pub async fn half_duplex_write_async(
713        &mut self,
714        data_mode: DataMode,
715        cmd: Command,
716        address: Address,
717        dummy: u8,
718        buffer: &[u8],
719    ) -> Result<(), Error> {
720        let _clock = SpiClockGuard::new(self.spi.info());
721
722        if buffer.is_empty() {
723            let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };
724            self.half_duplex_write_dma_async(data_mode, cmd, address, dummy, 0, tx_buffer)
725                .await?;
726            return Ok(());
727        }
728
729        // Transfers below the configured threshold can skip the DMA setup cost entirely.
730        if self.use_blocking_transfer(buffer.len()) {
731            self.dma_driver().disable_dma();
732            return self
733                .driver()
734                .half_duplex_write(data_mode, cmd, address, dummy, buffer);
735        }
736
737        let operation = DmaOperationKind::for_write(buffer);
738        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
739        let mut maybe_copy_buffer = match operation {
740            DmaOperationKind::Copied => {
741                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
742            }
743            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),
744        };
745
746        let chunk_size = maybe_copy_buffer.chunk_size();
747        if chunk_size == 0 {
748            return Err(Error::from(DmaError::BufferTooSmall));
749        }
750        if buffer.len() > chunk_size {
751            return match operation {
752                DmaOperationKind::Copied => Err(Error::from(DmaError::Overflow)),
753                DmaOperationKind::InPlace => Err(Error::MaxDmaTransferSizeExceeded),
754            };
755        }
756
757        let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(buffer)) };
758        self.half_duplex_write_dma_async(data_mode, cmd, address, dummy, buffer.len(), tx_buffer)
759            .await
760    }
761
762    async fn transfer_buffers_dma_async(
763        &mut self,
764        read_bytes: usize,
765        write_bytes: usize,
766        mut rx_buffer: impl DmaRxBuffer,
767        mut tx_buffer: impl DmaTxBuffer,
768    ) -> Result<(), Error> {
769        let _clock = SpiClockGuard::new(self.spi.info());
770
771        let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());
772        unsafe {
773            spi.start_dma_transfer(read_bytes, write_bytes, &mut rx_buffer, &mut tx_buffer)?;
774        }
775        spi.wait_for_idle_async().await;
776        spi.defuse();
777        Ok(())
778    }
779
780    async fn half_duplex_read_dma_async(
781        &mut self,
782        data_mode: DataMode,
783        cmd: Command,
784        address: Address,
785        dummy: u8,
786        bytes_to_read: usize,
787        mut rx_buffer: impl DmaRxBuffer,
788    ) -> Result<(), Error> {
789        let _clock = SpiClockGuard::new(self.spi.info());
790
791        let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());
792        unsafe {
793            spi.start_half_duplex_read(
794                data_mode,
795                cmd,
796                address,
797                dummy,
798                bytes_to_read,
799                &mut rx_buffer,
800            )?;
801        }
802        spi.wait_for_idle_async().await;
803        spi.defuse();
804        Ok(())
805    }
806
807    async fn half_duplex_write_dma_async(
808        &mut self,
809        data_mode: DataMode,
810        cmd: Command,
811        address: Address,
812        dummy: u8,
813        bytes_to_write: usize,
814        mut tx_buffer: impl DmaTxBuffer,
815    ) -> Result<(), Error> {
816        let _clock = SpiClockGuard::new(self.spi.info());
817
818        let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());
819        unsafe {
820            spi.start_half_duplex_write(
821                data_mode,
822                cmd,
823                address,
824                dummy,
825                bytes_to_write,
826                &mut tx_buffer,
827            )?;
828        }
829        spi.wait_for_idle_async().await;
830        spi.defuse();
831        Ok(())
832    }
833}
834
835// +1 to make sure we have enough descriptors to satisfy strict alignment requirements
836const LINK_DESCRIPTOR_COUNT: usize = MAX_DMA_SIZE.div_ceil(CHUNK_SIZE) + 2 + 1;
837
838enum MaybeCopyTxBuf<'a> {
839    Copy(&'a mut ScopedDmaTxBuf<'static>),
840    Direct(&'a mut [DmaDescriptor; LINK_DESCRIPTOR_COUNT]),
841}
842
843impl<'a> MaybeCopyTxBuf<'a> {
844    unsafe fn setup(&mut self, data: NonNull<[u8]>) -> NoBuffer {
845        match self {
846            MaybeCopyTxBuf::Copy(tx_buffer) => {
847                tx_buffer.as_mut_slice()[..data.len()].copy_from_slice(unsafe { data.as_ref() });
848                NoBuffer(tx_buffer.prepare())
849            }
850            MaybeCopyTxBuf::Direct(descriptors) => {
851                let (buffer, _) = unsafe { unwrap!(prepare_for_tx(&mut **descriptors, data, 1)) };
852                buffer
853            }
854        }
855    }
856
857    fn chunk_size(&self) -> usize {
858        match self {
859            MaybeCopyTxBuf::Copy(buffer) => buffer.capacity().min(MAX_DMA_SIZE),
860            MaybeCopyTxBuf::Direct(_) => MAX_DMA_SIZE,
861        }
862    }
863}
864
865#[allow(clippy::large_enum_variant)]
866enum MaybeCopyRxBuf<'a> {
867    Copy(&'a mut ScopedDmaRxBuf<'static>),
868    Direct {
869        descriptors: &'a mut [DmaDescriptor; LINK_DESCRIPTOR_COUNT],
870        #[cfg(dma_can_access_psram)]
871        align_buffer: [Option<ManualWritebackBuffer>; 2],
872    },
873}
874
875impl<'a> MaybeCopyRxBuf<'a> {
876    unsafe fn setup(&mut self, data: NonNull<[u8]>) -> NoBuffer {
877        match self {
878            MaybeCopyRxBuf::Copy(rx_buffer) => NoBuffer(rx_buffer.prepare()),
879            MaybeCopyRxBuf::Direct {
880                descriptors,
881                #[cfg(dma_can_access_psram)]
882                align_buffer,
883            } => {
884                let (buffer, _) = unsafe {
885                    prepare_for_rx(
886                        &mut **descriptors,
887                        #[cfg(dma_can_access_psram)]
888                        align_buffer,
889                        data,
890                    )
891                };
892                buffer
893            }
894        }
895    }
896
897    fn chunk_size(&self) -> usize {
898        match self {
899            MaybeCopyRxBuf::Copy(buffer) => buffer.capacity().min(MAX_DMA_SIZE),
900            MaybeCopyRxBuf::Direct { .. } => MAX_DMA_SIZE,
901        }
902    }
903
904    fn finish(&mut self, chunk: &mut [u8]) {
905        match self {
906            MaybeCopyRxBuf::Copy(buffer) => {
907                chunk.copy_from_slice(&buffer.as_slice()[..chunk.len()]);
908            }
909            MaybeCopyRxBuf::Direct {
910                #[cfg(dma_can_access_psram)]
911                align_buffer,
912                ..
913            } => {
914                #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
915                crate::dma::invalidate_received(NonNull::from(&*chunk));
916
917                #[cfg(dma_can_access_psram)]
918                for buffer in align_buffer.iter_mut() {
919                    if let Some(buffer) = buffer.as_mut() {
920                        buffer.write_back();
921                    }
922                    *buffer = None;
923                }
924            }
925        }
926    }
927}
928
929#[derive(Clone, Copy)]
930enum DmaOperationKind {
931    /// The entire slice must be copied into the internal buffer first.
932    Copied,
933
934    /// The slice can be transferred directly, with minimal copying done for alignment.
935    InPlace,
936}
937
938impl DmaOperationKind {
939    fn compute(buffer: &[u8], direction: TransferDirection) -> Self {
940        fn is_dma_compatible(buffer: &[u8], _direction: TransferDirection) -> bool {
941            // FIXME: lazy workaround for ESP32 TX DMA alignment requirements.
942            // `prepare_for_tx` and `prepare_for_rx` should be updated to handle ESP32.
943            #[cfg(spi_master_version = "1")]
944            if !((buffer.as_ptr() as usize).is_multiple_of(4) && buffer.len().is_multiple_of(4)) {
945                return false;
946            }
947
948            if is_slice_in_dram(buffer) {
949                return true;
950            }
951            #[cfg(dma_can_access_psram)]
952            if is_slice_in_psram(buffer) {
953                #[cfg(spi_master_version = "2")]
954                if _direction == TransferDirection::In {
955                    // The S2's DMA does not write an internal descriptor that follows a PSRAM
956                    // descriptor. A PSRAM buffer that does not end on a cache line would need an
957                    // internal tail bounce buffer, so the DMA receives into the driver's internal
958                    // buffer instead, and the CPU copies the data to the PSRAM buffer.
959                    let tail_bytes = (buffer.as_ptr() as usize + buffer.len()).wrapping_neg()
960                        & (crate::dma::aligned::PSRAM_CACHE_LINE - 1);
961                    if tail_bytes > 0 {
962                        return false;
963                    }
964                }
965
966                return true;
967            }
968
969            // TODO: C5+ DMA can read from flash
970
971            false
972        }
973
974        if is_dma_compatible(buffer, direction) {
975            Self::InPlace
976        } else {
977            Self::Copied
978        }
979    }
980
981    fn for_read(buffer: &mut [u8]) -> Self {
982        Self::compute(buffer, TransferDirection::In)
983    }
984
985    fn for_write(buffer: &[u8]) -> Self {
986        Self::compute(buffer, TransferDirection::Out)
987    }
988}
989
990impl<'d, Dm> SpiDma<'d, Dm>
991where
992    Dm: DriverMode,
993{
994    fn use_blocking_transfer(&self, transfer_size: usize) -> bool {
995        let threshold = self
996            .spi
997            .state()
998            .min_async_transfer_size
999            .load(Ordering::Relaxed);
1000        threshold > 0 && transfer_size < threshold
1001    }
1002
1003    fn spi(&self) -> &SpiWrapper<'_> {
1004        &self.spi
1005    }
1006
1007    fn driver(&self) -> Driver {
1008        Driver {
1009            info: self.spi.info(),
1010            state: self.spi.state(),
1011        }
1012    }
1013
1014    fn dma_driver(&self) -> DmaDriver {
1015        DmaDriver {
1016            driver: self.driver(),
1017            state: self.spi().dma_state(),
1018            dma_peripheral: self.spi.spi.dma_peripheral(),
1019        }
1020    }
1021
1022    fn is_done(&self) -> bool {
1023        if self.driver().busy() {
1024            return false;
1025        }
1026        if self.dma_driver().state.rx_transfer_in_progress.get() {
1027            // If this is an asymmetric transfer and the RX side is smaller, the RX channel
1028            // will never be "done" as it won't have enough descriptors/buffer to receive
1029            // the EOF bit from the SPI. So instead the RX channel will hit
1030            // a "descriptor empty" which means the DMA is written as much
1031            // of the received data as possible into the buffer and
1032            // discarded the rest. The user doesn't care about this discarded data.
1033
1034            if !self.channel.rx.is_done() && !self.channel.rx.has_dscr_empty_error() {
1035                return false;
1036            }
1037        }
1038        true
1039    }
1040
1041    fn wait_for_idle(&mut self) {
1042        while !self.is_done() {
1043            // Wait for the SPI to become idle
1044        }
1045        self.dma_driver().state.rx_transfer_in_progress.set(false);
1046        self.dma_driver().state.tx_transfer_in_progress.set(false);
1047        self.channel.rx.end_transfer();
1048        self.channel.tx.end_transfer();
1049        fence(Ordering::Acquire);
1050    }
1051
1052    /// # Safety
1053    ///
1054    /// The caller must ensure to not access the buffer contents while the
1055    /// transfer is in progress. Moving the buffer itself is allowed.
1056    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1057    unsafe fn start_transfer_dma<RX: DmaRxBuffer, TX: DmaTxBuffer>(
1058        &mut self,
1059        full_duplex: bool,
1060        bytes_to_read: usize,
1061        bytes_to_write: usize,
1062        rx_buffer: &mut RX,
1063        tx_buffer: &mut TX,
1064    ) -> Result<(), Error> {
1065        if bytes_to_read > MAX_DMA_SIZE || bytes_to_write > MAX_DMA_SIZE {
1066            return Err(Error::MaxDmaTransferSizeExceeded);
1067        }
1068
1069        self.dma_driver()
1070            .state
1071            .rx_transfer_in_progress
1072            .set(bytes_to_read > 0);
1073        self.dma_driver()
1074            .state
1075            .tx_transfer_in_progress
1076            .set(bytes_to_write > 0);
1077        unsafe {
1078            self.dma_driver().start_transfer_dma(
1079                full_duplex,
1080                bytes_to_read,
1081                bytes_to_write,
1082                rx_buffer,
1083                tx_buffer,
1084                &mut self.channel,
1085            )
1086        }
1087    }
1088
1089    /// # Safety
1090    ///
1091    /// The caller must ensure that the buffers are not accessed while the
1092    /// transfer is in progress. Moving the buffers is allowed.
1093    #[cfg(all(spi_master_version = "1", spi_address_workaround))]
1094    unsafe fn set_up_address_workaround(
1095        &mut self,
1096        cmd: Command,
1097        address: Address,
1098        dummy: u8,
1099    ) -> Result<(), Error> {
1100        if dummy > 0 {
1101            // FIXME: https://github.com/esp-rs/esp-hal/issues/2240
1102            error!("Dummy bits are not supported when there is no data to write");
1103            return Err(Error::Unsupported);
1104        }
1105
1106        let buffer = unsafe { self.dma_driver().tx_buffer() };
1107
1108        let bytes_to_write = address.width().div_ceil(8);
1109        // The address register is read in big-endian order,
1110        // we have to prepare the emulated write in the same way.
1111        let addr_bytes = address.value().to_be_bytes();
1112        let addr_bytes = &addr_bytes[4 - bytes_to_write..][..bytes_to_write];
1113        buffer.fill(addr_bytes);
1114
1115        self.driver().setup_half_duplex(
1116            true,
1117            cmd,
1118            Address::None,
1119            false,
1120            dummy,
1121            bytes_to_write == 0,
1122            address.mode(),
1123        )?;
1124
1125        let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1126
1127        unsafe { self.start_transfer_dma(false, 0, bytes_to_write, rx_buffer, buffer) }
1128    }
1129
1130    fn cancel_transfer(&mut self) {
1131        let state = self.dma_driver().state;
1132        if state.tx_transfer_in_progress.get() || state.rx_transfer_in_progress.get() {
1133            self.dma_driver().abort_transfer();
1134
1135            // We need to stop the DMA transfer, too.
1136            if state.tx_transfer_in_progress.get() {
1137                self.channel.tx.stop_transfer();
1138                state.tx_transfer_in_progress.set(false);
1139            }
1140            if state.rx_transfer_in_progress.get() {
1141                self.channel.rx.stop_transfer();
1142                state.rx_transfer_in_progress.set(false);
1143            }
1144        }
1145    }
1146
1147    /// # Safety
1148    ///
1149    /// The caller must ensure that the buffers are not accessed while the
1150    /// transfer is in progress. Moving the buffers is allowed.
1151    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1152    unsafe fn start_dma_write(
1153        &mut self,
1154        bytes_to_write: usize,
1155        buffer: &mut impl DmaTxBuffer,
1156    ) -> Result<(), Error> {
1157        let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1158
1159        unsafe { self.start_dma_transfer(0, bytes_to_write, rx_buffer, buffer) }
1160    }
1161
1162    /// Assigns copy buffers to the SPI driver.
1163    ///
1164    /// These buffers will be used to copy data when using the slice-based transfer functions.
1165    ///
1166    /// Data is copied in two cases:
1167    ///   - When the buffer is not located in a memory region that can be accessed by the DMA.
1168    #[cfg_attr(
1169        not(spi_master_dma_can_access_flash),
1170        doc = "The DMA cannot read flash memory."
1171    )]
1172    ///   - When the alignment of the buffer does not meet the DMA's requirements, the unaligned
1173    ///     parts of the buffer are copied.
1174    #[cfg_attr(
1175        spi_master_version = "1",
1176        doc = "On ESP32, transferring from internal SRAM requires copying the entire buffer if it is
1177not 4-byte aligned. This is a limitation of the current implementation."
1178    )]
1179    #[cfg_attr(
1180        spi_master_version = "2",
1181        doc = "On ESP32-S2, receiving into PSRAM requires the buffer's _end_ to be 16-byte
1182aligned, otherwise the driver requires copying the entire buffer."
1183    )]
1184    #[doc = ""]
1185    /// The maximum useful size for these buffers is 32736 bytes, any additional memory will
1186    /// be wasted.
1187    ///
1188    /// For an example of how to create these buffers, see the [`SpiDma`] documentation.
1189    #[instability::unstable]
1190    pub fn with_buffers(self, dma_rx_buf: DmaRxBuf, dma_tx_buf: DmaTxBuf) -> SpiDma<'d, Dm> {
1191        unsafe {
1192            (&mut *self.spi.dma_state().rx_buffer.get()).write(dma_rx_buf.into_scoped());
1193            (&mut *self.spi.dma_state().tx_buffer.get()).write(dma_tx_buf.into_scoped());
1194        }
1195        self
1196    }
1197
1198    /// Performs a DMA write.
1199    ///
1200    /// Returns a [`SpiDmaTransfer`] that owns the buffer and the
1201    /// SPI instance. The maximum amount of data to be sent is 32736
1202    /// bytes.
1203    #[allow(clippy::type_complexity)]
1204    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1205    #[instability::unstable]
1206    pub fn write_buffer<TX: DmaTxBuffer>(
1207        mut self,
1208        bytes_to_write: usize,
1209        mut buffer: TX,
1210    ) -> Result<SpiDmaTransfer<'d, Dm, TX>, (Error, Self, TX)> {
1211        let clock = SpiClockGuard::new(self.spi.info());
1212
1213        if let Err(e) = self.driver().setup_full_duplex() {
1214            return Err((e, self, buffer));
1215        };
1216        match unsafe { self.start_dma_write(bytes_to_write, &mut buffer) } {
1217            Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1218            Err(e) => Err((e, self, buffer)),
1219        }
1220    }
1221
1222    /// # Safety
1223    ///
1224    /// The caller must ensure that the buffers are not accessed while the
1225    /// transfer is in progress. Moving the buffers is allowed.
1226    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1227    unsafe fn start_dma_read(
1228        &mut self,
1229        bytes_to_read: usize,
1230        buffer: &mut impl DmaRxBuffer,
1231    ) -> Result<(), Error> {
1232        let tx_buffer = unsafe { self.dma_driver().tx_buffer() };
1233
1234        unsafe { self.start_dma_transfer(bytes_to_read, 0, buffer, tx_buffer) }
1235    }
1236
1237    /// Performs a DMA read.
1238    ///
1239    /// Returns a [`SpiDmaTransfer`] that owns the buffer and
1240    /// the SPI instance. The maximum amount of data to be
1241    /// received is 32736 bytes.
1242    #[allow(clippy::type_complexity)]
1243    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1244    #[instability::unstable]
1245    pub fn read_buffer<RX: DmaRxBuffer>(
1246        mut self,
1247        bytes_to_read: usize,
1248        mut buffer: RX,
1249    ) -> Result<SpiDmaTransfer<'d, Dm, RX>, (Error, Self, RX)> {
1250        let clock = SpiClockGuard::new(self.spi.info());
1251
1252        if let Err(e) = self.driver().setup_full_duplex() {
1253            return Err((e, self, buffer));
1254        };
1255        match unsafe { self.start_dma_read(bytes_to_read, &mut buffer) } {
1256            Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1257            Err(e) => Err((e, self, buffer)),
1258        }
1259    }
1260
1261    /// # Safety
1262    ///
1263    /// The caller must ensure that the buffers are not accessed while the
1264    /// transfer is in progress. Moving the buffers is allowed.
1265    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1266    unsafe fn start_dma_transfer(
1267        &mut self,
1268        bytes_to_read: usize,
1269        bytes_to_write: usize,
1270        rx_buffer: &mut impl DmaRxBuffer,
1271        tx_buffer: &mut impl DmaTxBuffer,
1272    ) -> Result<(), Error> {
1273        unsafe {
1274            self.start_transfer_dma(true, bytes_to_read, bytes_to_write, rx_buffer, tx_buffer)
1275        }
1276    }
1277
1278    /// Performs a DMA transfer.
1279    ///
1280    /// Returns a [`SpiDmaTransfer`] that owns the buffers and
1281    /// the SPI instance. The maximum amount of data to be
1282    /// sent/received is 32736 bytes.
1283    #[allow(clippy::type_complexity)]
1284    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1285    #[instability::unstable]
1286    pub fn transfer_buffers<RX: DmaRxBuffer, TX: DmaTxBuffer>(
1287        mut self,
1288        bytes_to_read: usize,
1289        mut rx_buffer: RX,
1290        bytes_to_write: usize,
1291        mut tx_buffer: TX,
1292    ) -> Result<SpiDmaTransfer<'d, Dm, (RX, TX)>, (Error, Self, RX, TX)> {
1293        let clock = SpiClockGuard::new(self.spi.info());
1294
1295        if let Err(e) = self.driver().setup_full_duplex() {
1296            return Err((e, self, rx_buffer, tx_buffer));
1297        };
1298        match unsafe {
1299            self.start_dma_transfer(
1300                bytes_to_read,
1301                bytes_to_write,
1302                &mut rx_buffer,
1303                &mut tx_buffer,
1304            )
1305        } {
1306            Ok(_) => Ok(SpiDmaTransfer::new(self, (rx_buffer, tx_buffer), clock)),
1307            Err(e) => Err((e, self, rx_buffer, tx_buffer)),
1308        }
1309    }
1310
1311    /// # Safety
1312    ///
1313    /// The caller must ensure that the buffers are not accessed while the
1314    /// transfer is in progress. Moving the buffers is allowed.
1315    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1316    unsafe fn start_half_duplex_read(
1317        &mut self,
1318        data_mode: DataMode,
1319        cmd: Command,
1320        address: Address,
1321        dummy: u8,
1322        bytes_to_read: usize,
1323        buffer: &mut impl DmaRxBuffer,
1324    ) -> Result<(), Error> {
1325        self.driver().setup_half_duplex(
1326            false,
1327            cmd,
1328            address,
1329            false,
1330            dummy,
1331            bytes_to_read == 0,
1332            data_mode,
1333        )?;
1334
1335        let tx_buffer = unsafe { self.dma_driver().tx_buffer() };
1336
1337        unsafe { self.start_transfer_dma(false, bytes_to_read, 0, buffer, tx_buffer) }
1338    }
1339
1340    /// Performs a half-duplex read operation using DMA.
1341    #[allow(clippy::type_complexity)]
1342    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1343    #[instability::unstable]
1344    pub fn half_duplex_read_buffer<RX: DmaRxBuffer>(
1345        mut self,
1346        data_mode: DataMode,
1347        cmd: Command,
1348        address: Address,
1349        dummy: u8,
1350        bytes_to_read: usize,
1351        mut buffer: RX,
1352    ) -> Result<SpiDmaTransfer<'d, Dm, RX>, (Error, Self, RX)> {
1353        let clock = SpiClockGuard::new(self.spi.info());
1354
1355        match unsafe {
1356            self.start_half_duplex_read(data_mode, cmd, address, dummy, bytes_to_read, &mut buffer)
1357        } {
1358            Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1359            Err(e) => Err((e, self, buffer)),
1360        }
1361    }
1362
1363    /// # Safety
1364    ///
1365    /// The caller must ensure that the buffers are not accessed while the
1366    /// transfer is in progress. Moving the buffers is allowed.
1367    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1368    unsafe fn start_half_duplex_write(
1369        &mut self,
1370        data_mode: DataMode,
1371        cmd: Command,
1372        address: Address,
1373        dummy: u8,
1374        bytes_to_write: usize,
1375        buffer: &mut impl DmaTxBuffer,
1376    ) -> Result<(), Error> {
1377        #[cfg(all(spi_master_version = "1", spi_address_workaround))]
1378        {
1379            // On the ESP32, if we don't have data, the address is always sent
1380            // on a single line, regardless of its data mode.
1381            if bytes_to_write == 0 && address.mode() != DataMode::SingleTwoDataLines {
1382                return unsafe { self.set_up_address_workaround(cmd, address, dummy) };
1383            }
1384        }
1385
1386        self.driver().setup_half_duplex(
1387            true,
1388            cmd,
1389            address,
1390            false,
1391            dummy,
1392            bytes_to_write == 0,
1393            data_mode,
1394        )?;
1395
1396        let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1397
1398        unsafe { self.start_transfer_dma(false, 0, bytes_to_write, rx_buffer, buffer) }
1399    }
1400
1401    /// Performs a half-duplex write operation using DMA.
1402    #[allow(clippy::type_complexity)]
1403    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1404    #[instability::unstable]
1405    pub fn half_duplex_write_buffer<TX: DmaTxBuffer>(
1406        mut self,
1407        data_mode: DataMode,
1408        cmd: Command,
1409        address: Address,
1410        dummy: u8,
1411        bytes_to_write: usize,
1412        mut buffer: TX,
1413    ) -> Result<SpiDmaTransfer<'d, Dm, TX>, (Error, Self, TX)> {
1414        let clock = SpiClockGuard::new(self.spi.info());
1415
1416        match unsafe {
1417            self.start_half_duplex_write(
1418                data_mode,
1419                cmd,
1420                address,
1421                dummy,
1422                bytes_to_write,
1423                &mut buffer,
1424            )
1425        } {
1426            Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1427            Err(e) => Err((e, self, buffer)),
1428        }
1429    }
1430
1431    #[doc_replace(
1432        "max_frequency" => {
1433            cfg(esp32h2) => "48MHz",
1434            _ => "80MHz",
1435        }
1436    )]
1437    /// Changes the bus configuration.
1438    ///
1439    /// # Errors
1440    ///
1441    /// [`ConfigError::FrequencyOutOfRange`] when frequency passed in config exceeds
1442    /// __max_frequency__ or is below 70 kHz.
1443    #[instability::unstable]
1444    pub fn apply_config(&mut self, config: &Config) -> Result<(), ConfigError> {
1445        self.driver().apply_config(config)
1446    }
1447
1448    fn transfer_buffers_dma(
1449        &mut self,
1450        read_bytes: usize,
1451        write_bytes: usize,
1452        mut rx_buffer: impl DmaRxBuffer,
1453        mut tx_buffer: impl DmaTxBuffer,
1454    ) -> Result<(), Error> {
1455        unsafe {
1456            self.start_dma_transfer(read_bytes, write_bytes, &mut rx_buffer, &mut tx_buffer)?;
1457        }
1458        self.wait_for_idle();
1459        Ok(())
1460    }
1461
1462    /// Reads data from the SPI bus using DMA.
1463    #[instability::unstable]
1464    pub fn read(&mut self, words: &mut [u8]) -> Result<(), Error> {
1465        let _clock = SpiClockGuard::new(self.spi.info());
1466
1467        self.driver().setup_full_duplex()?;
1468
1469        if self.use_blocking_transfer(words.len()) {
1470            self.dma_driver().disable_dma();
1471            return self.driver().read(words);
1472        }
1473
1474        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1475        let mut maybe_copy_buffer = match DmaOperationKind::for_read(words) {
1476            DmaOperationKind::Copied => {
1477                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
1478            }
1479            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
1480                descriptors: &mut descriptors,
1481                #[cfg(dma_can_access_psram)]
1482                align_buffer: [const { None }; 2],
1483            },
1484        };
1485
1486        if maybe_copy_buffer.chunk_size() == 0 {
1487            return Err(Error::from(DmaError::BufferTooSmall));
1488        }
1489
1490        for chunk in words.chunks_mut(maybe_copy_buffer.chunk_size()) {
1491            let read_bytes = chunk.len();
1492            let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *chunk)) };
1493            let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };
1494
1495            self.transfer_buffers_dma(read_bytes, 0, rx_buffer, tx_buffer)?;
1496
1497            maybe_copy_buffer.finish(chunk);
1498        }
1499
1500        Ok(())
1501    }
1502
1503    /// Writes data to the SPI bus using DMA.
1504    #[instability::unstable]
1505    pub fn write(&mut self, words: &[u8]) -> Result<(), Error> {
1506        let _clock = SpiClockGuard::new(self.spi.info());
1507
1508        self.driver().setup_full_duplex()?;
1509
1510        if self.use_blocking_transfer(words.len()) {
1511            self.dma_driver().disable_dma();
1512            return self.driver().write(words);
1513        }
1514
1515        let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1516        let mut maybe_copy_buffer = match DmaOperationKind::for_write(words) {
1517            DmaOperationKind::Copied => {
1518                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
1519            }
1520            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),
1521        };
1522
1523        if maybe_copy_buffer.chunk_size() == 0 {
1524            return Err(Error::from(DmaError::BufferTooSmall));
1525        }
1526
1527        for chunk in words.chunks(maybe_copy_buffer.chunk_size()) {
1528            let write_bytes = chunk.len();
1529            let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) };
1530            let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(chunk)) };
1531
1532            self.transfer_buffers_dma(0, write_bytes, rx_buffer, tx_buffer)?;
1533        }
1534
1535        Ok(())
1536    }
1537
1538    /// Transfers data to and from the SPI bus simultaneously using DMA.
1539    #[instability::unstable]
1540    pub fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Error> {
1541        let _clock = SpiClockGuard::new(self.spi.info());
1542
1543        self.driver().setup_full_duplex()?;
1544
1545        if self.use_blocking_transfer(read.len().max(write.len())) {
1546            self.dma_driver().disable_dma();
1547            if read.is_empty() {
1548                return self.driver().write(write);
1549            } else if write.is_empty() {
1550                return self.driver().read(read);
1551            } else {
1552                return self.driver().transfer(read, write);
1553            }
1554        }
1555
1556        let common_length = min(read.len(), write.len());
1557        let (read_common, read_remainder) = read.split_at_mut(common_length);
1558        let (write_common, write_remainder) = write.split_at(common_length);
1559
1560        // DmaOperationKind must be determined on the sub-slices actually passed to DMA.
1561        let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1562        let mut maybe_copy_rx_buffer = match DmaOperationKind::for_read(read_common) {
1563            DmaOperationKind::Copied => {
1564                MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
1565            }
1566            DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
1567                descriptors: &mut rx_descriptors,
1568                #[cfg(dma_can_access_psram)]
1569                align_buffer: [const { None }; 2],
1570            },
1571        };
1572
1573        let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1574        let mut maybe_copy_tx_buffer = match DmaOperationKind::for_write(write_common) {
1575            DmaOperationKind::Copied => {
1576                MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
1577            }
1578            DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut tx_descriptors),
1579        };
1580
1581        let chunk_size = min(
1582            maybe_copy_rx_buffer.chunk_size(),
1583            maybe_copy_tx_buffer.chunk_size(),
1584        );
1585
1586        if chunk_size == 0 {
1587            return Err(Error::from(DmaError::BufferTooSmall));
1588        }
1589
1590        for (read_chunk, write_chunk) in read_common
1591            .chunks_mut(chunk_size)
1592            .zip(write_common.chunks(chunk_size))
1593        {
1594            let read_bytes = read_chunk.len();
1595            let write_bytes = write_chunk.len();
1596            let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(NonNull::from(write_chunk)) };
1597            let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(NonNull::from(&mut *read_chunk)) };
1598
1599            self.transfer_buffers_dma(read_bytes, write_bytes, rx_buffer, tx_buffer)?;
1600
1601            maybe_copy_rx_buffer.finish(read_chunk);
1602        }
1603
1604        if !read_remainder.is_empty() {
1605            self.read(read_remainder)
1606        } else if !write_remainder.is_empty() {
1607            self.write(write_remainder)
1608        } else {
1609            Ok(())
1610        }
1611    }
1612
1613    /// Transfers data in place on the SPI bus using DMA.
1614    #[instability::unstable]
1615    pub fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Error> {
1616        let _clock = SpiClockGuard::new(self.spi.info());
1617
1618        self.driver().setup_full_duplex()?;
1619
1620        if self.use_blocking_transfer(words.len()) {
1621            self.dma_driver().disable_dma();
1622            return self.driver().transfer_in_place(words);
1623        }
1624
1625        let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1626        let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1627        let (mut maybe_copy_rx_buffer, mut maybe_copy_tx_buffer) =
1628            match DmaOperationKind::for_write(words) {
1629                DmaOperationKind::Copied => (
1630                    MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }),
1631                    MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }),
1632                ),
1633                DmaOperationKind::InPlace => (
1634                    MaybeCopyRxBuf::Direct {
1635                        descriptors: &mut rx_descriptors,
1636                        #[cfg(dma_can_access_psram)]
1637                        align_buffer: [const { None }; 2],
1638                    },
1639                    MaybeCopyTxBuf::Direct(&mut tx_descriptors),
1640                ),
1641            };
1642
1643        let chunk_size = min(
1644            maybe_copy_rx_buffer.chunk_size(),
1645            maybe_copy_tx_buffer.chunk_size(),
1646        );
1647
1648        if chunk_size == 0 {
1649            return Err(Error::from(DmaError::BufferTooSmall));
1650        }
1651
1652        for chunk in words.chunks_mut(chunk_size) {
1653            let bytes = chunk.len();
1654            let ptr = NonNull::from(&mut *chunk);
1655            let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(ptr) };
1656            let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(ptr) };
1657
1658            self.transfer_buffers_dma(bytes, bytes, rx_buffer, tx_buffer)?;
1659
1660            maybe_copy_rx_buffer.finish(chunk);
1661        }
1662
1663        Ok(())
1664    }
1665
1666    /// Half-duplex read.
1667    #[instability::unstable]
1668    pub fn half_duplex_read(
1669        &mut self,
1670        data_mode: DataMode,
1671        cmd: Command,
1672        address: Address,
1673        dummy: u8,
1674        buffer: &mut [u8],
1675    ) -> Result<(), Error> {
1676        let _clock = SpiClockGuard::new(self.spi.info());
1677
1678        let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1679        if rx_buffer.capacity() == 0 {
1680            return Err(Error::from(DmaError::BufferTooSmall));
1681        }
1682        if buffer.len() > rx_buffer.capacity() {
1683            return Err(Error::from(DmaError::Overflow));
1684        }
1685
1686        unsafe {
1687            self.start_half_duplex_read(data_mode, cmd, address, dummy, buffer.len(), rx_buffer)?;
1688        }
1689
1690        self.wait_for_idle();
1691
1692        buffer.copy_from_slice(&rx_buffer.as_slice()[..buffer.len()]);
1693
1694        Ok(())
1695    }
1696
1697    /// Half-duplex write.
1698    #[instability::unstable]
1699    pub fn half_duplex_write(
1700        &mut self,
1701        data_mode: DataMode,
1702        cmd: Command,
1703        address: Address,
1704        dummy: u8,
1705        buffer: &[u8],
1706    ) -> Result<(), Error> {
1707        let _clock = SpiClockGuard::new(self.spi.info());
1708
1709        let tx_buffer = unsafe { self.dma_driver().tx_buffer() };
1710        if tx_buffer.capacity() == 0 {
1711            return Err(Error::from(DmaError::BufferTooSmall));
1712        }
1713        if buffer.len() > tx_buffer.capacity() {
1714            return Err(Error::from(DmaError::Overflow));
1715        }
1716
1717        tx_buffer.as_mut_slice()[..buffer.len()].copy_from_slice(buffer);
1718
1719        unsafe {
1720            self.start_half_duplex_write(data_mode, cmd, address, dummy, buffer.len(), tx_buffer)?;
1721        }
1722
1723        self.wait_for_idle();
1724
1725        Ok(())
1726    }
1727}
1728
1729/// A structure representing a DMA transfer for SPI.
1730///
1731/// Holds references to the SPI instance, DMA buffers, and transfer status.
1732#[instability::unstable]
1733pub struct SpiDmaTransfer<'d, Dm, Buf>
1734where
1735    Dm: DriverMode,
1736{
1737    spi_dma: ManuallyDrop<SpiDma<'d, Dm>>,
1738    dma_buf: ManuallyDrop<Buf>,
1739    clock: ManuallyDrop<SpiClockGuard>,
1740}
1741
1742impl<Buf> SpiDmaTransfer<'_, Async, Buf> {
1743    /// Waits for the DMA transfer to complete asynchronously.
1744    ///
1745    /// Awaits the completion of both RX and TX operations.
1746    #[instability::unstable]
1747    pub async fn wait_for_done(&mut self) {
1748        self.spi_dma.wait_for_idle_async().await;
1749    }
1750}
1751
1752impl<'d, Dm, Buf> SpiDmaTransfer<'d, Dm, Buf>
1753where
1754    Dm: DriverMode,
1755{
1756    fn new(spi_dma: SpiDma<'d, Dm>, dma_buf: Buf, clock: SpiClockGuard) -> Self {
1757        Self {
1758            spi_dma: ManuallyDrop::new(spi_dma),
1759            dma_buf: ManuallyDrop::new(dma_buf),
1760            clock: ManuallyDrop::new(clock),
1761        }
1762    }
1763
1764    /// Returns whether the transfer is complete.
1765    ///
1766    /// Both RX and TX operations are done, and the SPI instance is no longer
1767    /// busy.
1768    #[instability::unstable]
1769    pub fn is_done(&self) -> bool {
1770        self.spi_dma.is_done()
1771    }
1772
1773    /// Waits for the DMA transfer to complete.
1774    ///
1775    /// Blocks until the transfer is finished and returns the
1776    /// `SpiDma` instance and the associated buffer.
1777    #[instability::unstable]
1778    pub fn wait(mut self) -> (SpiDma<'d, Dm>, Buf) {
1779        self.spi_dma.wait_for_idle();
1780        let retval = unsafe {
1781            (
1782                ManuallyDrop::take(&mut self.spi_dma),
1783                ManuallyDrop::take(&mut self.dma_buf),
1784            )
1785        };
1786        let _ = unsafe { ManuallyDrop::take(&mut self.clock) };
1787        core::mem::forget(self);
1788        retval
1789    }
1790
1791    /// Cancels the DMA transfer.
1792    #[instability::unstable]
1793    pub fn cancel(&mut self) {
1794        if !self.spi_dma.is_done() {
1795            self.spi_dma.cancel_transfer();
1796        }
1797    }
1798}
1799
1800impl<Dm, Buf> Drop for SpiDmaTransfer<'_, Dm, Buf>
1801where
1802    Dm: DriverMode,
1803{
1804    fn drop(&mut self) {
1805        if !self.is_done() {
1806            self.spi_dma.cancel_transfer();
1807            self.spi_dma.wait_for_idle();
1808        }
1809
1810        unsafe {
1811            ManuallyDrop::drop(&mut self.spi_dma);
1812            ManuallyDrop::drop(&mut self.dma_buf);
1813        }
1814        let _ = unsafe { ManuallyDrop::take(&mut self.clock) };
1815    }
1816}
1817
1818pub(super) struct DmaDriver {
1819    driver: Driver,
1820    dma_peripheral: crate::dma::DmaPeripheral,
1821    state: &'static DmaState,
1822}
1823
1824impl DmaDriver {
1825    unsafe fn rx_buffer(&self) -> &'static mut ScopedDmaRxBuf<'static> {
1826        unsafe { self.state.rx_buffer() }
1827    }
1828
1829    unsafe fn tx_buffer(&self) -> &'static mut ScopedDmaTxBuf<'static> {
1830        unsafe { self.state.tx_buffer() }
1831    }
1832
1833    fn abort_transfer(&self) {
1834        // The SPI peripheral is controlling how much data we transfer, so let's
1835        // update its counter.
1836        // 0 doesn't take effect on ESP32 and cuts the currently transmitted byte
1837        // immediately.
1838        // 1 seems to stop after transmitting the current byte which is somewhat less
1839        // impolite.
1840        self.driver.configure_datalen(1, 1);
1841        self.driver.update();
1842    }
1843
1844    fn disable_dma(&self) {
1845        #[cfg(not(any(spi_master_version = "1", spi_master_version = "2")))]
1846        self.regs().dma_conf().modify(|_, w| {
1847            w.dma_tx_ena().clear_bit();
1848            w.dma_rx_ena().clear_bit()
1849        });
1850
1851        // PDMA has no enable bit: whether the peripheral takes its data from the
1852        // descriptors or from the FIFO is decided by whether a link is armed, so
1853        // a stale link keeps routing a CPU-driven transfer into DMA. This mirrors
1854        // the clearing `start_transfer_dma` already does on the way *in*.
1855        #[cfg(any(spi_master_version = "1", spi_master_version = "2"))]
1856        {
1857            self.regs().dma_out_link().write(|w| unsafe { w.bits(0) });
1858            self.regs().dma_in_link().write(|w| unsafe { w.bits(0) });
1859        }
1860        #[cfg(spi_master_version = "1")]
1861        crate::dma::connect_spi_dma(self.dma_peripheral, false);
1862    }
1863
1864    fn regs(&self) -> &RegisterBlock {
1865        self.driver.regs()
1866    }
1867
1868    #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1869    unsafe fn start_transfer_dma<Dm: DriverMode>(
1870        &self,
1871        _full_duplex: bool,
1872        rx_len: usize,
1873        tx_len: usize,
1874        rx_buffer: &mut impl DmaRxBuffer,
1875        tx_buffer: &mut impl DmaTxBuffer,
1876        channel: &mut Channel<Dm, SpiMasterErased<'_>>,
1877    ) -> Result<(), Error> {
1878        #[cfg(spi_master_version = "2")]
1879        {
1880            // without this a transfer after a write will fail
1881            self.regs().dma_out_link().write(|w| unsafe { w.bits(0) });
1882            self.regs().dma_in_link().write(|w| unsafe { w.bits(0) });
1883        }
1884
1885        self.driver.configure_datalen(rx_len, tx_len);
1886
1887        // enable the MISO and MOSI if needed
1888        self.regs()
1889            .user()
1890            .modify(|_, w| w.usr_miso().bit(rx_len > 0).usr_mosi().bit(tx_len > 0));
1891
1892        self.enable_dma();
1893
1894        if rx_len > 0 {
1895            unsafe {
1896                channel
1897                    .rx
1898                    .prepare_transfer(self.dma_peripheral, rx_buffer)
1899                    .and_then(|_| channel.rx.start_transfer())?;
1900            }
1901        } else {
1902            #[cfg(spi_master_version = "1")]
1903            {
1904                // see https://github.com/espressif/esp-idf/commit/366e4397e9dae9d93fe69ea9d389b5743295886f
1905                // see https://github.com/espressif/esp-idf/commit/0c3653b1fd7151001143451d4aa95dbf15ee8506
1906                if _full_duplex {
1907                    self.regs()
1908                        .dma_in_link()
1909                        .modify(|_, w| unsafe { w.inlink_addr().bits(0) });
1910                    self.regs()
1911                        .dma_in_link()
1912                        .modify(|_, w| w.inlink_start().set_bit());
1913                }
1914            }
1915        }
1916        if tx_len > 0 {
1917            unsafe {
1918                channel
1919                    .tx
1920                    .prepare_transfer(self.dma_peripheral, tx_buffer)
1921                    .and_then(|_| channel.tx.start_transfer())?;
1922            }
1923        }
1924
1925        #[cfg(not(any(spi_master_version = "1", spi_master_version = "2")))]
1926        self.reset_dma();
1927
1928        self.driver.start_operation();
1929
1930        Ok(())
1931    }
1932
1933    fn enable_dma(&self) {
1934        cfg_select! {
1935            any(spi_master_version = "1", spi_master_version = "2") => {
1936                #[cfg(spi_master_version = "1")]
1937                crate::dma::connect_spi_dma(self.dma_peripheral, true);
1938                self.reset_dma();
1939            }
1940            _ => {
1941                self.regs().dma_conf().modify(|_, w| {
1942                    w.dma_tx_ena().set_bit();
1943                    w.dma_rx_ena().set_bit()
1944                });
1945            }
1946        }
1947    }
1948
1949    fn reset_dma(&self) {
1950        self.regs().dma_conf().toggle(|w, bit| {
1951            cfg_select! {
1952                any(spi_master_version = "1", spi_master_version = "2") => {
1953                    w.out_rst().bit(bit);
1954                    w.in_rst().bit(bit);
1955                    w.ahbm_fifo_rst().bit(bit);
1956                    w.ahbm_rst().bit(bit)
1957                }
1958                _ => {
1959                    w.rx_afifo_rst().bit(bit);
1960                    w.buf_afifo_rst().bit(bit);
1961                    w.dma_afifo_rst().bit(bit)
1962                }
1963            }
1964        });
1965
1966        self.clear_dma_interrupts();
1967    }
1968
1969    fn clear_dma_interrupts(&self) {
1970        self.regs().dma_int_clr().write(|w| {
1971            cfg_select! {
1972                any(spi_master_version = "1", spi_master_version = "2") => {
1973                    w.inlink_dscr_empty().clear_bit_by_one();
1974                    w.outlink_dscr_error().clear_bit_by_one();
1975                    w.inlink_dscr_error().clear_bit_by_one();
1976                    w.in_done().clear_bit_by_one();
1977                    w.in_err_eof().clear_bit_by_one();
1978                    w.in_suc_eof().clear_bit_by_one();
1979                    w.out_done().clear_bit_by_one();
1980                    w.out_eof().clear_bit_by_one();
1981                    w.out_total_eof().clear_bit_by_one()
1982                }
1983                _ => {
1984                    w.dma_infifo_full_err().clear_bit_by_one();
1985                    w.dma_outfifo_empty_err().clear_bit_by_one();
1986                    w.trans_done().clear_bit_by_one();
1987                    w.mst_rx_afifo_wfull_err().clear_bit_by_one();
1988                    w.mst_tx_afifo_rempty_err().clear_bit_by_one()
1989                }
1990            }
1991        });
1992    }
1993}
1994
1995struct DmaState {
1996    tx_transfer_in_progress: Cell<bool>,
1997    rx_transfer_in_progress: Cell<bool>,
1998
1999    rx_buffer: UnsafeCell<MaybeUninit<ScopedDmaRxBuf<'static>>>,
2000    tx_buffer: UnsafeCell<MaybeUninit<ScopedDmaTxBuf<'static>>>,
2001
2002    descriptors: UnsafeCell<InternalMemory<[DmaDescriptor; 2]>>,
2003
2004    #[cfg(all(spi_master_version = "1", spi_address_workaround))]
2005    default_tx_buffer: UnsafeCell<InternalMemory<[u8; 4]>>,
2006}
2007
2008impl DmaState {
2009    // Syntactic helper to get a mutable reference to the "empty" RX DMA buffer.
2010    //
2011    // # Safety
2012    //
2013    // The caller must ensure that Rust's aliasing rules are upheld.
2014    #[allow(
2015        clippy::mut_from_ref,
2016        reason = "Safety requirements ensure this is okay"
2017    )]
2018    unsafe fn rx_buffer(&self) -> &mut ScopedDmaRxBuf<'static> {
2019        unsafe { (&mut *self.rx_buffer.get()).assume_init_mut() }
2020    }
2021
2022    // Syntactic helper to get a mutable reference to the "empty" TX DMA buffer.
2023    //
2024    // # Safety
2025    //
2026    // The caller must ensure that Rust's aliasing rules are upheld.
2027    #[allow(
2028        clippy::mut_from_ref,
2029        reason = "Safety requirements ensure this is okay"
2030    )]
2031    unsafe fn tx_buffer(&self) -> &mut ScopedDmaTxBuf<'static> {
2032        unsafe { (&mut *self.tx_buffer.get()).assume_init_mut() }
2033    }
2034}
2035
2036// SAFETY: State belongs to the currently constructed driver instance. As such, it'll not be
2037// accessed concurrently in multiple threads.
2038unsafe impl Sync for DmaState {}
2039
2040for_each_spi_master!(
2041    (all $( ($peri:ident, $sys:ident, $sclk:ident $_cs:tt $_sio:tt $(, $is_qspi:tt)?)),* ) => {
2042        impl AnySpi<'_> {
2043            #[inline(always)]
2044            fn dma_state(&self) -> &'static DmaState {
2045                match &self.0 {
2046                    $(
2047                        super::any::Inner::$sys(_spi) => {
2048                            static DMA_STATE: DmaState = DmaState {
2049                                tx_transfer_in_progress: Cell::new(false),
2050                                rx_transfer_in_progress: Cell::new(false),
2051
2052                                rx_buffer: UnsafeCell::new(MaybeUninit::uninit()),
2053                                tx_buffer: UnsafeCell::new(MaybeUninit::uninit()),
2054
2055                                descriptors: UnsafeCell::new(InternalMemory::new([DmaDescriptor::EMPTY; 2])),
2056                                #[cfg(all(spi_master_version = "1", spi_address_workaround))]
2057                                default_tx_buffer: UnsafeCell::new(InternalMemory::new([0; 4])),
2058                            };
2059
2060                            &DMA_STATE
2061                        }
2062                    )*
2063                }
2064            }
2065        }
2066    };
2067);
2068
2069impl SpiWrapper<'_> {
2070    fn dma_state(&self) -> &'static DmaState {
2071        self.spi.dma_state()
2072    }
2073}
2074
2075with_spi_master_dma_engine! {
2076    ($engine:tt, $any_channel:ident) => {
2077        /// DMA channel trait for SPI peripherals.
2078        ///
2079        /// Implemented for each channel type that can serve a particular SPI instance `S`.
2080        #[instability::unstable]
2081        #[diagnostic::on_unimplemented(
2082            message = "The DMA channel cannot be used with this SPI peripheral",
2083            label = "This DMA channel",
2084            note = "Use a channel that matches the SPI instance."
2085        )]
2086        pub trait SpiMasterDmaChannel<'d, S>: crate::private::Sealed + Into<crate::dma::$any_channel<'d>> {}
2087
2088        crate::macros::impl_dma_channel_trait! {
2089            $engine,
2090            any_peri = AnySpi<'d>,
2091            peris = for_each_spi_master,
2092            ($peri:path, $ch:path) => {
2093                impl<'d> SpiMasterDmaChannel<'d, $peri> for $ch {}
2094            }
2095        }
2096
2097        // Proxy type so that the type-erased DMA channel can be named in the driver, regardless of the DMA engine.
2098        type SpiMasterErased<'d> = crate::dma::$any_channel<'d>;
2099    };
2100}