1use core::{
2 cell::{Cell, UnsafeCell},
3 cmp::min,
4 mem::{ManuallyDrop, MaybeUninit},
5 pin::Pin,
6 ptr::NonNull,
7 sync::atomic::{Ordering, fence},
8 task::{Context, Poll},
9};
10
11#[cfg(feature = "unstable")]
12use embedded_hal::spi::{ErrorType, SpiBus};
13use enumset::EnumSet;
14#[cfg(place_spi_master_driver_in_ram)]
15use procmacros::ram;
16
17use super::*;
18use crate::{
19 RegisterToggle,
20 dma::{
21 CHUNK_SIZE,
22 Channel,
23 DmaDescriptor,
24 DmaEligiblePeripheral,
25 DmaRxBuf,
26 DmaRxBuffer,
27 DmaTxBuf,
28 DmaTxBuffer,
29 NoBuffer,
30 ScopedDmaRxBuf,
31 ScopedDmaTxBuf,
32 TransferDirection,
33 aligned::{DmaAlignedMut, InternalMemory},
34 asynch::DmaRxFuture,
35 prepare_for_rx,
36 prepare_for_tx,
37 },
38 pac::spi2::RegisterBlock,
39 private::DropGuard,
40 soc::is_slice_in_dram,
41 spi::{DmaError, master::low_level::SpiClockGuard},
42};
43#[cfg(dma_can_access_psram)]
44use crate::{dma::ManualWritebackBuffer, soc::is_slice_in_psram};
45
46const MAX_DMA_SIZE: usize = 32736;
47
48impl<'d> Spi<'d, Blocking> {
49 #[doc_replace(
50 "dma_channel" => {
51 cfg(spi_master_dma_engine = "SPI_DMA") => "DMA_SPI2",
52 cfg(spi_master_dma_engine = "AHB_GDMA") => "DMA_CH0",
53 cfg(spi_master_dma_engine = "AXI_GDMA") => "DMA_AXI_CH0",
54 }
55 )]
56 #[instability::unstable]
75 pub fn with_dma(
76 self,
77 channel: impl SpiMasterDmaChannel<'d, AnySpi<'d>>,
78 ) -> SpiDma<'d, crate::Blocking> {
79 SpiDma::new_from_spi(self, channel.into())
80 }
81}
82
83#[doc_replace(
84 "dma_channel" => {
85 cfg(spi_master_dma_engine = "SPI_DMA") => "DMA_SPI2",
86 cfg(spi_master_dma_engine = "AHB_GDMA") => "DMA_CH0",
87 cfg(spi_master_dma_engine = "AXI_GDMA") => "DMA_AXI_CH0",
88 }
89)]
90#[cfg_attr(feature = "defmt", derive(defmt::Format))]
141pub struct SpiDma<'d, Dm>
142where
143 Dm: DriverMode,
144{
145 spi: SpiWrapper<'d>,
146 pub(crate) channel: Channel<Dm, SpiMasterErased<'d>>,
147}
148
149impl<Dm> crate::private::Sealed for SpiDma<'_, Dm> where Dm: DriverMode {}
150
151impl<Dm> core::fmt::Debug for SpiDma<'_, Dm>
152where
153 Dm: DriverMode + core::fmt::Debug,
154{
155 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
156 f.debug_struct("SpiDma").field("spi", &self.spi).finish()
157 }
158}
159
160#[instability::unstable]
161impl crate::interrupt::InterruptConfigurable for SpiDma<'_, Blocking> {
162 fn set_interrupt_handler(&mut self, handler: InterruptHandler) {
163 self.set_interrupt_handler(handler);
164 }
165}
166
167#[instability::unstable]
168impl<Dm> embassy_embedded_hal::SetConfig for SpiDma<'_, Dm>
169where
170 Dm: DriverMode,
171{
172 type Config = Config;
173 type ConfigError = ConfigError;
174
175 fn set_config(&mut self, config: &Self::Config) -> Result<(), Self::ConfigError> {
176 self.apply_config(config)
177 }
178}
179
180#[instability::unstable]
181impl<Dm> ErrorType for SpiDma<'_, Dm>
182where
183 Dm: DriverMode,
184{
185 type Error = Error;
186}
187
188#[instability::unstable]
189impl<Dm> SpiBus for SpiDma<'_, Dm>
190where
191 Dm: DriverMode,
192{
193 fn read(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
194 self.read(words)
195 }
196
197 fn write(&mut self, words: &[u8]) -> Result<(), Self::Error> {
198 self.write(words)
199 }
200
201 fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Self::Error> {
202 self.transfer(read, write)
203 }
204
205 fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
206 self.transfer_in_place(words)
207 }
208
209 fn flush(&mut self) -> Result<(), Self::Error> {
210 Ok(())
214 }
215}
216
217#[instability::unstable]
218impl embedded_hal_async::spi::SpiBus for SpiDma<'_, Async> {
219 async fn read(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
220 self.read_async(words).await
221 }
222
223 async fn write(&mut self, words: &[u8]) -> Result<(), Self::Error> {
224 self.write_async(words).await
225 }
226
227 async fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Self::Error> {
228 self.transfer_async(read, write).await
229 }
230
231 async fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Self::Error> {
232 self.transfer_in_place_async(words).await
233 }
234
235 async fn flush(&mut self) -> Result<(), Self::Error> {
236 Ok(())
240 }
241}
242
243impl<'d> SpiDma<'d, Blocking> {
244 #[instability::unstable]
246 pub fn into_async(self) -> SpiDma<'d, Async> {
247 self.spi
248 .set_interrupt_handler(self.spi.info().async_handler);
249 SpiDma {
250 spi: self.spi,
251 channel: self.channel.into_async(),
252 }
253 }
254
255 fn new_inner(spi: SpiWrapper<'d>, channel: SpiMasterErased<'d>) -> Self {
256 let channel = Channel::new(channel);
257 channel.runtime_ensure_compatible(spi.spi.dma_peripheral());
258
259 let state = spi.spi.dma_state();
260
261 state.tx_transfer_in_progress.set(false);
262 state.rx_transfer_in_progress.set(false);
263
264 let (tx_descriptors, rx_descriptors) = unsafe {
267 let descriptors = (&mut *state.descriptors.get()).get_mut().into_inner();
268 descriptors.fill(DmaDescriptor::EMPTY);
269 let (tx_descriptors, rx_descriptors) = descriptors.split_at_mut(1);
270 (
271 DmaAlignedMut::new_unchecked(tx_descriptors),
272 DmaAlignedMut::new_unchecked(rx_descriptors),
273 )
274 };
275
276 let tx_buffer = cfg_select! {
277 all(spi_master_version = "1", spi_address_workaround) => unsafe {
278 (&mut *state.default_tx_buffer.get()).get_mut().unsize()
279 },
280 _ => unsafe { DmaAlignedMut::new_unchecked(&mut [][..]) },
281 };
282
283 let rx_buffer = unwrap!(DmaRxBuf::new(rx_descriptors, unsafe {
284 DmaAlignedMut::new_unchecked(&mut [])
285 }));
286 let tx_buffer = unwrap!(DmaTxBuf::new(tx_descriptors, tx_buffer));
287
288 unsafe { (&mut *state.tx_buffer.get()).write(tx_buffer.into_scoped()) };
290 unsafe { (&mut *state.rx_buffer.get()).write(rx_buffer.into_scoped()) };
291
292 Self { spi, channel }
293 }
294
295 pub(super) fn new_from_spi(
296 spi_driver: Spi<'d, Blocking>,
297 channel: SpiMasterErased<'d>,
298 ) -> Self {
299 let spi = spi_driver.spi;
300
301 Self::new_inner(spi, channel)
302 }
303
304 #[instability::unstable]
306 pub fn listen(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {
307 self.driver().enable_listen(interrupts.into(), true);
308 }
309
310 #[instability::unstable]
312 pub fn unlisten(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {
313 self.driver().enable_listen(interrupts.into(), false);
314 }
315
316 #[instability::unstable]
318 pub fn interrupts(&mut self) -> EnumSet<SpiInterrupt> {
319 self.driver().interrupts()
320 }
321
322 #[instability::unstable]
324 pub fn clear_interrupts(&mut self, interrupts: impl Into<EnumSet<SpiInterrupt>>) {
325 self.driver().clear_interrupts(interrupts.into());
326 }
327
328 #[cfg_attr(
329 not(multi_core),
330 doc = "Registers an interrupt handler for the peripheral."
331 )]
332 #[cfg_attr(
333 multi_core,
334 doc = "Registers an interrupt handler for the peripheral on the current core."
335 )]
336 #[doc = ""]
337 #[instability::unstable]
347 pub fn set_interrupt_handler(&mut self, handler: InterruptHandler) {
348 self.spi.set_interrupt_handler(handler);
349 }
350}
351
352impl<'d> SpiDma<'d, Async> {
353 #[instability::unstable]
355 pub fn into_blocking(self) -> SpiDma<'d, Blocking> {
356 self.spi.disable_peri_interrupt_on_all_cores();
357 SpiDma {
358 spi: self.spi,
359 channel: self.channel.into_blocking(),
360 }
361 }
362
363 async fn wait_for_idle_async(&mut self) {
364 if self.dma_driver().state.rx_transfer_in_progress.get() {
365 _ = DmaRxFuture::new(&mut self.channel.rx).await;
366 self.dma_driver().state.rx_transfer_in_progress.set(false);
367 }
368
369 struct Fut(Driver);
370 impl Fut {
371 const DONE_EVENTS: EnumSet<SpiInterrupt> =
372 enumset::enum_set!(SpiInterrupt::TransferDone);
373 }
374 impl Future for Fut {
375 type Output = ();
376
377 fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
378 if !self.0.interrupts().is_disjoint(Self::DONE_EVENTS) {
379 #[cfg(any(spi_master_version = "1", spi_master_version = "2"))]
380 if self.0.busy() {
382 cx.waker().wake_by_ref();
383 return Poll::Pending;
384 }
385
386 self.0.clear_interrupts(Self::DONE_EVENTS);
387 return Poll::Ready(());
388 }
389
390 self.0.state.waker.register(cx.waker());
391 self.0.enable_listen(Self::DONE_EVENTS, true);
392 Poll::Pending
393 }
394 }
395 impl Drop for Fut {
396 fn drop(&mut self) {
397 self.0.enable_listen(Self::DONE_EVENTS, false);
398 }
399 }
400
401 if !self.is_done() {
402 Fut(self.driver()).await;
403 }
404
405 if self.dma_driver().state.tx_transfer_in_progress.get() {
406 if !self.channel.tx.is_done() {
408 self.channel.tx.stop_transfer();
409 }
410 self.dma_driver().state.tx_transfer_in_progress.set(false);
411 }
412 self.channel.rx.end_transfer();
413 self.channel.tx.end_transfer();
414 }
415
416 #[instability::unstable]
418 pub async fn read_async(&mut self, words: &mut [u8]) -> Result<(), Error> {
419 if words.is_empty() {
420 return Ok(());
421 }
422
423 let _clock = SpiClockGuard::new(self.spi.info());
424
425 self.driver().setup_full_duplex()?;
426
427 if self.use_blocking_transfer(words.len()) {
428 self.dma_driver().disable_dma();
429 return self.driver().read(words);
430 }
431
432 let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
433 let mut maybe_copy_buffer = match DmaOperationKind::for_read(words) {
434 DmaOperationKind::Copied => {
435 MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
436 }
437 DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
438 descriptors: &mut descriptors,
439 #[cfg(dma_can_access_psram)]
440 align_buffer: [const { None }; 2],
441 },
442 };
443
444 if maybe_copy_buffer.chunk_size() == 0 {
445 return Err(Error::from(DmaError::BufferTooSmall));
446 }
447
448 for chunk in words.chunks_mut(maybe_copy_buffer.chunk_size()) {
449 let read_bytes = chunk.len();
450 let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *chunk)) };
451 let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };
452
453 self.transfer_buffers_dma_async(read_bytes, 0, rx_buffer, tx_buffer)
454 .await?;
455
456 maybe_copy_buffer.finish(chunk);
457 }
458
459 Ok(())
460 }
461
462 #[instability::unstable]
464 pub async fn write_async(&mut self, words: &[u8]) -> Result<(), Error> {
465 if words.is_empty() {
466 return Ok(());
467 }
468
469 let _clock = SpiClockGuard::new(self.spi.info());
470
471 self.driver().setup_full_duplex()?;
472
473 if self.use_blocking_transfer(words.len()) {
474 self.dma_driver().disable_dma();
475 return self.driver().write(words);
476 }
477
478 let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
479 let mut maybe_copy_buffer = match DmaOperationKind::for_write(words) {
480 DmaOperationKind::Copied => {
481 MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
482 }
483 DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),
484 };
485
486 if maybe_copy_buffer.chunk_size() == 0 {
487 return Err(Error::from(DmaError::BufferTooSmall));
488 }
489
490 for chunk in words.chunks(maybe_copy_buffer.chunk_size()) {
491 let write_bytes = chunk.len();
492 let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) };
493 let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(chunk)) };
494
495 self.transfer_buffers_dma_async(0, write_bytes, rx_buffer, tx_buffer)
496 .await?;
497 }
498
499 Ok(())
500 }
501
502 #[instability::unstable]
505 pub async fn transfer_async(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Error> {
506 if read.is_empty() && write.is_empty() {
507 return Ok(());
508 }
509
510 let _clock = SpiClockGuard::new(self.spi.info());
511
512 self.driver().setup_full_duplex()?;
513
514 if self.use_blocking_transfer(read.len().max(write.len())) {
515 self.dma_driver().disable_dma();
516 return if read.is_empty() {
517 self.driver().write(write)
518 } else if write.is_empty() {
519 self.driver().read(read)
520 } else {
521 self.driver().transfer(read, write)
522 };
523 }
524
525 let common_length = min(read.len(), write.len());
526 let (read_common, read_remainder) = read.split_at_mut(common_length);
527 let (write_common, write_remainder) = write.split_at(common_length);
528
529 let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
531 let mut maybe_copy_rx_buffer = match DmaOperationKind::for_read(read_common) {
532 DmaOperationKind::Copied => {
533 MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
534 }
535 DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
536 descriptors: &mut rx_descriptors,
537 #[cfg(dma_can_access_psram)]
538 align_buffer: [const { None }; 2],
539 },
540 };
541
542 let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
543 let mut maybe_copy_tx_buffer = match DmaOperationKind::for_write(write_common) {
544 DmaOperationKind::Copied => {
545 MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
546 }
547 DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut tx_descriptors),
548 };
549
550 let chunk_size = min(
551 maybe_copy_rx_buffer.chunk_size(),
552 maybe_copy_tx_buffer.chunk_size(),
553 );
554
555 if chunk_size == 0 {
556 return Err(Error::from(DmaError::BufferTooSmall));
557 }
558
559 for (read_chunk, write_chunk) in read_common
560 .chunks_mut(chunk_size)
561 .zip(write_common.chunks(chunk_size))
562 {
563 let read_bytes = read_chunk.len();
564 let write_bytes = write_chunk.len();
565 let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(NonNull::from(write_chunk)) };
566 let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(NonNull::from(&mut *read_chunk)) };
567
568 self.transfer_buffers_dma_async(read_bytes, write_bytes, rx_buffer, tx_buffer)
569 .await?;
570
571 maybe_copy_rx_buffer.finish(read_chunk);
572 }
573
574 if !read_remainder.is_empty() {
575 self.read_async(read_remainder).await
576 } else if !write_remainder.is_empty() {
577 self.write_async(write_remainder).await
578 } else {
579 Ok(())
580 }
581 }
582
583 #[instability::unstable]
586 pub async fn transfer_in_place_async(&mut self, words: &mut [u8]) -> Result<(), Error> {
587 if words.is_empty() {
588 return Ok(());
589 }
590
591 let _clock = SpiClockGuard::new(self.spi.info());
592 self.driver().setup_full_duplex()?;
593
594 if self.use_blocking_transfer(words.len()) {
595 self.dma_driver().disable_dma();
596 return self.driver().transfer_in_place(words);
597 }
598
599 let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
600 let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
601 let (mut maybe_copy_rx_buffer, mut maybe_copy_tx_buffer) =
602 match DmaOperationKind::for_write(words) {
603 DmaOperationKind::Copied => (
604 MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }),
605 MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }),
606 ),
607 DmaOperationKind::InPlace => (
608 MaybeCopyRxBuf::Direct {
609 descriptors: &mut rx_descriptors,
610 #[cfg(dma_can_access_psram)]
611 align_buffer: [const { None }; 2],
612 },
613 MaybeCopyTxBuf::Direct(&mut tx_descriptors),
614 ),
615 };
616
617 let chunk_size = min(
618 maybe_copy_rx_buffer.chunk_size(),
619 maybe_copy_tx_buffer.chunk_size(),
620 );
621
622 if chunk_size == 0 {
623 return Err(Error::from(DmaError::BufferTooSmall));
624 }
625
626 for chunk in words.chunks_mut(chunk_size) {
627 let bytes = chunk.len();
628 let ptr = NonNull::from(&mut *chunk);
629 let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(ptr) };
630 let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(ptr) };
631
632 self.transfer_buffers_dma_async(bytes, bytes, rx_buffer, tx_buffer)
633 .await?;
634
635 maybe_copy_rx_buffer.finish(chunk);
636 }
637
638 Ok(())
639 }
640
641 #[instability::unstable]
648 pub async fn half_duplex_read_async(
649 &mut self,
650 data_mode: DataMode,
651 cmd: Command,
652 address: Address,
653 dummy: u8,
654 buffer: &mut [u8],
655 ) -> Result<(), Error> {
656 let _clock = SpiClockGuard::new(self.spi.info());
657
658 if buffer.is_empty() {
659 let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) };
660 self.half_duplex_read_dma_async(data_mode, cmd, address, dummy, 0, rx_buffer)
661 .await?;
662 return Ok(());
663 }
664
665 if self.use_blocking_transfer(buffer.len()) {
667 self.dma_driver().disable_dma();
668 return self
669 .driver()
670 .half_duplex_read(data_mode, cmd, address, dummy, buffer);
671 }
672
673 let operation = DmaOperationKind::for_read(buffer);
674 let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
675 let mut maybe_copy_buffer = match operation {
676 DmaOperationKind::Copied => {
677 MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
678 }
679 DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
680 descriptors: &mut descriptors,
681 #[cfg(dma_can_access_psram)]
682 align_buffer: [const { None }; 2],
683 },
684 };
685
686 let chunk_size = maybe_copy_buffer.chunk_size();
687 if chunk_size == 0 {
688 return Err(Error::from(DmaError::BufferTooSmall));
689 }
690 if buffer.len() > chunk_size {
691 return match operation {
692 DmaOperationKind::Copied => Err(Error::from(DmaError::Overflow)),
693 DmaOperationKind::InPlace => Err(Error::MaxDmaTransferSizeExceeded),
694 };
695 }
696
697 let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *buffer)) };
698 self.half_duplex_read_dma_async(data_mode, cmd, address, dummy, buffer.len(), rx_buffer)
699 .await?;
700 maybe_copy_buffer.finish(buffer);
701
702 Ok(())
703 }
704
705 #[instability::unstable]
712 pub async fn half_duplex_write_async(
713 &mut self,
714 data_mode: DataMode,
715 cmd: Command,
716 address: Address,
717 dummy: u8,
718 buffer: &[u8],
719 ) -> Result<(), Error> {
720 let _clock = SpiClockGuard::new(self.spi.info());
721
722 if buffer.is_empty() {
723 let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };
724 self.half_duplex_write_dma_async(data_mode, cmd, address, dummy, 0, tx_buffer)
725 .await?;
726 return Ok(());
727 }
728
729 if self.use_blocking_transfer(buffer.len()) {
731 self.dma_driver().disable_dma();
732 return self
733 .driver()
734 .half_duplex_write(data_mode, cmd, address, dummy, buffer);
735 }
736
737 let operation = DmaOperationKind::for_write(buffer);
738 let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
739 let mut maybe_copy_buffer = match operation {
740 DmaOperationKind::Copied => {
741 MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
742 }
743 DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),
744 };
745
746 let chunk_size = maybe_copy_buffer.chunk_size();
747 if chunk_size == 0 {
748 return Err(Error::from(DmaError::BufferTooSmall));
749 }
750 if buffer.len() > chunk_size {
751 return match operation {
752 DmaOperationKind::Copied => Err(Error::from(DmaError::Overflow)),
753 DmaOperationKind::InPlace => Err(Error::MaxDmaTransferSizeExceeded),
754 };
755 }
756
757 let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(buffer)) };
758 self.half_duplex_write_dma_async(data_mode, cmd, address, dummy, buffer.len(), tx_buffer)
759 .await
760 }
761
762 async fn transfer_buffers_dma_async(
763 &mut self,
764 read_bytes: usize,
765 write_bytes: usize,
766 mut rx_buffer: impl DmaRxBuffer,
767 mut tx_buffer: impl DmaTxBuffer,
768 ) -> Result<(), Error> {
769 let _clock = SpiClockGuard::new(self.spi.info());
770
771 let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());
772 unsafe {
773 spi.start_dma_transfer(read_bytes, write_bytes, &mut rx_buffer, &mut tx_buffer)?;
774 }
775 spi.wait_for_idle_async().await;
776 spi.defuse();
777 Ok(())
778 }
779
780 async fn half_duplex_read_dma_async(
781 &mut self,
782 data_mode: DataMode,
783 cmd: Command,
784 address: Address,
785 dummy: u8,
786 bytes_to_read: usize,
787 mut rx_buffer: impl DmaRxBuffer,
788 ) -> Result<(), Error> {
789 let _clock = SpiClockGuard::new(self.spi.info());
790
791 let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());
792 unsafe {
793 spi.start_half_duplex_read(
794 data_mode,
795 cmd,
796 address,
797 dummy,
798 bytes_to_read,
799 &mut rx_buffer,
800 )?;
801 }
802 spi.wait_for_idle_async().await;
803 spi.defuse();
804 Ok(())
805 }
806
807 async fn half_duplex_write_dma_async(
808 &mut self,
809 data_mode: DataMode,
810 cmd: Command,
811 address: Address,
812 dummy: u8,
813 bytes_to_write: usize,
814 mut tx_buffer: impl DmaTxBuffer,
815 ) -> Result<(), Error> {
816 let _clock = SpiClockGuard::new(self.spi.info());
817
818 let mut spi = DropGuard::new(&mut *self, |spi| spi.cancel_transfer());
819 unsafe {
820 spi.start_half_duplex_write(
821 data_mode,
822 cmd,
823 address,
824 dummy,
825 bytes_to_write,
826 &mut tx_buffer,
827 )?;
828 }
829 spi.wait_for_idle_async().await;
830 spi.defuse();
831 Ok(())
832 }
833}
834
835const LINK_DESCRIPTOR_COUNT: usize = MAX_DMA_SIZE.div_ceil(CHUNK_SIZE) + 2 + 1;
837
838enum MaybeCopyTxBuf<'a> {
839 Copy(&'a mut ScopedDmaTxBuf<'static>),
840 Direct(&'a mut [DmaDescriptor; LINK_DESCRIPTOR_COUNT]),
841}
842
843impl<'a> MaybeCopyTxBuf<'a> {
844 unsafe fn setup(&mut self, data: NonNull<[u8]>) -> NoBuffer {
845 match self {
846 MaybeCopyTxBuf::Copy(tx_buffer) => {
847 tx_buffer.as_mut_slice()[..data.len()].copy_from_slice(unsafe { data.as_ref() });
848 NoBuffer(tx_buffer.prepare())
849 }
850 MaybeCopyTxBuf::Direct(descriptors) => {
851 let (buffer, _) = unsafe { unwrap!(prepare_for_tx(&mut **descriptors, data, 1)) };
852 buffer
853 }
854 }
855 }
856
857 fn chunk_size(&self) -> usize {
858 match self {
859 MaybeCopyTxBuf::Copy(buffer) => buffer.capacity().min(MAX_DMA_SIZE),
860 MaybeCopyTxBuf::Direct(_) => MAX_DMA_SIZE,
861 }
862 }
863}
864
865#[allow(clippy::large_enum_variant)]
866enum MaybeCopyRxBuf<'a> {
867 Copy(&'a mut ScopedDmaRxBuf<'static>),
868 Direct {
869 descriptors: &'a mut [DmaDescriptor; LINK_DESCRIPTOR_COUNT],
870 #[cfg(dma_can_access_psram)]
871 align_buffer: [Option<ManualWritebackBuffer>; 2],
872 },
873}
874
875impl<'a> MaybeCopyRxBuf<'a> {
876 unsafe fn setup(&mut self, data: NonNull<[u8]>) -> NoBuffer {
877 match self {
878 MaybeCopyRxBuf::Copy(rx_buffer) => NoBuffer(rx_buffer.prepare()),
879 MaybeCopyRxBuf::Direct {
880 descriptors,
881 #[cfg(dma_can_access_psram)]
882 align_buffer,
883 } => {
884 let (buffer, _) = unsafe {
885 prepare_for_rx(
886 &mut **descriptors,
887 #[cfg(dma_can_access_psram)]
888 align_buffer,
889 data,
890 )
891 };
892 buffer
893 }
894 }
895 }
896
897 fn chunk_size(&self) -> usize {
898 match self {
899 MaybeCopyRxBuf::Copy(buffer) => buffer.capacity().min(MAX_DMA_SIZE),
900 MaybeCopyRxBuf::Direct { .. } => MAX_DMA_SIZE,
901 }
902 }
903
904 fn finish(&mut self, chunk: &mut [u8]) {
905 match self {
906 MaybeCopyRxBuf::Copy(buffer) => {
907 chunk.copy_from_slice(&buffer.as_slice()[..chunk.len()]);
908 }
909 MaybeCopyRxBuf::Direct {
910 #[cfg(dma_can_access_psram)]
911 align_buffer,
912 ..
913 } => {
914 #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
915 crate::dma::invalidate_received(NonNull::from(&*chunk));
916
917 #[cfg(dma_can_access_psram)]
918 for buffer in align_buffer.iter_mut() {
919 if let Some(buffer) = buffer.as_mut() {
920 buffer.write_back();
921 }
922 *buffer = None;
923 }
924 }
925 }
926 }
927}
928
929#[derive(Clone, Copy)]
930enum DmaOperationKind {
931 Copied,
933
934 InPlace,
936}
937
938impl DmaOperationKind {
939 fn compute(buffer: &[u8], direction: TransferDirection) -> Self {
940 fn is_dma_compatible(buffer: &[u8], _direction: TransferDirection) -> bool {
941 #[cfg(spi_master_version = "1")]
944 if !((buffer.as_ptr() as usize).is_multiple_of(4) && buffer.len().is_multiple_of(4)) {
945 return false;
946 }
947
948 if is_slice_in_dram(buffer) {
949 return true;
950 }
951 #[cfg(dma_can_access_psram)]
952 if is_slice_in_psram(buffer) {
953 #[cfg(spi_master_version = "2")]
954 if _direction == TransferDirection::In {
955 let tail_bytes = (buffer.as_ptr() as usize + buffer.len()).wrapping_neg()
960 & (crate::dma::aligned::PSRAM_CACHE_LINE - 1);
961 if tail_bytes > 0 {
962 return false;
963 }
964 }
965
966 return true;
967 }
968
969 false
972 }
973
974 if is_dma_compatible(buffer, direction) {
975 Self::InPlace
976 } else {
977 Self::Copied
978 }
979 }
980
981 fn for_read(buffer: &mut [u8]) -> Self {
982 Self::compute(buffer, TransferDirection::In)
983 }
984
985 fn for_write(buffer: &[u8]) -> Self {
986 Self::compute(buffer, TransferDirection::Out)
987 }
988}
989
990impl<'d, Dm> SpiDma<'d, Dm>
991where
992 Dm: DriverMode,
993{
994 fn use_blocking_transfer(&self, transfer_size: usize) -> bool {
995 let threshold = self
996 .spi
997 .state()
998 .min_async_transfer_size
999 .load(Ordering::Relaxed);
1000 threshold > 0 && transfer_size < threshold
1001 }
1002
1003 fn spi(&self) -> &SpiWrapper<'_> {
1004 &self.spi
1005 }
1006
1007 fn driver(&self) -> Driver {
1008 Driver {
1009 info: self.spi.info(),
1010 state: self.spi.state(),
1011 }
1012 }
1013
1014 fn dma_driver(&self) -> DmaDriver {
1015 DmaDriver {
1016 driver: self.driver(),
1017 state: self.spi().dma_state(),
1018 dma_peripheral: self.spi.spi.dma_peripheral(),
1019 }
1020 }
1021
1022 fn is_done(&self) -> bool {
1023 if self.driver().busy() {
1024 return false;
1025 }
1026 if self.dma_driver().state.rx_transfer_in_progress.get() {
1027 if !self.channel.rx.is_done() && !self.channel.rx.has_dscr_empty_error() {
1035 return false;
1036 }
1037 }
1038 true
1039 }
1040
1041 fn wait_for_idle(&mut self) {
1042 while !self.is_done() {
1043 }
1045 self.dma_driver().state.rx_transfer_in_progress.set(false);
1046 self.dma_driver().state.tx_transfer_in_progress.set(false);
1047 self.channel.rx.end_transfer();
1048 self.channel.tx.end_transfer();
1049 fence(Ordering::Acquire);
1050 }
1051
1052 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1057 unsafe fn start_transfer_dma<RX: DmaRxBuffer, TX: DmaTxBuffer>(
1058 &mut self,
1059 full_duplex: bool,
1060 bytes_to_read: usize,
1061 bytes_to_write: usize,
1062 rx_buffer: &mut RX,
1063 tx_buffer: &mut TX,
1064 ) -> Result<(), Error> {
1065 if bytes_to_read > MAX_DMA_SIZE || bytes_to_write > MAX_DMA_SIZE {
1066 return Err(Error::MaxDmaTransferSizeExceeded);
1067 }
1068
1069 self.dma_driver()
1070 .state
1071 .rx_transfer_in_progress
1072 .set(bytes_to_read > 0);
1073 self.dma_driver()
1074 .state
1075 .tx_transfer_in_progress
1076 .set(bytes_to_write > 0);
1077 unsafe {
1078 self.dma_driver().start_transfer_dma(
1079 full_duplex,
1080 bytes_to_read,
1081 bytes_to_write,
1082 rx_buffer,
1083 tx_buffer,
1084 &mut self.channel,
1085 )
1086 }
1087 }
1088
1089 #[cfg(all(spi_master_version = "1", spi_address_workaround))]
1094 unsafe fn set_up_address_workaround(
1095 &mut self,
1096 cmd: Command,
1097 address: Address,
1098 dummy: u8,
1099 ) -> Result<(), Error> {
1100 if dummy > 0 {
1101 error!("Dummy bits are not supported when there is no data to write");
1103 return Err(Error::Unsupported);
1104 }
1105
1106 let buffer = unsafe { self.dma_driver().tx_buffer() };
1107
1108 let bytes_to_write = address.width().div_ceil(8);
1109 let addr_bytes = address.value().to_be_bytes();
1112 let addr_bytes = &addr_bytes[4 - bytes_to_write..][..bytes_to_write];
1113 buffer.fill(addr_bytes);
1114
1115 self.driver().setup_half_duplex(
1116 true,
1117 cmd,
1118 Address::None,
1119 false,
1120 dummy,
1121 bytes_to_write == 0,
1122 address.mode(),
1123 )?;
1124
1125 let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1126
1127 unsafe { self.start_transfer_dma(false, 0, bytes_to_write, rx_buffer, buffer) }
1128 }
1129
1130 fn cancel_transfer(&mut self) {
1131 let state = self.dma_driver().state;
1132 if state.tx_transfer_in_progress.get() || state.rx_transfer_in_progress.get() {
1133 self.dma_driver().abort_transfer();
1134
1135 if state.tx_transfer_in_progress.get() {
1137 self.channel.tx.stop_transfer();
1138 state.tx_transfer_in_progress.set(false);
1139 }
1140 if state.rx_transfer_in_progress.get() {
1141 self.channel.rx.stop_transfer();
1142 state.rx_transfer_in_progress.set(false);
1143 }
1144 }
1145 }
1146
1147 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1152 unsafe fn start_dma_write(
1153 &mut self,
1154 bytes_to_write: usize,
1155 buffer: &mut impl DmaTxBuffer,
1156 ) -> Result<(), Error> {
1157 let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1158
1159 unsafe { self.start_dma_transfer(0, bytes_to_write, rx_buffer, buffer) }
1160 }
1161
1162 #[cfg_attr(
1169 not(spi_master_dma_can_access_flash),
1170 doc = "The DMA cannot read flash memory."
1171 )]
1172 #[cfg_attr(
1175 spi_master_version = "1",
1176 doc = "On ESP32, transferring from internal SRAM requires copying the entire buffer if it is
1177not 4-byte aligned. This is a limitation of the current implementation."
1178 )]
1179 #[cfg_attr(
1180 spi_master_version = "2",
1181 doc = "On ESP32-S2, receiving into PSRAM requires the buffer's _end_ to be 16-byte
1182aligned, otherwise the driver requires copying the entire buffer."
1183 )]
1184 #[doc = ""]
1185 #[instability::unstable]
1190 pub fn with_buffers(self, dma_rx_buf: DmaRxBuf, dma_tx_buf: DmaTxBuf) -> SpiDma<'d, Dm> {
1191 unsafe {
1192 (&mut *self.spi.dma_state().rx_buffer.get()).write(dma_rx_buf.into_scoped());
1193 (&mut *self.spi.dma_state().tx_buffer.get()).write(dma_tx_buf.into_scoped());
1194 }
1195 self
1196 }
1197
1198 #[allow(clippy::type_complexity)]
1204 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1205 #[instability::unstable]
1206 pub fn write_buffer<TX: DmaTxBuffer>(
1207 mut self,
1208 bytes_to_write: usize,
1209 mut buffer: TX,
1210 ) -> Result<SpiDmaTransfer<'d, Dm, TX>, (Error, Self, TX)> {
1211 let clock = SpiClockGuard::new(self.spi.info());
1212
1213 if let Err(e) = self.driver().setup_full_duplex() {
1214 return Err((e, self, buffer));
1215 };
1216 match unsafe { self.start_dma_write(bytes_to_write, &mut buffer) } {
1217 Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1218 Err(e) => Err((e, self, buffer)),
1219 }
1220 }
1221
1222 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1227 unsafe fn start_dma_read(
1228 &mut self,
1229 bytes_to_read: usize,
1230 buffer: &mut impl DmaRxBuffer,
1231 ) -> Result<(), Error> {
1232 let tx_buffer = unsafe { self.dma_driver().tx_buffer() };
1233
1234 unsafe { self.start_dma_transfer(bytes_to_read, 0, buffer, tx_buffer) }
1235 }
1236
1237 #[allow(clippy::type_complexity)]
1243 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1244 #[instability::unstable]
1245 pub fn read_buffer<RX: DmaRxBuffer>(
1246 mut self,
1247 bytes_to_read: usize,
1248 mut buffer: RX,
1249 ) -> Result<SpiDmaTransfer<'d, Dm, RX>, (Error, Self, RX)> {
1250 let clock = SpiClockGuard::new(self.spi.info());
1251
1252 if let Err(e) = self.driver().setup_full_duplex() {
1253 return Err((e, self, buffer));
1254 };
1255 match unsafe { self.start_dma_read(bytes_to_read, &mut buffer) } {
1256 Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1257 Err(e) => Err((e, self, buffer)),
1258 }
1259 }
1260
1261 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1266 unsafe fn start_dma_transfer(
1267 &mut self,
1268 bytes_to_read: usize,
1269 bytes_to_write: usize,
1270 rx_buffer: &mut impl DmaRxBuffer,
1271 tx_buffer: &mut impl DmaTxBuffer,
1272 ) -> Result<(), Error> {
1273 unsafe {
1274 self.start_transfer_dma(true, bytes_to_read, bytes_to_write, rx_buffer, tx_buffer)
1275 }
1276 }
1277
1278 #[allow(clippy::type_complexity)]
1284 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1285 #[instability::unstable]
1286 pub fn transfer_buffers<RX: DmaRxBuffer, TX: DmaTxBuffer>(
1287 mut self,
1288 bytes_to_read: usize,
1289 mut rx_buffer: RX,
1290 bytes_to_write: usize,
1291 mut tx_buffer: TX,
1292 ) -> Result<SpiDmaTransfer<'d, Dm, (RX, TX)>, (Error, Self, RX, TX)> {
1293 let clock = SpiClockGuard::new(self.spi.info());
1294
1295 if let Err(e) = self.driver().setup_full_duplex() {
1296 return Err((e, self, rx_buffer, tx_buffer));
1297 };
1298 match unsafe {
1299 self.start_dma_transfer(
1300 bytes_to_read,
1301 bytes_to_write,
1302 &mut rx_buffer,
1303 &mut tx_buffer,
1304 )
1305 } {
1306 Ok(_) => Ok(SpiDmaTransfer::new(self, (rx_buffer, tx_buffer), clock)),
1307 Err(e) => Err((e, self, rx_buffer, tx_buffer)),
1308 }
1309 }
1310
1311 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1316 unsafe fn start_half_duplex_read(
1317 &mut self,
1318 data_mode: DataMode,
1319 cmd: Command,
1320 address: Address,
1321 dummy: u8,
1322 bytes_to_read: usize,
1323 buffer: &mut impl DmaRxBuffer,
1324 ) -> Result<(), Error> {
1325 self.driver().setup_half_duplex(
1326 false,
1327 cmd,
1328 address,
1329 false,
1330 dummy,
1331 bytes_to_read == 0,
1332 data_mode,
1333 )?;
1334
1335 let tx_buffer = unsafe { self.dma_driver().tx_buffer() };
1336
1337 unsafe { self.start_transfer_dma(false, bytes_to_read, 0, buffer, tx_buffer) }
1338 }
1339
1340 #[allow(clippy::type_complexity)]
1342 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1343 #[instability::unstable]
1344 pub fn half_duplex_read_buffer<RX: DmaRxBuffer>(
1345 mut self,
1346 data_mode: DataMode,
1347 cmd: Command,
1348 address: Address,
1349 dummy: u8,
1350 bytes_to_read: usize,
1351 mut buffer: RX,
1352 ) -> Result<SpiDmaTransfer<'d, Dm, RX>, (Error, Self, RX)> {
1353 let clock = SpiClockGuard::new(self.spi.info());
1354
1355 match unsafe {
1356 self.start_half_duplex_read(data_mode, cmd, address, dummy, bytes_to_read, &mut buffer)
1357 } {
1358 Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1359 Err(e) => Err((e, self, buffer)),
1360 }
1361 }
1362
1363 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1368 unsafe fn start_half_duplex_write(
1369 &mut self,
1370 data_mode: DataMode,
1371 cmd: Command,
1372 address: Address,
1373 dummy: u8,
1374 bytes_to_write: usize,
1375 buffer: &mut impl DmaTxBuffer,
1376 ) -> Result<(), Error> {
1377 #[cfg(all(spi_master_version = "1", spi_address_workaround))]
1378 {
1379 if bytes_to_write == 0 && address.mode() != DataMode::SingleTwoDataLines {
1382 return unsafe { self.set_up_address_workaround(cmd, address, dummy) };
1383 }
1384 }
1385
1386 self.driver().setup_half_duplex(
1387 true,
1388 cmd,
1389 address,
1390 false,
1391 dummy,
1392 bytes_to_write == 0,
1393 data_mode,
1394 )?;
1395
1396 let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1397
1398 unsafe { self.start_transfer_dma(false, 0, bytes_to_write, rx_buffer, buffer) }
1399 }
1400
1401 #[allow(clippy::type_complexity)]
1403 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1404 #[instability::unstable]
1405 pub fn half_duplex_write_buffer<TX: DmaTxBuffer>(
1406 mut self,
1407 data_mode: DataMode,
1408 cmd: Command,
1409 address: Address,
1410 dummy: u8,
1411 bytes_to_write: usize,
1412 mut buffer: TX,
1413 ) -> Result<SpiDmaTransfer<'d, Dm, TX>, (Error, Self, TX)> {
1414 let clock = SpiClockGuard::new(self.spi.info());
1415
1416 match unsafe {
1417 self.start_half_duplex_write(
1418 data_mode,
1419 cmd,
1420 address,
1421 dummy,
1422 bytes_to_write,
1423 &mut buffer,
1424 )
1425 } {
1426 Ok(_) => Ok(SpiDmaTransfer::new(self, buffer, clock)),
1427 Err(e) => Err((e, self, buffer)),
1428 }
1429 }
1430
1431 #[doc_replace(
1432 "max_frequency" => {
1433 cfg(esp32h2) => "48MHz",
1434 _ => "80MHz",
1435 }
1436 )]
1437 #[instability::unstable]
1444 pub fn apply_config(&mut self, config: &Config) -> Result<(), ConfigError> {
1445 self.driver().apply_config(config)
1446 }
1447
1448 fn transfer_buffers_dma(
1449 &mut self,
1450 read_bytes: usize,
1451 write_bytes: usize,
1452 mut rx_buffer: impl DmaRxBuffer,
1453 mut tx_buffer: impl DmaTxBuffer,
1454 ) -> Result<(), Error> {
1455 unsafe {
1456 self.start_dma_transfer(read_bytes, write_bytes, &mut rx_buffer, &mut tx_buffer)?;
1457 }
1458 self.wait_for_idle();
1459 Ok(())
1460 }
1461
1462 #[instability::unstable]
1464 pub fn read(&mut self, words: &mut [u8]) -> Result<(), Error> {
1465 let _clock = SpiClockGuard::new(self.spi.info());
1466
1467 self.driver().setup_full_duplex()?;
1468
1469 if self.use_blocking_transfer(words.len()) {
1470 self.dma_driver().disable_dma();
1471 return self.driver().read(words);
1472 }
1473
1474 let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1475 let mut maybe_copy_buffer = match DmaOperationKind::for_read(words) {
1476 DmaOperationKind::Copied => {
1477 MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
1478 }
1479 DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
1480 descriptors: &mut descriptors,
1481 #[cfg(dma_can_access_psram)]
1482 align_buffer: [const { None }; 2],
1483 },
1484 };
1485
1486 if maybe_copy_buffer.chunk_size() == 0 {
1487 return Err(Error::from(DmaError::BufferTooSmall));
1488 }
1489
1490 for chunk in words.chunks_mut(maybe_copy_buffer.chunk_size()) {
1491 let read_bytes = chunk.len();
1492 let rx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(&mut *chunk)) };
1493 let tx_buffer = unsafe { NoBuffer(self.spi.dma_state().tx_buffer().prepare()) };
1494
1495 self.transfer_buffers_dma(read_bytes, 0, rx_buffer, tx_buffer)?;
1496
1497 maybe_copy_buffer.finish(chunk);
1498 }
1499
1500 Ok(())
1501 }
1502
1503 #[instability::unstable]
1505 pub fn write(&mut self, words: &[u8]) -> Result<(), Error> {
1506 let _clock = SpiClockGuard::new(self.spi.info());
1507
1508 self.driver().setup_full_duplex()?;
1509
1510 if self.use_blocking_transfer(words.len()) {
1511 self.dma_driver().disable_dma();
1512 return self.driver().write(words);
1513 }
1514
1515 let mut descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1516 let mut maybe_copy_buffer = match DmaOperationKind::for_write(words) {
1517 DmaOperationKind::Copied => {
1518 MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
1519 }
1520 DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut descriptors),
1521 };
1522
1523 if maybe_copy_buffer.chunk_size() == 0 {
1524 return Err(Error::from(DmaError::BufferTooSmall));
1525 }
1526
1527 for chunk in words.chunks(maybe_copy_buffer.chunk_size()) {
1528 let write_bytes = chunk.len();
1529 let rx_buffer = unsafe { NoBuffer(self.spi.dma_state().rx_buffer().prepare()) };
1530 let tx_buffer = unsafe { maybe_copy_buffer.setup(NonNull::from(chunk)) };
1531
1532 self.transfer_buffers_dma(0, write_bytes, rx_buffer, tx_buffer)?;
1533 }
1534
1535 Ok(())
1536 }
1537
1538 #[instability::unstable]
1540 pub fn transfer(&mut self, read: &mut [u8], write: &[u8]) -> Result<(), Error> {
1541 let _clock = SpiClockGuard::new(self.spi.info());
1542
1543 self.driver().setup_full_duplex()?;
1544
1545 if self.use_blocking_transfer(read.len().max(write.len())) {
1546 self.dma_driver().disable_dma();
1547 if read.is_empty() {
1548 return self.driver().write(write);
1549 } else if write.is_empty() {
1550 return self.driver().read(read);
1551 } else {
1552 return self.driver().transfer(read, write);
1553 }
1554 }
1555
1556 let common_length = min(read.len(), write.len());
1557 let (read_common, read_remainder) = read.split_at_mut(common_length);
1558 let (write_common, write_remainder) = write.split_at(common_length);
1559
1560 let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1562 let mut maybe_copy_rx_buffer = match DmaOperationKind::for_read(read_common) {
1563 DmaOperationKind::Copied => {
1564 MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() })
1565 }
1566 DmaOperationKind::InPlace => MaybeCopyRxBuf::Direct {
1567 descriptors: &mut rx_descriptors,
1568 #[cfg(dma_can_access_psram)]
1569 align_buffer: [const { None }; 2],
1570 },
1571 };
1572
1573 let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1574 let mut maybe_copy_tx_buffer = match DmaOperationKind::for_write(write_common) {
1575 DmaOperationKind::Copied => {
1576 MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() })
1577 }
1578 DmaOperationKind::InPlace => MaybeCopyTxBuf::Direct(&mut tx_descriptors),
1579 };
1580
1581 let chunk_size = min(
1582 maybe_copy_rx_buffer.chunk_size(),
1583 maybe_copy_tx_buffer.chunk_size(),
1584 );
1585
1586 if chunk_size == 0 {
1587 return Err(Error::from(DmaError::BufferTooSmall));
1588 }
1589
1590 for (read_chunk, write_chunk) in read_common
1591 .chunks_mut(chunk_size)
1592 .zip(write_common.chunks(chunk_size))
1593 {
1594 let read_bytes = read_chunk.len();
1595 let write_bytes = write_chunk.len();
1596 let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(NonNull::from(write_chunk)) };
1597 let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(NonNull::from(&mut *read_chunk)) };
1598
1599 self.transfer_buffers_dma(read_bytes, write_bytes, rx_buffer, tx_buffer)?;
1600
1601 maybe_copy_rx_buffer.finish(read_chunk);
1602 }
1603
1604 if !read_remainder.is_empty() {
1605 self.read(read_remainder)
1606 } else if !write_remainder.is_empty() {
1607 self.write(write_remainder)
1608 } else {
1609 Ok(())
1610 }
1611 }
1612
1613 #[instability::unstable]
1615 pub fn transfer_in_place(&mut self, words: &mut [u8]) -> Result<(), Error> {
1616 let _clock = SpiClockGuard::new(self.spi.info());
1617
1618 self.driver().setup_full_duplex()?;
1619
1620 if self.use_blocking_transfer(words.len()) {
1621 self.dma_driver().disable_dma();
1622 return self.driver().transfer_in_place(words);
1623 }
1624
1625 let mut rx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1626 let mut tx_descriptors = [DmaDescriptor::EMPTY; LINK_DESCRIPTOR_COUNT];
1627 let (mut maybe_copy_rx_buffer, mut maybe_copy_tx_buffer) =
1628 match DmaOperationKind::for_write(words) {
1629 DmaOperationKind::Copied => (
1630 MaybeCopyRxBuf::Copy(unsafe { self.spi.dma_state().rx_buffer() }),
1631 MaybeCopyTxBuf::Copy(unsafe { self.spi.dma_state().tx_buffer() }),
1632 ),
1633 DmaOperationKind::InPlace => (
1634 MaybeCopyRxBuf::Direct {
1635 descriptors: &mut rx_descriptors,
1636 #[cfg(dma_can_access_psram)]
1637 align_buffer: [const { None }; 2],
1638 },
1639 MaybeCopyTxBuf::Direct(&mut tx_descriptors),
1640 ),
1641 };
1642
1643 let chunk_size = min(
1644 maybe_copy_rx_buffer.chunk_size(),
1645 maybe_copy_tx_buffer.chunk_size(),
1646 );
1647
1648 if chunk_size == 0 {
1649 return Err(Error::from(DmaError::BufferTooSmall));
1650 }
1651
1652 for chunk in words.chunks_mut(chunk_size) {
1653 let bytes = chunk.len();
1654 let ptr = NonNull::from(&mut *chunk);
1655 let tx_buffer = unsafe { maybe_copy_tx_buffer.setup(ptr) };
1656 let rx_buffer = unsafe { maybe_copy_rx_buffer.setup(ptr) };
1657
1658 self.transfer_buffers_dma(bytes, bytes, rx_buffer, tx_buffer)?;
1659
1660 maybe_copy_rx_buffer.finish(chunk);
1661 }
1662
1663 Ok(())
1664 }
1665
1666 #[instability::unstable]
1668 pub fn half_duplex_read(
1669 &mut self,
1670 data_mode: DataMode,
1671 cmd: Command,
1672 address: Address,
1673 dummy: u8,
1674 buffer: &mut [u8],
1675 ) -> Result<(), Error> {
1676 let _clock = SpiClockGuard::new(self.spi.info());
1677
1678 let rx_buffer = unsafe { self.dma_driver().rx_buffer() };
1679 if rx_buffer.capacity() == 0 {
1680 return Err(Error::from(DmaError::BufferTooSmall));
1681 }
1682 if buffer.len() > rx_buffer.capacity() {
1683 return Err(Error::from(DmaError::Overflow));
1684 }
1685
1686 unsafe {
1687 self.start_half_duplex_read(data_mode, cmd, address, dummy, buffer.len(), rx_buffer)?;
1688 }
1689
1690 self.wait_for_idle();
1691
1692 buffer.copy_from_slice(&rx_buffer.as_slice()[..buffer.len()]);
1693
1694 Ok(())
1695 }
1696
1697 #[instability::unstable]
1699 pub fn half_duplex_write(
1700 &mut self,
1701 data_mode: DataMode,
1702 cmd: Command,
1703 address: Address,
1704 dummy: u8,
1705 buffer: &[u8],
1706 ) -> Result<(), Error> {
1707 let _clock = SpiClockGuard::new(self.spi.info());
1708
1709 let tx_buffer = unsafe { self.dma_driver().tx_buffer() };
1710 if tx_buffer.capacity() == 0 {
1711 return Err(Error::from(DmaError::BufferTooSmall));
1712 }
1713 if buffer.len() > tx_buffer.capacity() {
1714 return Err(Error::from(DmaError::Overflow));
1715 }
1716
1717 tx_buffer.as_mut_slice()[..buffer.len()].copy_from_slice(buffer);
1718
1719 unsafe {
1720 self.start_half_duplex_write(data_mode, cmd, address, dummy, buffer.len(), tx_buffer)?;
1721 }
1722
1723 self.wait_for_idle();
1724
1725 Ok(())
1726 }
1727}
1728
1729#[instability::unstable]
1733pub struct SpiDmaTransfer<'d, Dm, Buf>
1734where
1735 Dm: DriverMode,
1736{
1737 spi_dma: ManuallyDrop<SpiDma<'d, Dm>>,
1738 dma_buf: ManuallyDrop<Buf>,
1739 clock: ManuallyDrop<SpiClockGuard>,
1740}
1741
1742impl<Buf> SpiDmaTransfer<'_, Async, Buf> {
1743 #[instability::unstable]
1747 pub async fn wait_for_done(&mut self) {
1748 self.spi_dma.wait_for_idle_async().await;
1749 }
1750}
1751
1752impl<'d, Dm, Buf> SpiDmaTransfer<'d, Dm, Buf>
1753where
1754 Dm: DriverMode,
1755{
1756 fn new(spi_dma: SpiDma<'d, Dm>, dma_buf: Buf, clock: SpiClockGuard) -> Self {
1757 Self {
1758 spi_dma: ManuallyDrop::new(spi_dma),
1759 dma_buf: ManuallyDrop::new(dma_buf),
1760 clock: ManuallyDrop::new(clock),
1761 }
1762 }
1763
1764 #[instability::unstable]
1769 pub fn is_done(&self) -> bool {
1770 self.spi_dma.is_done()
1771 }
1772
1773 #[instability::unstable]
1778 pub fn wait(mut self) -> (SpiDma<'d, Dm>, Buf) {
1779 self.spi_dma.wait_for_idle();
1780 let retval = unsafe {
1781 (
1782 ManuallyDrop::take(&mut self.spi_dma),
1783 ManuallyDrop::take(&mut self.dma_buf),
1784 )
1785 };
1786 let _ = unsafe { ManuallyDrop::take(&mut self.clock) };
1787 core::mem::forget(self);
1788 retval
1789 }
1790
1791 #[instability::unstable]
1793 pub fn cancel(&mut self) {
1794 if !self.spi_dma.is_done() {
1795 self.spi_dma.cancel_transfer();
1796 }
1797 }
1798}
1799
1800impl<Dm, Buf> Drop for SpiDmaTransfer<'_, Dm, Buf>
1801where
1802 Dm: DriverMode,
1803{
1804 fn drop(&mut self) {
1805 if !self.is_done() {
1806 self.spi_dma.cancel_transfer();
1807 self.spi_dma.wait_for_idle();
1808 }
1809
1810 unsafe {
1811 ManuallyDrop::drop(&mut self.spi_dma);
1812 ManuallyDrop::drop(&mut self.dma_buf);
1813 }
1814 let _ = unsafe { ManuallyDrop::take(&mut self.clock) };
1815 }
1816}
1817
1818pub(super) struct DmaDriver {
1819 driver: Driver,
1820 dma_peripheral: crate::dma::DmaPeripheral,
1821 state: &'static DmaState,
1822}
1823
1824impl DmaDriver {
1825 unsafe fn rx_buffer(&self) -> &'static mut ScopedDmaRxBuf<'static> {
1826 unsafe { self.state.rx_buffer() }
1827 }
1828
1829 unsafe fn tx_buffer(&self) -> &'static mut ScopedDmaTxBuf<'static> {
1830 unsafe { self.state.tx_buffer() }
1831 }
1832
1833 fn abort_transfer(&self) {
1834 self.driver.configure_datalen(1, 1);
1841 self.driver.update();
1842 }
1843
1844 fn disable_dma(&self) {
1845 #[cfg(not(any(spi_master_version = "1", spi_master_version = "2")))]
1846 self.regs().dma_conf().modify(|_, w| {
1847 w.dma_tx_ena().clear_bit();
1848 w.dma_rx_ena().clear_bit()
1849 });
1850
1851 #[cfg(any(spi_master_version = "1", spi_master_version = "2"))]
1856 {
1857 self.regs().dma_out_link().write(|w| unsafe { w.bits(0) });
1858 self.regs().dma_in_link().write(|w| unsafe { w.bits(0) });
1859 }
1860 #[cfg(spi_master_version = "1")]
1861 crate::dma::connect_spi_dma(self.dma_peripheral, false);
1862 }
1863
1864 fn regs(&self) -> &RegisterBlock {
1865 self.driver.regs()
1866 }
1867
1868 #[cfg_attr(place_spi_master_driver_in_ram, ram)]
1869 unsafe fn start_transfer_dma<Dm: DriverMode>(
1870 &self,
1871 _full_duplex: bool,
1872 rx_len: usize,
1873 tx_len: usize,
1874 rx_buffer: &mut impl DmaRxBuffer,
1875 tx_buffer: &mut impl DmaTxBuffer,
1876 channel: &mut Channel<Dm, SpiMasterErased<'_>>,
1877 ) -> Result<(), Error> {
1878 #[cfg(spi_master_version = "2")]
1879 {
1880 self.regs().dma_out_link().write(|w| unsafe { w.bits(0) });
1882 self.regs().dma_in_link().write(|w| unsafe { w.bits(0) });
1883 }
1884
1885 self.driver.configure_datalen(rx_len, tx_len);
1886
1887 self.regs()
1889 .user()
1890 .modify(|_, w| w.usr_miso().bit(rx_len > 0).usr_mosi().bit(tx_len > 0));
1891
1892 self.enable_dma();
1893
1894 if rx_len > 0 {
1895 unsafe {
1896 channel
1897 .rx
1898 .prepare_transfer(self.dma_peripheral, rx_buffer)
1899 .and_then(|_| channel.rx.start_transfer())?;
1900 }
1901 } else {
1902 #[cfg(spi_master_version = "1")]
1903 {
1904 if _full_duplex {
1907 self.regs()
1908 .dma_in_link()
1909 .modify(|_, w| unsafe { w.inlink_addr().bits(0) });
1910 self.regs()
1911 .dma_in_link()
1912 .modify(|_, w| w.inlink_start().set_bit());
1913 }
1914 }
1915 }
1916 if tx_len > 0 {
1917 unsafe {
1918 channel
1919 .tx
1920 .prepare_transfer(self.dma_peripheral, tx_buffer)
1921 .and_then(|_| channel.tx.start_transfer())?;
1922 }
1923 }
1924
1925 #[cfg(not(any(spi_master_version = "1", spi_master_version = "2")))]
1926 self.reset_dma();
1927
1928 self.driver.start_operation();
1929
1930 Ok(())
1931 }
1932
1933 fn enable_dma(&self) {
1934 cfg_select! {
1935 any(spi_master_version = "1", spi_master_version = "2") => {
1936 #[cfg(spi_master_version = "1")]
1937 crate::dma::connect_spi_dma(self.dma_peripheral, true);
1938 self.reset_dma();
1939 }
1940 _ => {
1941 self.regs().dma_conf().modify(|_, w| {
1942 w.dma_tx_ena().set_bit();
1943 w.dma_rx_ena().set_bit()
1944 });
1945 }
1946 }
1947 }
1948
1949 fn reset_dma(&self) {
1950 self.regs().dma_conf().toggle(|w, bit| {
1951 cfg_select! {
1952 any(spi_master_version = "1", spi_master_version = "2") => {
1953 w.out_rst().bit(bit);
1954 w.in_rst().bit(bit);
1955 w.ahbm_fifo_rst().bit(bit);
1956 w.ahbm_rst().bit(bit)
1957 }
1958 _ => {
1959 w.rx_afifo_rst().bit(bit);
1960 w.buf_afifo_rst().bit(bit);
1961 w.dma_afifo_rst().bit(bit)
1962 }
1963 }
1964 });
1965
1966 self.clear_dma_interrupts();
1967 }
1968
1969 fn clear_dma_interrupts(&self) {
1970 self.regs().dma_int_clr().write(|w| {
1971 cfg_select! {
1972 any(spi_master_version = "1", spi_master_version = "2") => {
1973 w.inlink_dscr_empty().clear_bit_by_one();
1974 w.outlink_dscr_error().clear_bit_by_one();
1975 w.inlink_dscr_error().clear_bit_by_one();
1976 w.in_done().clear_bit_by_one();
1977 w.in_err_eof().clear_bit_by_one();
1978 w.in_suc_eof().clear_bit_by_one();
1979 w.out_done().clear_bit_by_one();
1980 w.out_eof().clear_bit_by_one();
1981 w.out_total_eof().clear_bit_by_one()
1982 }
1983 _ => {
1984 w.dma_infifo_full_err().clear_bit_by_one();
1985 w.dma_outfifo_empty_err().clear_bit_by_one();
1986 w.trans_done().clear_bit_by_one();
1987 w.mst_rx_afifo_wfull_err().clear_bit_by_one();
1988 w.mst_tx_afifo_rempty_err().clear_bit_by_one()
1989 }
1990 }
1991 });
1992 }
1993}
1994
1995struct DmaState {
1996 tx_transfer_in_progress: Cell<bool>,
1997 rx_transfer_in_progress: Cell<bool>,
1998
1999 rx_buffer: UnsafeCell<MaybeUninit<ScopedDmaRxBuf<'static>>>,
2000 tx_buffer: UnsafeCell<MaybeUninit<ScopedDmaTxBuf<'static>>>,
2001
2002 descriptors: UnsafeCell<InternalMemory<[DmaDescriptor; 2]>>,
2003
2004 #[cfg(all(spi_master_version = "1", spi_address_workaround))]
2005 default_tx_buffer: UnsafeCell<InternalMemory<[u8; 4]>>,
2006}
2007
2008impl DmaState {
2009 #[allow(
2015 clippy::mut_from_ref,
2016 reason = "Safety requirements ensure this is okay"
2017 )]
2018 unsafe fn rx_buffer(&self) -> &mut ScopedDmaRxBuf<'static> {
2019 unsafe { (&mut *self.rx_buffer.get()).assume_init_mut() }
2020 }
2021
2022 #[allow(
2028 clippy::mut_from_ref,
2029 reason = "Safety requirements ensure this is okay"
2030 )]
2031 unsafe fn tx_buffer(&self) -> &mut ScopedDmaTxBuf<'static> {
2032 unsafe { (&mut *self.tx_buffer.get()).assume_init_mut() }
2033 }
2034}
2035
2036unsafe impl Sync for DmaState {}
2039
2040for_each_spi_master!(
2041 (all $( ($peri:ident, $sys:ident, $sclk:ident $_cs:tt $_sio:tt $(, $is_qspi:tt)?)),* ) => {
2042 impl AnySpi<'_> {
2043 #[inline(always)]
2044 fn dma_state(&self) -> &'static DmaState {
2045 match &self.0 {
2046 $(
2047 super::any::Inner::$sys(_spi) => {
2048 static DMA_STATE: DmaState = DmaState {
2049 tx_transfer_in_progress: Cell::new(false),
2050 rx_transfer_in_progress: Cell::new(false),
2051
2052 rx_buffer: UnsafeCell::new(MaybeUninit::uninit()),
2053 tx_buffer: UnsafeCell::new(MaybeUninit::uninit()),
2054
2055 descriptors: UnsafeCell::new(InternalMemory::new([DmaDescriptor::EMPTY; 2])),
2056 #[cfg(all(spi_master_version = "1", spi_address_workaround))]
2057 default_tx_buffer: UnsafeCell::new(InternalMemory::new([0; 4])),
2058 };
2059
2060 &DMA_STATE
2061 }
2062 )*
2063 }
2064 }
2065 }
2066 };
2067);
2068
2069impl SpiWrapper<'_> {
2070 fn dma_state(&self) -> &'static DmaState {
2071 self.spi.dma_state()
2072 }
2073}
2074
2075with_spi_master_dma_engine! {
2076 ($engine:tt, $any_channel:ident) => {
2077 #[instability::unstable]
2081 #[diagnostic::on_unimplemented(
2082 message = "The DMA channel cannot be used with this SPI peripheral",
2083 label = "This DMA channel",
2084 note = "Use a channel that matches the SPI instance."
2085 )]
2086 pub trait SpiMasterDmaChannel<'d, S>: crate::private::Sealed + Into<crate::dma::$any_channel<'d>> {}
2087
2088 crate::macros::impl_dma_channel_trait! {
2089 $engine,
2090 any_peri = AnySpi<'d>,
2091 peris = for_each_spi_master,
2092 ($peri:path, $ch:path) => {
2093 impl<'d> SpiMasterDmaChannel<'d, $peri> for $ch {}
2094 }
2095 }
2096
2097 type SpiMasterErased<'d> = crate::dma::$any_channel<'d>;
2099 };
2100}