1use core::{
2 mem::ManuallyDrop,
3 ops::{Deref, DerefMut},
4};
5
6use enumset::EnumSet;
7
8#[cfg(dma_mem2mem_requires_peripheral)]
9use crate::dma::DmaEligiblePeripheral;
10use crate::{
11 Async,
12 Blocking,
13 DriverMode,
14 dma::{
15 BurstConfig,
16 Channel,
17 ChannelRx,
18 ChannelTx,
19 DmaChannel,
20 DmaDescriptor,
21 DmaError,
22 DmaPeripheral,
23 DmaRxBuf,
24 DmaRxBuffer,
25 DmaRxInterrupt,
26 DmaTxBuf,
27 DmaTxBuffer,
28 DmaTxInterrupt,
29 aligned::DmaAlignedMut,
30 },
31};
32
33#[diagnostic::on_unimplemented(
38 message = "this DMA channel does not support memory-to-memory transfers",
39 note = "Use a channel with `mem2mem = true` in device metadata. See `Mem2Mem::new`."
40)]
41pub trait Mem2MemCapableChannel<'d>: DmaChannel {
42 #[doc(hidden)]
43 type Erased: DmaChannel + From<Self>;
44
45 #[cfg(not(dma_mem2mem_requires_peripheral))]
47 fn mem2mem_id(&self) -> DmaPeripheral;
48
49 #[doc(hidden)]
50 #[allow(private_interfaces)]
51 fn into_channel(self) -> ErasedChannel<'d, Blocking>;
52}
53
54for_each_mem2mem_channel! {
56 (engines $( ($engine:literal, $variant:ident, $any_ch:ident) ),* ) => {
57 struct ErasedChannel<'d, Dm: DriverMode> {
58 rx: ErasedChannelRx<'d, Dm>,
59 tx: ErasedChannelTx<'d, Dm>,
60 }
61
62 enum ErasedChannelRx<'d, Dm: DriverMode> {
63 $(
64 $variant(ChannelRx<Dm, <crate::dma::$any_ch<'d> as DmaChannel>::Rx>),
65 )*
66 }
67
68 impl<Dm: DriverMode> ErasedChannelRx<'_, Dm> {
69 delegate::delegate! {
70 to match self {
71 $( Self::$variant(channel) => channel, )*
72 } {
73 fn has_error(&self) -> bool;
74 fn pending_in_interrupts(&self) -> EnumSet<DmaRxInterrupt>;
75 #[cfg(dma_mem2mem_requires_peripheral)]
76 fn runtime_ensure_compatible(&self, peripheral: DmaPeripheral);
77 }
78
79 to match self {
80 $( Self::$variant(channel) => channel, )*
81 } {
82 fn set_mem2mem_mode(&mut self, value: bool);
83 fn start_transfer(&mut self) -> Result<(), DmaError>;
84 fn stop_transfer(&mut self);
85 fn end_transfer(&mut self);
86 unsafe fn prepare_transfer<BUF: DmaRxBuffer>(
87 &mut self,
88 peri: DmaPeripheral,
89 buffer: &mut BUF,
90 ) -> Result<(), DmaError>;
91 }
92 }
93 }
94
95 impl<'d> ErasedChannelRx<'d, Blocking> {
96 fn into_async(self) -> ErasedChannelRx<'d, Async> {
97 match self {
98 $( Self::$variant(channel) => ErasedChannelRx::$variant(channel.into_async()), )*
99 }
100 }
101 }
102
103 impl<'d> ErasedChannelRx<'d, Async> {
104 fn into_blocking(self) -> ErasedChannelRx<'d, Blocking> {
105 match self {
106 $( Self::$variant(channel) => ErasedChannelRx::$variant(channel.into_blocking()), )*
107 }
108 }
109 }
110
111 enum ErasedChannelTx<'d, Dm: DriverMode> {
112 $(
113 $variant(ChannelTx<Dm, <crate::dma::$any_ch<'d> as DmaChannel>::Tx>),
114 )*
115 }
116
117 impl<Dm: DriverMode> ErasedChannelTx<'_, Dm> {
118 delegate::delegate! {
119 to match self {
120 $( Self::$variant(channel) => channel, )*
121 } {
122 fn has_error(&self) -> bool;
123 fn pending_out_interrupts(&self) -> EnumSet<DmaTxInterrupt>;
124 }
125
126 to match self {
127 $( Self::$variant(channel) => channel, )*
128 } {
129 fn start_transfer(&mut self) -> Result<(), DmaError>;
130 fn stop_transfer(&mut self);
131 fn end_transfer(&mut self);
132 unsafe fn prepare_transfer<BUF: DmaTxBuffer>(
133 &mut self,
134 peri: DmaPeripheral,
135 buffer: &mut BUF,
136 ) -> Result<(), DmaError>;
137 }
138 }
139 }
140
141 impl<'d> ErasedChannelTx<'d, Blocking> {
142 fn into_async(self) -> ErasedChannelTx<'d, Async> {
143 match self {
144 $( Self::$variant(channel) => ErasedChannelTx::$variant(channel.into_async()), )*
145 }
146 }
147 }
148
149 impl<'d> ErasedChannelTx<'d, Async> {
150 fn into_blocking(self) -> ErasedChannelTx<'d, Blocking> {
151 match self {
152 $( Self::$variant(channel) => ErasedChannelTx::$variant(channel.into_blocking()), )*
153 }
154 }
155 }
156 };
157}
158
159for_each_mem2mem_channel! {
160 ($engine:literal, $variant:ident, $any_ch:ident, $($hw:literal, $id:literal),+) => {
161 impl<'d> Mem2MemCapableChannel<'d> for crate::dma::$any_ch<'d> {
162 type Erased = Self;
163
164 #[cfg(not(dma_mem2mem_requires_peripheral))]
165 fn mem2mem_id(&self) -> DmaPeripheral {
166 match self.channel_index() {
167 $( $hw => DmaPeripheral($id), )+
168 ch => panic!(
169 "Channel {} does not support memory-to-memory transfers",
170 ch
171 ),
172 }
173 }
174
175 fn into_channel(self) -> ErasedChannel<'d, Blocking> {
176 let channel = Channel::new(self);
177 ErasedChannel {
178 rx: ErasedChannelRx::$variant(channel.rx),
179 tx: ErasedChannelTx::$variant(channel.tx),
180 }
181 }
182 }
183 };
184 ($engine:literal, $variant:ident, $any_ch:ident, $ch:ident, $id:literal) => {
185 impl<'d> Mem2MemCapableChannel<'d> for crate::peripherals::$ch<'d> {
186 type Erased = crate::dma::$any_ch<'d>;
187
188 #[cfg(not(dma_mem2mem_requires_peripheral))]
189 fn mem2mem_id(&self) -> DmaPeripheral {
190 DmaPeripheral($id)
191 }
192
193 fn into_channel(self) -> ErasedChannel<'d, Blocking> {
194 let channel = Channel::new(crate::dma::$any_ch::from(self));
195 ErasedChannel {
196 rx: ErasedChannelRx::$variant(channel.rx),
197 tx: ErasedChannelTx::$variant(channel.tx),
198 }
199 }
200 }
201 };
202}
203
204pub struct Mem2Mem<'d, Dm>
210where
211 Dm: DriverMode,
212{
213 pub rx: Mem2MemRx<'d, Dm>,
215 pub tx: Mem2MemTx<'d, Dm>,
217}
218
219impl<'d> Mem2Mem<'d, Blocking> {
220 pub fn new<CH>(
222 channel: CH,
223 #[cfg(dma_mem2mem_requires_peripheral)] peripheral: impl DmaEligiblePeripheral<CH::Erased>,
224 ) -> Self
225 where
226 CH: Mem2MemCapableChannel<'d>,
227 {
228 let dma_peri = cfg_select! {
229 dma_mem2mem_requires_peripheral => peripheral.dma_peripheral(),
230 _ => channel.mem2mem_id(),
231 };
232 Self::new_inner(channel, dma_peri)
233 }
234
235 #[cfg(dma_mem2mem_requires_peripheral)]
243 pub unsafe fn new_unsafe<CH>(channel: CH, peripheral: DmaPeripheral) -> Self
244 where
245 CH: Mem2MemCapableChannel<'d>,
246 {
247 Self::new_inner(channel, peripheral)
248 }
249
250 pub fn into_async(self) -> Mem2Mem<'d, Async> {
252 Mem2Mem {
253 rx: self.rx.into_async(),
254 tx: self.tx.into_async(),
255 }
256 }
257}
258
259impl<'d> Mem2Mem<'d, Blocking> {
260 fn new_inner(channel: impl Mem2MemCapableChannel<'d>, peripheral: DmaPeripheral) -> Self {
261 let mut channel = channel.into_channel();
262
263 #[cfg(dma_mem2mem_requires_peripheral)]
264 channel.rx.runtime_ensure_compatible(peripheral);
265
266 #[cfg(dma_supports_mem2mem)]
267 channel.rx.set_mem2mem_mode(true);
268
269 Mem2Mem {
270 rx: Mem2MemRx {
271 channel: channel.rx,
272 peripheral,
273 },
274 tx: Mem2MemTx {
275 channel: channel.tx,
276 peripheral,
277 },
278 }
279 }
280
281 pub fn with_descriptors(
283 self,
284 rx_descriptors: &'d mut [DmaDescriptor],
285 tx_descriptors: &'d mut [DmaDescriptor],
286 config: BurstConfig,
287 ) -> Result<SimpleMem2Mem<'d, Blocking>, DmaError> {
288 SimpleMem2Mem::new(self, rx_descriptors, tx_descriptors, config)
289 }
290}
291
292pub struct Mem2MemRx<'d, Dm>
294where
295 Dm: DriverMode,
296{
297 channel: ErasedChannelRx<'d, Dm>,
298 peripheral: DmaPeripheral,
299}
300
301impl<'d> Mem2MemRx<'d, Blocking> {
302 pub fn into_async(self) -> Mem2MemRx<'d, Async> {
304 Mem2MemRx {
305 channel: self.channel.into_async(),
306 peripheral: self.peripheral,
307 }
308 }
309}
310
311impl<'d, Dm> Mem2MemRx<'d, Dm>
312where
313 Dm: DriverMode,
314{
315 pub fn receive<BUF>(
317 mut self,
318 mut buf: BUF,
319 ) -> Result<Mem2MemRxTransfer<'d, BUF, Dm>, (DmaError, Self, BUF)>
320 where
321 BUF: DmaRxBuffer,
322 {
323 let result = unsafe {
324 self.channel
325 .prepare_transfer(self.peripheral, &mut buf)
326 .and_then(|_| self.channel.start_transfer())
327 };
328
329 if let Err(e) = result {
330 return Err((e, self, buf));
331 }
332
333 Ok(Mem2MemRxTransfer {
334 m2m: ManuallyDrop::new(self),
335 buf_view: ManuallyDrop::new(buf.into_view()),
336 })
337 }
338}
339
340pub struct Mem2MemRxTransfer<'d, BUF, Dm>
343where
344 BUF: DmaRxBuffer,
345 Dm: DriverMode,
346{
347 m2m: ManuallyDrop<Mem2MemRx<'d, Dm>>,
348 buf_view: ManuallyDrop<BUF::View>,
349}
350
351impl<'d, BUF, Dm> Mem2MemRxTransfer<'d, BUF, Dm>
352where
353 BUF: DmaRxBuffer,
354 Dm: DriverMode,
355{
356 pub fn is_done(&self) -> bool {
358 let done_interrupts = DmaRxInterrupt::DescriptorError | DmaRxInterrupt::DescriptorEmpty;
359 !self
360 .m2m
361 .channel
362 .pending_in_interrupts()
363 .is_disjoint(done_interrupts)
364 }
365
366 pub fn wait(self) -> (Result<(), DmaError>, Mem2MemRx<'d, Dm>, BUF::Final) {
368 while !self.is_done() {}
369
370 let (mut m2m, view) = self.release();
371 m2m.channel.end_transfer();
372
373 let result = if m2m.channel.has_error() {
374 Err(DmaError::DescriptorError)
375 } else {
376 Ok(())
377 };
378
379 (result, m2m, BUF::from_view(view))
380 }
381
382 pub fn stop(self) -> (Mem2MemRx<'d, Dm>, BUF::Final) {
384 let (mut m2m, view) = self.release();
385
386 m2m.channel.stop_transfer();
387
388 (m2m, BUF::from_view(view))
389 }
390
391 fn release(mut self) -> (Mem2MemRx<'d, Dm>, BUF::View) {
392 let result = unsafe {
395 let m2m = ManuallyDrop::take(&mut self.m2m);
396 let view = ManuallyDrop::take(&mut self.buf_view);
397 (m2m, view)
398 };
399 core::mem::forget(self);
400 result
401 }
402}
403
404impl<'d, BUF, Dm> Deref for Mem2MemRxTransfer<'d, BUF, Dm>
405where
406 BUF: DmaRxBuffer,
407 Dm: DriverMode,
408{
409 type Target = BUF::View;
410
411 fn deref(&self) -> &Self::Target {
412 &self.buf_view
413 }
414}
415
416impl<'d, BUF, Dm> DerefMut for Mem2MemRxTransfer<'d, BUF, Dm>
417where
418 BUF: DmaRxBuffer,
419 Dm: DriverMode,
420{
421 fn deref_mut(&mut self) -> &mut Self::Target {
422 &mut self.buf_view
423 }
424}
425
426impl<'d, BUF, Dm> Drop for Mem2MemRxTransfer<'d, BUF, Dm>
427where
428 BUF: DmaRxBuffer,
429 Dm: DriverMode,
430{
431 fn drop(&mut self) {
432 self.m2m.channel.stop_transfer();
433
434 let view = unsafe {
437 ManuallyDrop::drop(&mut self.m2m);
438 ManuallyDrop::take(&mut self.buf_view)
439 };
440 let _ = BUF::from_view(view);
441 }
442}
443
444pub struct Mem2MemTx<'d, Dm>
446where
447 Dm: DriverMode,
448{
449 channel: ErasedChannelTx<'d, Dm>,
450 peripheral: DmaPeripheral,
451}
452
453impl<'d> Mem2MemTx<'d, Blocking> {
454 pub fn into_async(self) -> Mem2MemTx<'d, Async> {
456 Mem2MemTx {
457 channel: self.channel.into_async(),
458 peripheral: self.peripheral,
459 }
460 }
461}
462
463impl<'d, Dm> Mem2MemTx<'d, Dm>
464where
465 Dm: DriverMode,
466{
467 pub fn send<BUF>(
469 mut self,
470 mut buf: BUF,
471 ) -> Result<Mem2MemTxTransfer<'d, BUF, Dm>, (DmaError, Self, BUF)>
472 where
473 BUF: DmaTxBuffer,
474 {
475 let result = unsafe {
476 self.channel
477 .prepare_transfer(self.peripheral, &mut buf)
478 .and_then(|_| self.channel.start_transfer())
479 };
480
481 if let Err(e) = result {
482 return Err((e, self, buf));
483 }
484
485 Ok(Mem2MemTxTransfer {
486 m2m: ManuallyDrop::new(self),
487 buf_view: ManuallyDrop::new(buf.into_view()),
488 })
489 }
490}
491
492pub struct Mem2MemTxTransfer<'d, BUF, Dm>
495where
496 BUF: DmaTxBuffer,
497 Dm: DriverMode,
498{
499 m2m: ManuallyDrop<Mem2MemTx<'d, Dm>>,
500 buf_view: ManuallyDrop<BUF::View>,
501}
502
503impl<'d, BUF, Dm> Mem2MemTxTransfer<'d, BUF, Dm>
504where
505 BUF: DmaTxBuffer,
506 Dm: DriverMode,
507{
508 pub fn is_done(&self) -> bool {
510 let done_interrupts = DmaTxInterrupt::DescriptorError | DmaTxInterrupt::TotalEof;
511 !self
512 .m2m
513 .channel
514 .pending_out_interrupts()
515 .is_disjoint(done_interrupts)
516 }
517
518 pub fn wait(self) -> (Result<(), DmaError>, Mem2MemTx<'d, Dm>, BUF::Final) {
520 while !self.is_done() {}
521
522 let (mut m2m, view) = self.release();
523 m2m.channel.end_transfer();
524
525 let result = if m2m.channel.has_error() {
526 Err(DmaError::DescriptorError)
527 } else {
528 Ok(())
529 };
530
531 (result, m2m, BUF::from_view(view))
532 }
533
534 pub fn stop(self) -> (Mem2MemTx<'d, Dm>, BUF::Final) {
536 let (mut m2m, view) = self.release();
537
538 m2m.channel.stop_transfer();
539
540 (m2m, BUF::from_view(view))
541 }
542
543 fn release(mut self) -> (Mem2MemTx<'d, Dm>, BUF::View) {
544 let result = unsafe {
547 let m2m = ManuallyDrop::take(&mut self.m2m);
548 let view = ManuallyDrop::take(&mut self.buf_view);
549 (m2m, view)
550 };
551 core::mem::forget(self);
552 result
553 }
554}
555
556impl<'d, BUF, Dm> Deref for Mem2MemTxTransfer<'d, BUF, Dm>
557where
558 BUF: DmaTxBuffer,
559 Dm: DriverMode,
560{
561 type Target = BUF::View;
562
563 fn deref(&self) -> &Self::Target {
564 &self.buf_view
565 }
566}
567
568impl<'d, BUF, Dm> DerefMut for Mem2MemTxTransfer<'d, BUF, Dm>
569where
570 BUF: DmaTxBuffer,
571 Dm: DriverMode,
572{
573 fn deref_mut(&mut self) -> &mut Self::Target {
574 &mut self.buf_view
575 }
576}
577
578impl<'d, BUF, Dm> Drop for Mem2MemTxTransfer<'d, BUF, Dm>
579where
580 BUF: DmaTxBuffer,
581 Dm: DriverMode,
582{
583 fn drop(&mut self) {
584 self.m2m.channel.stop_transfer();
585
586 let view = unsafe {
589 ManuallyDrop::drop(&mut self.m2m);
590 ManuallyDrop::take(&mut self.buf_view)
591 };
592 let _ = BUF::from_view(view);
593 }
594}
595
596pub struct SimpleMem2Mem<'d, Dm>
599where
600 Dm: DriverMode,
601{
602 state: State<'d, Dm>,
603 config: BurstConfig,
604}
605
606enum State<'d, Dm: DriverMode> {
607 Idle(
608 Mem2Mem<'d, Dm>,
609 DmaAlignedMut<'d, [DmaDescriptor]>,
610 DmaAlignedMut<'d, [DmaDescriptor]>,
611 ),
612 Active(
613 Mem2MemRxTransfer<'d, DmaRxBuf, Dm>,
614 Mem2MemTxTransfer<'d, DmaTxBuf, Dm>,
615 ),
616 InUse,
617}
618
619impl<'d, Dm> SimpleMem2Mem<'d, Dm>
620where
621 Dm: DriverMode,
622{
623 pub fn new(
625 mem2mem: Mem2Mem<'d, Dm>,
626 rx_descriptors: &'d mut [DmaDescriptor],
627 tx_descriptors: &'d mut [DmaDescriptor],
628 config: BurstConfig,
629 ) -> Result<Self, DmaError> {
630 if rx_descriptors.is_empty() || tx_descriptors.is_empty() {
631 return Err(DmaError::OutOfDescriptors);
632 }
633
634 let rx_descriptors = unsafe { DmaAlignedMut::new_unchecked(rx_descriptors) };
637 let tx_descriptors = unsafe { DmaAlignedMut::new_unchecked(tx_descriptors) };
638
639 Ok(Self {
640 state: State::Idle(mem2mem, rx_descriptors, tx_descriptors),
641 config,
642 })
643 }
644}
645
646impl<'d, Dm> SimpleMem2Mem<'d, Dm>
647where
648 Dm: DriverMode,
649{
650 pub fn start_transfer(
652 &mut self,
653 rx_buffer: &mut [u8],
654 tx_buffer: &[u8],
655 ) -> Result<SimpleMem2MemTransfer<'_, 'd, Dm>, DmaError> {
656 let State::Idle(mem2mem, mut rx_descriptors, mut tx_descriptors) =
657 core::mem::replace(&mut self.state, State::InUse)
658 else {
659 panic!("SimpleMem2MemTransfer was forgotten with core::mem::forget or similar");
660 };
661
662 let rx_buffer = DmaAlignedMut::new(unsafe {
667 core::slice::from_raw_parts_mut(rx_buffer.as_mut_ptr(), rx_buffer.len())
668 })?;
669 let tx_buffer = unsafe {
670 DmaAlignedMut::new_unchecked(core::slice::from_raw_parts_mut(
671 tx_buffer.as_ptr() as _,
672 tx_buffer.len(),
673 ))
674 };
675 let rx_descriptors = unsafe {
676 DmaAlignedMut::new_unchecked(core::slice::from_raw_parts_mut(
677 rx_descriptors.as_mut_ptr(),
678 rx_descriptors.len(),
679 ))
680 };
681 let tx_descriptors = unsafe {
682 DmaAlignedMut::new_unchecked(core::slice::from_raw_parts_mut(
683 tx_descriptors.as_mut_ptr(),
684 tx_descriptors.len(),
685 ))
686 };
687
688 let dma_rx_buf = unwrap!(
692 DmaRxBuf::new_with_config(rx_descriptors, rx_buffer, self.config),
693 "There's no way to get the descriptors back yet"
694 );
695
696 let rx = match mem2mem.rx.receive(dma_rx_buf) {
697 Ok(rx) => rx,
698 Err((err, rx, buf)) => {
699 let (rx_descriptors, _rx_buffer) = buf.split();
700 self.state = State::Idle(
701 Mem2Mem { rx, tx: mem2mem.tx },
702 rx_descriptors,
703 tx_descriptors,
704 );
705 return Err(err);
706 }
707 };
708
709 let dma_tx_buf = unwrap!(
710 DmaTxBuf::new_with_config(tx_descriptors, tx_buffer, self.config),
711 "There's no way to get the descriptors back yet"
712 );
713
714 let tx = match mem2mem.tx.send(dma_tx_buf) {
715 Ok(tx) => tx,
716 Err((err, tx, buf)) => {
717 let (tx_descriptors, _tx_buffer) = buf.split();
718 let (rx, buf) = rx.stop();
719 let (rx_descriptors, _rx_buffer) = buf.split();
720 self.state = State::Idle(Mem2Mem { rx, tx }, rx_descriptors, tx_descriptors);
721 return Err(err);
722 }
723 };
724
725 self.state = State::Active(rx, tx);
726
727 Ok(SimpleMem2MemTransfer(self))
728 }
729}
730
731impl<Dm> Drop for SimpleMem2Mem<'_, Dm>
732where
733 Dm: DriverMode,
734{
735 fn drop(&mut self) {
736 if !matches!(&mut self.state, State::Idle(_, _, _)) {
737 panic!("SimpleMem2MemTransfer was forgotten with core::mem::forget or similar");
738 }
739 }
740}
741
742pub struct SimpleMem2MemTransfer<'a, 'd, Dm>(&'a mut SimpleMem2Mem<'d, Dm>)
745where
746 Dm: DriverMode;
747
748impl<Dm> SimpleMem2MemTransfer<'_, '_, Dm>
749where
750 Dm: DriverMode,
751{
752 pub fn is_done(&self) -> bool {
754 let State::Active(rx, tx) = &self.0.state else {
755 unreachable!()
756 };
757
758 tx.is_done()
761 && rx
762 .m2m
763 .channel
764 .pending_in_interrupts()
765 .contains(DmaRxInterrupt::SuccessfulEof)
766 }
767
768 pub fn wait(self) -> Result<(), DmaError> {
770 while !self.is_done() {}
771 Ok(())
772 }
773}
774
775impl<Dm> Drop for SimpleMem2MemTransfer<'_, '_, Dm>
776where
777 Dm: DriverMode,
778{
779 fn drop(&mut self) {
780 let State::Active(rx, tx) = core::mem::replace(&mut self.0.state, State::InUse) else {
781 unreachable!()
782 };
783
784 let (tx, dma_tx_buf) = tx.stop();
785 let (rx, dma_rx_buf) = rx.stop();
786
787 let (tx_descriptors, _tx_buffer) = dma_tx_buf.split();
788 let (rx_descriptors, _rx_buffer) = dma_rx_buf.split();
789
790 self.0.state = State::Idle(Mem2Mem { rx, tx }, rx_descriptors, tx_descriptors);
791 }
792}