1use core::{
2 mem::ManuallyDrop,
3 ops::{Deref, DerefMut},
4};
5
6use enumset::EnumSet;
7
8#[cfg(dma_mem2mem_requires_peripheral)]
9use crate::dma::DmaEligiblePeripheral;
10use crate::{
11 Async,
12 Blocking,
13 DriverMode,
14 dma::{
15 BurstConfig,
16 Channel,
17 ChannelRx,
18 ChannelTx,
19 DmaChannel,
20 DmaDescriptor,
21 DmaError,
22 DmaPeripheral,
23 DmaRxBuf,
24 DmaRxBuffer,
25 DmaRxInterrupt,
26 DmaTxBuf,
27 DmaTxBuffer,
28 DmaTxInterrupt,
29 aligned::DmaAlignedMut,
30 },
31};
32
33#[diagnostic::on_unimplemented(
38 message = "this DMA channel does not support memory-to-memory transfers",
39 note = "Use a channel with `mem2mem = true` in device metadata. See `Mem2Mem::new`."
40)]
41pub trait Mem2MemCapableChannel<'d>: DmaChannel {
42 #[doc(hidden)]
43 type Erased: DmaChannel + From<Self>;
44
45 #[cfg(not(dma_mem2mem_requires_peripheral))]
47 fn mem2mem_id(&self) -> DmaPeripheral;
48
49 #[doc(hidden)]
50 #[allow(private_interfaces)]
51 fn into_channel(self) -> ErasedChannel<'d, Blocking>;
52}
53
54for_each_mem2mem_channel! {
56 (engines $( ($engine:literal, $variant:ident, $any_ch:ident) ),* ) => {
57 struct ErasedChannel<'d, Dm: DriverMode> {
58 rx: ErasedChannelRx<'d, Dm>,
59 tx: ErasedChannelTx<'d, Dm>,
60 }
61
62 enum ErasedChannelRx<'d, Dm: DriverMode> {
63 $(
64 $variant(ChannelRx<Dm, <crate::dma::$any_ch<'d> as DmaChannel>::Rx>),
65 )*
66 }
67
68 impl<Dm: DriverMode> ErasedChannelRx<'_, Dm> {
69 delegate::delegate! {
70 to match self {
71 $( Self::$variant(channel) => channel, )*
72 } {
73 fn has_error(&self) -> bool;
74 fn pending_in_interrupts(&self) -> EnumSet<DmaRxInterrupt>;
75 #[cfg(dma_mem2mem_requires_peripheral)]
76 fn runtime_ensure_compatible(&self, peripheral: DmaPeripheral);
77 }
78
79 to match self {
80 $( Self::$variant(channel) => channel, )*
81 } {
82 fn set_mem2mem_mode(&mut self, value: bool);
83 fn start_transfer(&mut self) -> Result<(), DmaError>;
84 fn stop_transfer(&mut self);
85 unsafe fn prepare_transfer<BUF: DmaRxBuffer>(
86 &mut self,
87 peri: DmaPeripheral,
88 buffer: &mut BUF,
89 ) -> Result<(), DmaError>;
90 }
91 }
92 }
93
94 impl<'d> ErasedChannelRx<'d, Blocking> {
95 fn into_async(self) -> ErasedChannelRx<'d, Async> {
96 match self {
97 $( Self::$variant(channel) => ErasedChannelRx::$variant(channel.into_async()), )*
98 }
99 }
100 }
101
102 impl<'d> ErasedChannelRx<'d, Async> {
103 fn into_blocking(self) -> ErasedChannelRx<'d, Blocking> {
104 match self {
105 $( Self::$variant(channel) => ErasedChannelRx::$variant(channel.into_blocking()), )*
106 }
107 }
108 }
109
110 enum ErasedChannelTx<'d, Dm: DriverMode> {
111 $(
112 $variant(ChannelTx<Dm, <crate::dma::$any_ch<'d> as DmaChannel>::Tx>),
113 )*
114 }
115
116 impl<Dm: DriverMode> ErasedChannelTx<'_, Dm> {
117 delegate::delegate! {
118 to match self {
119 $( Self::$variant(channel) => channel, )*
120 } {
121 fn has_error(&self) -> bool;
122 fn pending_out_interrupts(&self) -> EnumSet<DmaTxInterrupt>;
123 }
124
125 to match self {
126 $( Self::$variant(channel) => channel, )*
127 } {
128 fn start_transfer(&mut self) -> Result<(), DmaError>;
129 fn stop_transfer(&mut self);
130 unsafe fn prepare_transfer<BUF: DmaTxBuffer>(
131 &mut self,
132 peri: DmaPeripheral,
133 buffer: &mut BUF,
134 ) -> Result<(), DmaError>;
135 }
136 }
137 }
138
139 impl<'d> ErasedChannelTx<'d, Blocking> {
140 fn into_async(self) -> ErasedChannelTx<'d, Async> {
141 match self {
142 $( Self::$variant(channel) => ErasedChannelTx::$variant(channel.into_async()), )*
143 }
144 }
145 }
146
147 impl<'d> ErasedChannelTx<'d, Async> {
148 fn into_blocking(self) -> ErasedChannelTx<'d, Blocking> {
149 match self {
150 $( Self::$variant(channel) => ErasedChannelTx::$variant(channel.into_blocking()), )*
151 }
152 }
153 }
154 };
155}
156
157for_each_mem2mem_channel! {
158 ($engine:literal, $variant:ident, $any_ch:ident, $($hw:literal, $id:literal),+) => {
159 impl<'d> Mem2MemCapableChannel<'d> for crate::dma::$any_ch<'d> {
160 type Erased = Self;
161
162 #[cfg(not(dma_mem2mem_requires_peripheral))]
163 fn mem2mem_id(&self) -> DmaPeripheral {
164 match self.channel_index() {
165 $( $hw => DmaPeripheral($id), )+
166 ch => panic!(
167 "Channel {} does not support memory-to-memory transfers",
168 ch
169 ),
170 }
171 }
172
173 fn into_channel(self) -> ErasedChannel<'d, Blocking> {
174 let channel = Channel::new(self);
175 ErasedChannel {
176 rx: ErasedChannelRx::$variant(channel.rx),
177 tx: ErasedChannelTx::$variant(channel.tx),
178 }
179 }
180 }
181 };
182 ($engine:literal, $variant:ident, $any_ch:ident, $ch:ident, $id:literal) => {
183 impl<'d> Mem2MemCapableChannel<'d> for crate::peripherals::$ch<'d> {
184 type Erased = crate::dma::$any_ch<'d>;
185
186 #[cfg(not(dma_mem2mem_requires_peripheral))]
187 fn mem2mem_id(&self) -> DmaPeripheral {
188 DmaPeripheral($id)
189 }
190
191 fn into_channel(self) -> ErasedChannel<'d, Blocking> {
192 let channel = Channel::new(crate::dma::$any_ch::from(self));
193 ErasedChannel {
194 rx: ErasedChannelRx::$variant(channel.rx),
195 tx: ErasedChannelTx::$variant(channel.tx),
196 }
197 }
198 }
199 };
200}
201
202pub struct Mem2Mem<'d, Dm>
208where
209 Dm: DriverMode,
210{
211 pub rx: Mem2MemRx<'d, Dm>,
213 pub tx: Mem2MemTx<'d, Dm>,
215}
216
217impl<'d> Mem2Mem<'d, Blocking> {
218 pub fn new<CH>(
220 channel: CH,
221 #[cfg(dma_mem2mem_requires_peripheral)] peripheral: impl DmaEligiblePeripheral<CH::Erased>,
222 ) -> Self
223 where
224 CH: Mem2MemCapableChannel<'d>,
225 {
226 let dma_peri = cfg_select! {
227 dma_mem2mem_requires_peripheral => peripheral.dma_peripheral(),
228 _ => channel.mem2mem_id(),
229 };
230 Self::new_inner(channel, dma_peri)
231 }
232
233 #[cfg(dma_mem2mem_requires_peripheral)]
241 pub unsafe fn new_unsafe<CH>(channel: CH, peripheral: DmaPeripheral) -> Self
242 where
243 CH: Mem2MemCapableChannel<'d>,
244 {
245 Self::new_inner(channel, peripheral)
246 }
247
248 pub fn into_async(self) -> Mem2Mem<'d, Async> {
250 Mem2Mem {
251 rx: self.rx.into_async(),
252 tx: self.tx.into_async(),
253 }
254 }
255}
256
257impl<'d> Mem2Mem<'d, Blocking> {
258 fn new_inner(channel: impl Mem2MemCapableChannel<'d>, peripheral: DmaPeripheral) -> Self {
259 let mut channel = channel.into_channel();
260
261 #[cfg(dma_mem2mem_requires_peripheral)]
262 channel.rx.runtime_ensure_compatible(peripheral);
263
264 #[cfg(dma_supports_mem2mem)]
265 channel.rx.set_mem2mem_mode(true);
266
267 Mem2Mem {
268 rx: Mem2MemRx {
269 channel: channel.rx,
270 peripheral,
271 },
272 tx: Mem2MemTx {
273 channel: channel.tx,
274 peripheral,
275 },
276 }
277 }
278
279 pub fn with_descriptors(
281 self,
282 rx_descriptors: &'d mut [DmaDescriptor],
283 tx_descriptors: &'d mut [DmaDescriptor],
284 config: BurstConfig,
285 ) -> Result<SimpleMem2Mem<'d, Blocking>, DmaError> {
286 SimpleMem2Mem::new(self, rx_descriptors, tx_descriptors, config)
287 }
288}
289
290pub struct Mem2MemRx<'d, Dm>
292where
293 Dm: DriverMode,
294{
295 channel: ErasedChannelRx<'d, Dm>,
296 peripheral: DmaPeripheral,
297}
298
299impl<'d> Mem2MemRx<'d, Blocking> {
300 pub fn into_async(self) -> Mem2MemRx<'d, Async> {
302 Mem2MemRx {
303 channel: self.channel.into_async(),
304 peripheral: self.peripheral,
305 }
306 }
307}
308
309impl<'d, Dm> Mem2MemRx<'d, Dm>
310where
311 Dm: DriverMode,
312{
313 pub fn receive<BUF>(
315 mut self,
316 mut buf: BUF,
317 ) -> Result<Mem2MemRxTransfer<'d, BUF, Dm>, (DmaError, Self, BUF)>
318 where
319 BUF: DmaRxBuffer,
320 {
321 let result = unsafe {
322 self.channel
323 .prepare_transfer(self.peripheral, &mut buf)
324 .and_then(|_| self.channel.start_transfer())
325 };
326
327 if let Err(e) = result {
328 return Err((e, self, buf));
329 }
330
331 Ok(Mem2MemRxTransfer {
332 m2m: ManuallyDrop::new(self),
333 buf_view: ManuallyDrop::new(buf.into_view()),
334 })
335 }
336}
337
338pub struct Mem2MemRxTransfer<'d, BUF, Dm>
341where
342 BUF: DmaRxBuffer,
343 Dm: DriverMode,
344{
345 m2m: ManuallyDrop<Mem2MemRx<'d, Dm>>,
346 buf_view: ManuallyDrop<BUF::View>,
347}
348
349impl<'d, BUF, Dm> Mem2MemRxTransfer<'d, BUF, Dm>
350where
351 BUF: DmaRxBuffer,
352 Dm: DriverMode,
353{
354 pub fn is_done(&self) -> bool {
356 let done_interrupts = DmaRxInterrupt::DescriptorError | DmaRxInterrupt::DescriptorEmpty;
357 !self
358 .m2m
359 .channel
360 .pending_in_interrupts()
361 .is_disjoint(done_interrupts)
362 }
363
364 pub fn wait(self) -> (Result<(), DmaError>, Mem2MemRx<'d, Dm>, BUF::Final) {
366 while !self.is_done() {}
367
368 let (m2m, view) = self.release();
369
370 let result = if m2m.channel.has_error() {
371 Err(DmaError::DescriptorError)
372 } else {
373 Ok(())
374 };
375
376 (result, m2m, BUF::from_view(view))
377 }
378
379 pub fn stop(self) -> (Mem2MemRx<'d, Dm>, BUF::Final) {
381 let (mut m2m, view) = self.release();
382
383 m2m.channel.stop_transfer();
384
385 (m2m, BUF::from_view(view))
386 }
387
388 fn release(mut self) -> (Mem2MemRx<'d, Dm>, BUF::View) {
389 let result = unsafe {
392 let m2m = ManuallyDrop::take(&mut self.m2m);
393 let view = ManuallyDrop::take(&mut self.buf_view);
394 (m2m, view)
395 };
396 core::mem::forget(self);
397 result
398 }
399}
400
401impl<'d, BUF, Dm> Deref for Mem2MemRxTransfer<'d, BUF, Dm>
402where
403 BUF: DmaRxBuffer,
404 Dm: DriverMode,
405{
406 type Target = BUF::View;
407
408 fn deref(&self) -> &Self::Target {
409 &self.buf_view
410 }
411}
412
413impl<'d, BUF, Dm> DerefMut for Mem2MemRxTransfer<'d, BUF, Dm>
414where
415 BUF: DmaRxBuffer,
416 Dm: DriverMode,
417{
418 fn deref_mut(&mut self) -> &mut Self::Target {
419 &mut self.buf_view
420 }
421}
422
423impl<'d, BUF, Dm> Drop for Mem2MemRxTransfer<'d, BUF, Dm>
424where
425 BUF: DmaRxBuffer,
426 Dm: DriverMode,
427{
428 fn drop(&mut self) {
429 self.m2m.channel.stop_transfer();
430
431 let view = unsafe {
434 ManuallyDrop::drop(&mut self.m2m);
435 ManuallyDrop::take(&mut self.buf_view)
436 };
437 let _ = BUF::from_view(view);
438 }
439}
440
441pub struct Mem2MemTx<'d, Dm>
443where
444 Dm: DriverMode,
445{
446 channel: ErasedChannelTx<'d, Dm>,
447 peripheral: DmaPeripheral,
448}
449
450impl<'d> Mem2MemTx<'d, Blocking> {
451 pub fn into_async(self) -> Mem2MemTx<'d, Async> {
453 Mem2MemTx {
454 channel: self.channel.into_async(),
455 peripheral: self.peripheral,
456 }
457 }
458}
459
460impl<'d, Dm> Mem2MemTx<'d, Dm>
461where
462 Dm: DriverMode,
463{
464 pub fn send<BUF>(
466 mut self,
467 mut buf: BUF,
468 ) -> Result<Mem2MemTxTransfer<'d, BUF, Dm>, (DmaError, Self, BUF)>
469 where
470 BUF: DmaTxBuffer,
471 {
472 let result = unsafe {
473 self.channel
474 .prepare_transfer(self.peripheral, &mut buf)
475 .and_then(|_| self.channel.start_transfer())
476 };
477
478 if let Err(e) = result {
479 return Err((e, self, buf));
480 }
481
482 Ok(Mem2MemTxTransfer {
483 m2m: ManuallyDrop::new(self),
484 buf_view: ManuallyDrop::new(buf.into_view()),
485 })
486 }
487}
488
489pub struct Mem2MemTxTransfer<'d, BUF, Dm>
492where
493 BUF: DmaTxBuffer,
494 Dm: DriverMode,
495{
496 m2m: ManuallyDrop<Mem2MemTx<'d, Dm>>,
497 buf_view: ManuallyDrop<BUF::View>,
498}
499
500impl<'d, BUF, Dm> Mem2MemTxTransfer<'d, BUF, Dm>
501where
502 BUF: DmaTxBuffer,
503 Dm: DriverMode,
504{
505 pub fn is_done(&self) -> bool {
507 let done_interrupts = DmaTxInterrupt::DescriptorError | DmaTxInterrupt::TotalEof;
508 !self
509 .m2m
510 .channel
511 .pending_out_interrupts()
512 .is_disjoint(done_interrupts)
513 }
514
515 pub fn wait(self) -> (Result<(), DmaError>, Mem2MemTx<'d, Dm>, BUF::Final) {
517 while !self.is_done() {}
518
519 let (m2m, view) = self.release();
520
521 let result = if m2m.channel.has_error() {
522 Err(DmaError::DescriptorError)
523 } else {
524 Ok(())
525 };
526
527 (result, m2m, BUF::from_view(view))
528 }
529
530 pub fn stop(self) -> (Mem2MemTx<'d, Dm>, BUF::Final) {
532 let (mut m2m, view) = self.release();
533
534 m2m.channel.stop_transfer();
535
536 (m2m, BUF::from_view(view))
537 }
538
539 fn release(mut self) -> (Mem2MemTx<'d, Dm>, BUF::View) {
540 let result = unsafe {
543 let m2m = ManuallyDrop::take(&mut self.m2m);
544 let view = ManuallyDrop::take(&mut self.buf_view);
545 (m2m, view)
546 };
547 core::mem::forget(self);
548 result
549 }
550}
551
552impl<'d, BUF, Dm> Deref for Mem2MemTxTransfer<'d, BUF, Dm>
553where
554 BUF: DmaTxBuffer,
555 Dm: DriverMode,
556{
557 type Target = BUF::View;
558
559 fn deref(&self) -> &Self::Target {
560 &self.buf_view
561 }
562}
563
564impl<'d, BUF, Dm> DerefMut for Mem2MemTxTransfer<'d, BUF, Dm>
565where
566 BUF: DmaTxBuffer,
567 Dm: DriverMode,
568{
569 fn deref_mut(&mut self) -> &mut Self::Target {
570 &mut self.buf_view
571 }
572}
573
574impl<'d, BUF, Dm> Drop for Mem2MemTxTransfer<'d, BUF, Dm>
575where
576 BUF: DmaTxBuffer,
577 Dm: DriverMode,
578{
579 fn drop(&mut self) {
580 self.m2m.channel.stop_transfer();
581
582 let view = unsafe {
585 ManuallyDrop::drop(&mut self.m2m);
586 ManuallyDrop::take(&mut self.buf_view)
587 };
588 let _ = BUF::from_view(view);
589 }
590}
591
592pub struct SimpleMem2Mem<'d, Dm>
595where
596 Dm: DriverMode,
597{
598 state: State<'d, Dm>,
599 config: BurstConfig,
600}
601
602enum State<'d, Dm: DriverMode> {
603 Idle(
604 Mem2Mem<'d, Dm>,
605 DmaAlignedMut<'d, [DmaDescriptor]>,
606 DmaAlignedMut<'d, [DmaDescriptor]>,
607 ),
608 Active(
609 Mem2MemRxTransfer<'d, DmaRxBuf, Dm>,
610 Mem2MemTxTransfer<'d, DmaTxBuf, Dm>,
611 ),
612 InUse,
613}
614
615impl<'d, Dm> SimpleMem2Mem<'d, Dm>
616where
617 Dm: DriverMode,
618{
619 pub fn new(
621 mem2mem: Mem2Mem<'d, Dm>,
622 rx_descriptors: &'d mut [DmaDescriptor],
623 tx_descriptors: &'d mut [DmaDescriptor],
624 config: BurstConfig,
625 ) -> Result<Self, DmaError> {
626 if rx_descriptors.is_empty() || tx_descriptors.is_empty() {
627 return Err(DmaError::OutOfDescriptors);
628 }
629
630 let rx_descriptors = unsafe { DmaAlignedMut::new_unchecked(rx_descriptors) };
633 let tx_descriptors = unsafe { DmaAlignedMut::new_unchecked(tx_descriptors) };
634
635 Ok(Self {
636 state: State::Idle(mem2mem, rx_descriptors, tx_descriptors),
637 config,
638 })
639 }
640}
641
642impl<'d, Dm> SimpleMem2Mem<'d, Dm>
643where
644 Dm: DriverMode,
645{
646 pub fn start_transfer(
648 &mut self,
649 rx_buffer: &mut [u8],
650 tx_buffer: &[u8],
651 ) -> Result<SimpleMem2MemTransfer<'_, 'd, Dm>, DmaError> {
652 let State::Idle(mem2mem, mut rx_descriptors, mut tx_descriptors) =
653 core::mem::replace(&mut self.state, State::InUse)
654 else {
655 panic!("SimpleMem2MemTransfer was forgotten with core::mem::forget or similar");
656 };
657
658 let rx_buffer = DmaAlignedMut::new(unsafe {
663 core::slice::from_raw_parts_mut(rx_buffer.as_mut_ptr(), rx_buffer.len())
664 })?;
665 let tx_buffer = unsafe {
666 DmaAlignedMut::new_unchecked(core::slice::from_raw_parts_mut(
667 tx_buffer.as_ptr() as _,
668 tx_buffer.len(),
669 ))
670 };
671 let rx_descriptors = unsafe {
672 DmaAlignedMut::new_unchecked(core::slice::from_raw_parts_mut(
673 rx_descriptors.as_mut_ptr(),
674 rx_descriptors.len(),
675 ))
676 };
677 let tx_descriptors = unsafe {
678 DmaAlignedMut::new_unchecked(core::slice::from_raw_parts_mut(
679 tx_descriptors.as_mut_ptr(),
680 tx_descriptors.len(),
681 ))
682 };
683
684 let dma_rx_buf = unwrap!(
688 DmaRxBuf::new_with_config(rx_descriptors, rx_buffer, self.config),
689 "There's no way to get the descriptors back yet"
690 );
691
692 let rx = match mem2mem.rx.receive(dma_rx_buf) {
693 Ok(rx) => rx,
694 Err((err, rx, buf)) => {
695 let (rx_descriptors, _rx_buffer) = buf.split();
696 self.state = State::Idle(
697 Mem2Mem { rx, tx: mem2mem.tx },
698 rx_descriptors,
699 tx_descriptors,
700 );
701 return Err(err);
702 }
703 };
704
705 let dma_tx_buf = unwrap!(
706 DmaTxBuf::new_with_config(tx_descriptors, tx_buffer, self.config),
707 "There's no way to get the descriptors back yet"
708 );
709
710 let tx = match mem2mem.tx.send(dma_tx_buf) {
711 Ok(tx) => tx,
712 Err((err, tx, buf)) => {
713 let (tx_descriptors, _tx_buffer) = buf.split();
714 let (rx, buf) = rx.stop();
715 let (rx_descriptors, _rx_buffer) = buf.split();
716 self.state = State::Idle(Mem2Mem { rx, tx }, rx_descriptors, tx_descriptors);
717 return Err(err);
718 }
719 };
720
721 self.state = State::Active(rx, tx);
722
723 Ok(SimpleMem2MemTransfer(self))
724 }
725}
726
727impl<Dm> Drop for SimpleMem2Mem<'_, Dm>
728where
729 Dm: DriverMode,
730{
731 fn drop(&mut self) {
732 if !matches!(&mut self.state, State::Idle(_, _, _)) {
733 panic!("SimpleMem2MemTransfer was forgotten with core::mem::forget or similar");
734 }
735 }
736}
737
738pub struct SimpleMem2MemTransfer<'a, 'd, Dm>(&'a mut SimpleMem2Mem<'d, Dm>)
741where
742 Dm: DriverMode;
743
744impl<Dm> SimpleMem2MemTransfer<'_, '_, Dm>
745where
746 Dm: DriverMode,
747{
748 pub fn is_done(&self) -> bool {
750 let State::Active(rx, tx) = &self.0.state else {
751 unreachable!()
752 };
753
754 tx.is_done()
757 && rx
758 .m2m
759 .channel
760 .pending_in_interrupts()
761 .contains(DmaRxInterrupt::SuccessfulEof)
762 }
763
764 pub fn wait(self) -> Result<(), DmaError> {
766 while !self.is_done() {}
767 Ok(())
768 }
769}
770
771impl<Dm> Drop for SimpleMem2MemTransfer<'_, '_, Dm>
772where
773 Dm: DriverMode,
774{
775 fn drop(&mut self) {
776 let State::Active(rx, tx) = core::mem::replace(&mut self.0.state, State::InUse) else {
777 unreachable!()
778 };
779
780 let (tx, dma_tx_buf) = tx.stop();
781 let (rx, dma_rx_buf) = rx.stop();
782
783 let (tx_descriptors, _tx_buffer) = dma_tx_buf.split();
784 let (rx_descriptors, _rx_buffer) = dma_rx_buf.split();
785
786 self.0.state = State::Idle(Mem2Mem { rx, tx }, rx_descriptors, tx_descriptors);
787 }
788}