Skip to main content

esp_hal/dma/
aligned.rs

1//! Helper types for DMA buffers.
2
3use core::ops::{Deref, DerefMut};
4
5use procmacros::doc_replace;
6
7#[cfg(dma_can_access_psram)]
8use crate::soc::is_valid_psram_address;
9use crate::{
10    dma::{DmaAlignmentError, DmaBufError},
11    soc::{is_uncached_ram_address, is_valid_ram_address},
12};
13
14/// DMA appropriate wrapper type for internal memory values.
15///
16/// The value wrapped in this type is guaranteed to be safely useable
17/// as a DMA buffer or descriptor array, meaning DMA or cache management
18/// operations will not corrupt surrounding data.
19///
20/// [`DmaAlignedMut`] is a reference type that carries this guarantee.
21// ESP32-P4 internal memory is cached, enforce alignment to avoid memory
22// corruption. Technically only needed for IN buffers and descriptor lists.
23#[derive(Debug)]
24#[cfg_attr(feature = "defmt", derive(defmt::Format))]
25#[cfg_attr(soc_internal_memory_cached, repr(C, align(64)))] // dcache cache line
26#[cfg_attr(not(soc_internal_memory_cached), repr(C, align(4)))] // Worst-case word alignment
27#[instability::unstable]
28pub struct InternalMemory<T>(T);
29
30impl<T> InternalMemory<T> {
31    /// Creates a new value aligned for DMA operations in internal memory.
32    #[instability::unstable]
33    pub const fn new(init: T) -> Self {
34        Self(init)
35    }
36
37    /// Returns a const pointer to the wrapped value.
38    ///
39    /// The returned pointer is aligned appropriately for DMA and points at the
40    /// same address that the DMA engine will access.
41    #[instability::unstable]
42    pub const fn as_ptr(&self) -> *const T {
43        &raw const self.0
44    }
45
46    /// Returns a [`DmaAlignedMut`] to the underlying value.
47    ///
48    /// # Panics
49    ///
50    /// Panics if the value is not located at a valid internal RAM address.
51    #[instability::unstable]
52    pub fn get_mut(&mut self) -> DmaAlignedMut<'_, T> {
53        assert!(is_valid_ram_address(&raw const self.0 as usize));
54        DmaAlignedMut(&mut self.0)
55    }
56
57    /// Returns a [`DmaAlignedRef`] to the underlying value.
58    ///
59    /// The reference can be used to read the value and to invalidate its cache
60    /// lines, but not to mutate or write it back. This allows polling
61    /// DMA-written state (e.g. a descriptor ownership bit) through a shared
62    /// borrow.
63    ///
64    /// # Panics
65    ///
66    /// Panics if the value is not located at a valid internal RAM address.
67    #[instability::unstable]
68    pub fn get_ref(&self) -> DmaAlignedRef<'_, T> {
69        assert!(is_valid_ram_address(&raw const self.0 as usize));
70        DmaAlignedRef(&self.0)
71    }
72}
73
74/// The length of the data cache line in front of PSRAM.
75#[cfg(dma_can_access_psram)]
76pub(crate) const PSRAM_CACHE_LINE: usize = cfg_select! {
77    // TODO(esp32p4): PSRAM is cached through the L2 cache,
78    // whose line size is configurable (64 or 128
79    // bytes) and is not encoded anywhere yet. Assume the
80    // larger, always-safe value until the L2 line size is
81    // available.
82    soc_internal_memory_cached => 128,
83    esp32s31 => 64,
84    any(esp32, esp32c5, esp32c61) => 32, // TODO: fixed 32-bytes, metadata-ify
85    _ => crate::soc::CONFIG_DATA_CACHE_LINE_SIZE,
86};
87
88/// Returns the DMA alignment (applying to both the start address and the
89/// length) required by the memory region containing `addr`, or `None` if
90/// `addr` is not in a DMA-capable region.
91///
92/// Both the address and size of a DMA buffer must be a multiple of this value
93/// so that cache maintenance on the buffer cannot corrupt neighboring data.
94pub(crate) fn region_dma_alignment(addr: usize) -> Option<usize> {
95    // Uncached memory has no cache lines. The DMA burst alignment check is separate.
96    if is_uncached_ram_address(addr) {
97        return Some(1);
98    }
99
100    if is_valid_ram_address(addr) {
101        return Some(core::mem::align_of::<InternalMemory<()>>());
102    }
103
104    #[cfg(dma_can_access_psram)]
105    if is_valid_psram_address(addr) {
106        return Some(PSRAM_CACHE_LINE);
107    }
108
109    None
110}
111
112/// Validates that the value at `addr` spanning `size` bytes lives in a
113/// DMA-capable memory region and is aligned correctly for that region.
114fn validate_dma_alignment(addr: usize, size: usize) -> Result<(), DmaBufError> {
115    // Zero-sized values never alias a cache line, so any address is fine.
116    if size == 0 {
117        return Ok(());
118    }
119
120    let Some(alignment) = region_dma_alignment(addr) else {
121        return Err(DmaBufError::UnsupportedMemoryRegion);
122    };
123
124    if !size.is_multiple_of(alignment) {
125        return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Size));
126    }
127    if !addr.is_multiple_of(alignment) {
128        return Err(DmaBufError::InvalidAlignment(DmaAlignmentError::Address));
129    }
130
131    Ok(())
132}
133
134/// Returns whether the value at `addr` spanning `size` bytes occupies a
135/// cached memory region and therefore needs explicit cache maintenance.
136#[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
137pub(crate) fn region_needs_cache_op(addr: usize, size: usize) -> bool {
138    // Do not do cache operations on zero-sized values.
139    if size == 0 {
140        return false;
141    }
142
143    let mut in_cached_region = false;
144
145    #[cfg(soc_internal_memory_cached)]
146    {
147        in_cached_region |= is_valid_ram_address(addr);
148    }
149    #[cfg(dma_can_access_psram)]
150    {
151        in_cached_region |= is_valid_psram_address(addr);
152    }
153
154    in_cached_region
155}
156
157/// A mutable reference to an [`InternalMemory`] object.
158#[derive(Debug)]
159#[cfg_attr(feature = "defmt", derive(defmt::Format))]
160#[instability::unstable]
161pub struct DmaAlignedMut<'a, T>(&'a mut T)
162where
163    T: ?Sized;
164
165impl<'a, T: ?Sized> DmaAlignedMut<'a, T> {
166    #[doc_replace(
167        "align_req" => {
168            cfg(soc_internal_memory_cached) => "64",
169            _ => "4"
170        },
171        "uncached_note" => {
172            cfg(esp32p4) => "This does not apply to `#[ram(unstable(uncached))]` variables.",
173            _ => ""
174        }
175    )]
176    /// Creates a new [`DmaAlignedMut`] from a mutable variable, if it is
177    /// provably compatible.
178    ///
179    /// In internal memory, the address and size of the variable
180    /// must be at least __align_req__ byte aligned. __uncached_note__
181    #[instability::unstable]
182    pub fn new(ptr: &'a mut T) -> Result<Self, DmaBufError> {
183        let addr = ptr as *mut T as *mut () as usize;
184        validate_dma_alignment(addr, core::mem::size_of_val(ptr))?;
185        Ok(Self(ptr))
186    }
187
188    /// Creates a new [`DmaAlignedMut`] from *any* mutable slice.
189    ///
190    /// # Safety
191    ///
192    /// The caller must ensure that the reference is properly aligned for the memory region it
193    /// occupies and the cachelines occupied by the reference do not overlap with any other data
194    /// that can be corrupted by a cacheline invalidation operation.
195    #[instability::unstable]
196    pub const unsafe fn new_unchecked(ptr: &'a mut T) -> Self {
197        Self(ptr)
198    }
199
200    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
201    fn do_cache_op(&self) -> bool {
202        region_needs_cache_op(
203            self.0 as *const T as *const () as usize,
204            core::mem::size_of_val(self.0),
205        )
206    }
207
208    /// Writes back the data from the cache to memory.
209    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
210    #[instability::unstable]
211    pub fn writeback(&mut self) {
212        if !self.do_cache_op() {
213            return;
214        }
215
216        // SAFETY: we own the cachelines and can't trash anything else
217        unsafe {
218            crate::soc::cache_writeback_addr(
219                self.0 as *const T as *const () as u32,
220                core::mem::size_of_val(self.0) as u32,
221            );
222        }
223    }
224
225    /// Invalidates the cache lines for this data.
226    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
227    #[instability::unstable]
228    pub fn invalidate(&self) {
229        if !self.do_cache_op() {
230            return;
231        }
232
233        // SAFETY: we own the cachelines and can't trash anything else
234        unsafe {
235            crate::soc::cache_invalidate_addr(
236                self.0 as *const T as *const () as u32,
237                core::mem::size_of_val(self.0) as u32,
238            );
239        }
240    }
241
242    /// Converts this object into a mutable reference.
243    pub fn into_inner(self) -> &'a mut T {
244        self.0
245    }
246
247    /// Reborrows this object with a different lifetime.
248    pub fn reborrow<'b>(&'b mut self) -> DmaAlignedMut<'b, T> {
249        DmaAlignedMut(self.0)
250    }
251}
252
253impl<'a, T, const N: usize> DmaAlignedMut<'a, [T; N]> {
254    /// Converts this object into a slice reference.
255    pub fn unsize(self) -> DmaAlignedMut<'a, [T]> {
256        DmaAlignedMut(self.0)
257    }
258}
259
260impl<'a, T: ?Sized> Deref for DmaAlignedMut<'a, T> {
261    type Target = T;
262
263    fn deref(&self) -> &Self::Target {
264        self.0
265    }
266}
267
268impl<'a, T: ?Sized> DerefMut for DmaAlignedMut<'a, T> {
269    fn deref_mut(&mut self) -> &mut Self::Target {
270        self.0
271    }
272}
273
274/// A shared reference to an [`InternalMemory`] object.
275///
276/// Unlike [`DmaAlignedMut`], a [`DmaAlignedRef`] only allows *reading* the
277/// value and *invalidating* its cache lines (discarding the CPU-cached copy so
278/// a subsequent read observes data written by DMA). It can neither mutate the
279/// value nor write it back, so it can be obtained from a shared borrow and used
280/// to poll DMA-written, interior-mutable state (e.g. a descriptor's ownership
281/// bit) without exclusive access.
282#[derive(Debug)]
283#[cfg_attr(feature = "defmt", derive(defmt::Format))]
284#[instability::unstable]
285pub struct DmaAlignedRef<'a, T>(&'a T)
286where
287    T: ?Sized;
288
289impl<'a, T: ?Sized> DmaAlignedRef<'a, T> {
290    #[doc_replace(
291        "align_req" => {
292            cfg(soc_internal_memory_cached) => "64",
293            _ => "4"
294        },
295        "uncached_note" => {
296            cfg(esp32p4) => "This does not apply to `#[ram(unstable(uncached))]` values.",
297            _ => ""
298        }
299    )]
300    /// Creates a new [`DmaAlignedRef`] from a shared reference, if it is
301    /// provably compatible.
302    ///
303    /// In internal memory, the address and size of the value
304    /// must be at least __align_req__ byte aligned. __uncached_note__
305    #[instability::unstable]
306    pub fn new(ptr: &'a T) -> Result<Self, DmaBufError> {
307        let addr = ptr as *const T as *const () as usize;
308        validate_dma_alignment(addr, core::mem::size_of_val(ptr))?;
309        Ok(Self(ptr))
310    }
311
312    /// Creates a new [`DmaAlignedRef`] from *any* shared reference.
313    ///
314    /// # Safety
315    ///
316    /// The caller must ensure that the reference is properly aligned for the memory region it
317    /// occupies and the cachelines occupied by the reference do not overlap with any other data
318    /// that can be corrupted by a cacheline invalidation operation.
319    #[instability::unstable]
320    pub const unsafe fn new_unchecked(ptr: &'a T) -> Self {
321        Self(ptr)
322    }
323
324    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
325    fn do_cache_op(&self) -> bool {
326        region_needs_cache_op(
327            self.0 as *const T as *const () as usize,
328            core::mem::size_of_val(self.0),
329        )
330    }
331
332    /// Invalidates the cache lines for this data.
333    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
334    #[instability::unstable]
335    pub fn invalidate(&self) {
336        if !self.do_cache_op() {
337            return;
338        }
339
340        // SAFETY: the wrapped value owns the cachelines it occupies, so
341        // invalidating them cannot discard CPU writes to neighbouring data.
342        unsafe {
343            crate::soc::cache_invalidate_addr(
344                self.0 as *const T as *const () as u32,
345                core::mem::size_of_val(self.0) as u32,
346            );
347        }
348    }
349
350    /// Writes back the cached copy of this data to memory.
351    ///
352    /// This only flushes the CPU cache to memory (so a DMA engine reading from
353    /// memory observes the latest CPU writes); it does not mutate the value, so
354    /// it is sound to perform through a shared reference.
355    #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
356    #[instability::unstable]
357    pub fn writeback(&self) {
358        if !self.do_cache_op() {
359            return;
360        }
361
362        // SAFETY: the wrapped value owns the cachelines it occupies, so writing
363        // them back cannot affect neighbouring data.
364        unsafe {
365            crate::soc::cache_writeback_addr(
366                self.0 as *const T as *const () as u32,
367                core::mem::size_of_val(self.0) as u32,
368            );
369        }
370    }
371
372    /// Converts this object into a shared reference.
373    pub fn into_inner(self) -> &'a T {
374        self.0
375    }
376}
377
378impl<'a, T: ?Sized> Deref for DmaAlignedRef<'a, T> {
379    type Target = T;
380
381    fn deref(&self) -> &Self::Target {
382        self.0
383    }
384}