Skip to main content

esp_hal/aes/dma/
gcm.rs

1//! AES-GCM on the AES-DMA backend.
2//!
3//! One operation is one DMA transfer. The input chain holds the additional data, its zero padding,
4//! the text, its zero padding and the length block. The hardware hashes them in that order.
5//! The hardware writes whole blocks, so the last partial block of output goes to a bounce block.
6//!
7//! The DMA of the ESP32-S2 fails to write a RAM descriptor that follows a PSRAM descriptor. The
8//! last bytes of an AES-GCM output are written to the bounce block in RAM, so the ESP32-S2 must
9//! refuse an output in PSRAM.
10//!
11//! See <https://github.com/espressif/esp-idf/blob/ab95e70748f2eddac6f87217b0cf62533684a6ba/components/mbedtls/port/aes/esp_aes_gcm.c#L690-L789>
12//! and <https://github.com/espressif/esp-idf/blob/ab95e70748f2eddac6f87217b0cf62533684a6ba/components/mbedtls/port/aes/dma/esp_aes_dma_core.c#L1366-L1600>.
13
14use core::{hint::black_box, mem::ManuallyDrop, ptr::NonNull};
15
16use super::{AesDma, AesDmaBackend, AesTransfer, CipherMode, DriverState};
17use crate::{
18    aes::{
19        AES_WORK_QUEUE,
20        AesOperation,
21        BLOCK_SIZE,
22        Error,
23        Key,
24        Mode,
25        Operation,
26        cipher_modes::{CryptoBuffers, UnsafeCryptoBuffers},
27        read_words,
28        write_words,
29    },
30    dma::{
31        BurstConfig,
32        DmaDescriptor,
33        DmaEligiblePeripheral,
34        DmaRxBuffer,
35        DmaTxBuffer,
36        NoBuffer,
37        Preparation,
38        aligned::InternalMemory,
39        prepare_for_rx,
40        prepare_for_tx,
41    },
42    work_queue::{Handle, Poll, Status, WorkQueueFrontend},
43};
44
45/// The longest additional data one operation takes: one DMA descriptor.
46const MAX_AAD_LEN: usize = 4_080;
47
48/// The longest text one operation takes: five DMA descriptors, which cover TLS 1.3's largest
49/// record of 16,385 bytes.
50const MAX_TEXT_LEN: usize = 5 * 4_080;
51
52// A descriptor takes at most 4,080 bytes, or fewer where PSRAM needs a larger alignment. The
53// input chain takes up to 2 descriptors for the additional data, 6 for the text and 3 of the
54// backend's own blocks.
55pub(super) const INPUT_DESCRIPTORS: usize = 2 + 6 + 3;
56
57// The output chain takes up to 6 descriptors for the text, a head and a tail for PSRAM, and the
58// bounce block.
59pub(super) const OUTPUT_DESCRIPTORS: usize = 6 + 2 + 1;
60
61/// A block the DMA reads or writes.
62type Block = InternalMemory<[u8; BLOCK_SIZE]>;
63
64/// The backend's blocks for one GCM operation.
65pub(super) struct GcmDma {
66    // Each block is its own `InternalMemory`, so it has a cache line of its own where internal
67    // memory is cached.
68    aad_tail: Block,
69    text_tail: Block,
70    lengths: Block,
71    output_tail: Block,
72}
73
74impl GcmDma {
75    pub(super) const fn new() -> Self {
76        Self {
77            aad_tail: InternalMemory::new([0; BLOCK_SIZE]),
78            text_tail: InternalMemory::new([0; BLOCK_SIZE]),
79            lengths: InternalMemory::new([0; BLOCK_SIZE]),
80            output_tail: InternalMemory::new([0; BLOCK_SIZE]),
81        }
82    }
83
84    /// Zeroes the blocks, which hold the last partial blocks of the input and the output.
85    fn clear(&mut self) {
86        for block in [
87            &mut self.aad_tail,
88            &mut self.text_tail,
89            &mut self.lengths,
90            &mut self.output_tail,
91        ] {
92            unsafe { core::ptr::write_volatile(&mut *block.get_mut(), [0; BLOCK_SIZE]) };
93        }
94    }
95}
96
97/// The state of one GCM operation, which the backend reads and writes while it owns the item.
98pub(crate) struct GcmState {
99    j0: [u8; BLOCK_SIZE],
100    aad: NonNull<[u8]>,
101    /// The tag the backend writes when it encrypts, and checks when it decrypts.
102    tag: NonNull<[u8; 16]>,
103    result: Result<(), GcmError>,
104}
105
106// Safety: the pointers are only dereferenced while the work queue owns the item, and the handle
107// borrows the data they point to for that time.
108unsafe impl Send for GcmState {}
109unsafe impl Sync for GcmState {}
110
111impl GcmState {
112    /// Ends the operation on a backend that cannot run GCM.
113    pub(crate) fn set_unsupported(&mut self) {
114        self.result = Err(GcmError::Unsupported);
115    }
116}
117
118/// An error that ends an AES-GCM operation.
119#[derive(Clone, Copy, PartialEq, Eq, Debug)]
120#[cfg_attr(feature = "defmt", derive(defmt::Format))]
121#[non_exhaustive]
122#[instability::unstable]
123pub enum GcmError {
124    /// The authentication tag does not match. The output is zeroed.
125    TagMismatch,
126
127    /// The running backend cannot process AES-GCM. Only [`AesDmaBackend`] can.
128    Unsupported,
129
130    /// [`AesGcmHandle::cancel`] cancelled the operation before it started. When
131    /// [`AesGcmHandle::wait`] or [`AesGcmHandle::wait_blocking`] returns this for a decryption,
132    /// the output is zeroed.
133    Cancelled,
134}
135
136/// An AES-GCM work queue user.
137///
138/// Operations run on [`AesDmaBackend`], which must be started. On
139/// [`AesBackend`](crate::aes::AesBackend), every operation ends with [`GcmError::Unsupported`]. A
140/// nonce must never repeat under one key, because a repeated nonce breaks the authentication.
141#[instability::unstable]
142pub struct AesGcmContext {
143    state: GcmState,
144    frontend: WorkQueueFrontend<AesOperation>,
145}
146
147impl AesGcmContext {
148    /// Creates a new context that encrypts and decrypts with `key`.
149    #[instability::unstable]
150    pub fn new(key: impl Into<Key>) -> Self {
151        Self {
152            state: GcmState {
153                j0: [0; BLOCK_SIZE],
154                aad: NonNull::slice_from_raw_parts(NonNull::dangling(), 0),
155                tag: NonNull::dangling(),
156                result: Ok(()),
157            },
158            frontend: WorkQueueFrontend::new(AesOperation {
159                mode: Operation::Encrypt,
160                cipher_mode: NonNull::dangling(),
161                buffers: unsafe { CryptoBuffers::new_in_place(&mut []).into_inner() },
162                key: key.into(),
163                gcm: None,
164            }),
165        }
166    }
167
168    /// Starts encrypting `input` into `output`, and computing the tag over `aad` and the
169    /// ciphertext.
170    ///
171    /// The tag is written to `tag` when the operation completes. The nonce must never repeat
172    /// under this context's key.
173    ///
174    /// Dropping the returned handle before the operation completes cancels the operation.
175    ///
176    /// # Errors
177    ///
178    /// - [`Error::IncorrectBufferLength`] when `input` is empty or longer than 20,400 bytes, or
179    ///   `aad` is longer than 4,080 bytes.
180    /// - [`Error::AadNotInRam`] when the DMA cannot read `aad`, for example from flash.
181    /// - [`Error::BuffersNotEqual`] when `input` and `output` differ in length.
182    #[cfg_attr(
183        esp32s2,
184        doc = "- [`Error::OutputInPsram`] when `output` is in PSRAM. `input` and `aad` can be in PSRAM."
185    )]
186    #[instability::unstable]
187    pub fn encrypt<'t>(
188        &'t mut self,
189        nonce: &[u8; 12],
190        aad: &'t [u8],
191        input: &'t [u8],
192        output: &'t mut [u8],
193        tag: &'t mut [u8; 16],
194    ) -> Result<AesGcmHandle<'t>, Error> {
195        validate(aad, input.len(), output)?;
196        let buffers = unsafe { CryptoBuffers::new(input, output)?.into_inner() };
197        Ok(self.post(Operation::Encrypt, nonce, aad, buffers, NonNull::from(tag)))
198    }
199
200    /// Starts encrypting `buffer` in place, and computing the tag over `aad` and the
201    /// ciphertext.
202    ///
203    /// Operates as [`Self::encrypt`] does.
204    ///
205    /// # Errors
206    ///
207    /// As [`Self::encrypt`].
208    #[instability::unstable]
209    pub fn encrypt_in_place<'t>(
210        &'t mut self,
211        nonce: &[u8; 12],
212        aad: &'t [u8],
213        buffer: &'t mut [u8],
214        tag: &'t mut [u8; 16],
215    ) -> Result<AesGcmHandle<'t>, Error> {
216        validate(aad, buffer.len(), buffer)?;
217        let buffers = unsafe { CryptoBuffers::new_in_place(buffer).into_inner() };
218        Ok(self.post(Operation::Encrypt, nonce, aad, buffers, NonNull::from(tag)))
219    }
220
221    /// Starts decrypting `input` into `output`, and checking `tag` over `aad` and `input`.
222    ///
223    /// When the tag does not match, the operation ends with [`GcmError::TagMismatch`] and
224    /// `output` is zeroed.
225    ///
226    /// # Errors
227    ///
228    /// As [`Self::encrypt`].
229    #[instability::unstable]
230    pub fn decrypt<'t>(
231        &'t mut self,
232        nonce: &[u8; 12],
233        aad: &'t [u8],
234        input: &'t [u8],
235        output: &'t mut [u8],
236        tag: &'t [u8; 16],
237    ) -> Result<AesGcmHandle<'t>, Error> {
238        validate(aad, input.len(), output)?;
239        let buffers = unsafe { CryptoBuffers::new(input, output)?.into_inner() };
240        Ok(self.post(Operation::Decrypt, nonce, aad, buffers, NonNull::from(tag)))
241    }
242
243    /// Starts decrypting `buffer` in place, and checking `tag` over `aad` and `buffer`.
244    ///
245    /// Operates as [`Self::decrypt`] does.
246    ///
247    /// # Errors
248    ///
249    /// As [`Self::encrypt`].
250    #[instability::unstable]
251    pub fn decrypt_in_place<'t>(
252        &'t mut self,
253        nonce: &[u8; 12],
254        aad: &'t [u8],
255        buffer: &'t mut [u8],
256        tag: &'t [u8; 16],
257    ) -> Result<AesGcmHandle<'t>, Error> {
258        validate(aad, buffer.len(), buffer)?;
259        let buffers = unsafe { CryptoBuffers::new_in_place(buffer).into_inner() };
260        Ok(self.post(Operation::Decrypt, nonce, aad, buffers, NonNull::from(tag)))
261    }
262
263    fn post<'t>(
264        &'t mut self,
265        mode: Operation,
266        nonce: &[u8; 12],
267        aad: &'t [u8],
268        buffers: UnsafeCryptoBuffers,
269        tag: NonNull<[u8; 16]>,
270    ) -> AesGcmHandle<'t> {
271        // A 96-bit nonce gives J0 = nonce || 0x00000001, without GHASH.
272        self.state.j0[..12].copy_from_slice(nonce);
273        self.state.j0[12..].copy_from_slice(&1u32.to_be_bytes());
274        self.state.aad = NonNull::from(aad);
275        self.state.tag = tag;
276        self.state.result = Ok(());
277        let state = NonNull::from(&mut self.state);
278
279        let data = self.frontend.data_mut();
280        data.mode = mode;
281        data.buffers = buffers;
282        data.gcm = Some(state);
283
284        AesGcmHandle {
285            handle: self.frontend.post(&AES_WORK_QUEUE),
286            state,
287            output: buffers.output,
288            decrypt: mode == Operation::Decrypt,
289        }
290    }
291}
292
293fn validate(aad: &[u8], text_len: usize, output: &[u8]) -> Result<(), Error> {
294    if text_len == 0 || text_len > MAX_TEXT_LEN || aad.len() > MAX_AAD_LEN {
295        return Err(Error::IncorrectBufferLength);
296    }
297
298    // The backend copies a text in flash to the output first. Only the text has an output to be
299    // copied into, so the DMA reads the additional data where it is.
300    if !aad.is_empty() && !crate::soc::is_valid_memory_address(aad.as_ptr() as usize) {
301        return Err(Error::AadNotInRam);
302    }
303
304    // The DMA of the ESP32-S2 fails to write a RAM descriptor that follows a PSRAM descriptor.
305    // The last bytes of an AES-GCM output are written to the bounce block in RAM, so the check
306    // must refuse an output in PSRAM.
307    cfg_select! {
308        esp32s2 => {
309            if crate::psram::psram_range().contains(&(output.as_ptr() as usize)) {
310                return Err(Error::OutputInPsram);
311            }
312        }
313        _ => {
314            let _ = output;
315        }
316    }
317
318    Ok(())
319}
320
321/// The handle to a pending AES-GCM operation.
322///
323/// Dropping it before the operation completes cancels the operation.
324#[instability::unstable]
325pub struct AesGcmHandle<'t> {
326    handle: Handle<'t, AesOperation>,
327    state: NonNull<GcmState>,
328    output: NonNull<[u8]>,
329    decrypt: bool,
330}
331
332impl AesGcmHandle<'_> {
333    /// Polls the operation. Returns whether it has completed.
334    #[instability::unstable]
335    pub fn poll(&mut self) -> bool {
336        self.handle.poll()
337    }
338
339    /// Polls the operation to completion, by busy-looping, and returns its result.
340    #[instability::unstable]
341    pub fn wait_blocking(self) -> Result<(), GcmError> {
342        let Self {
343            handle,
344            state,
345            output,
346            decrypt,
347        } = self;
348        result(handle.wait_blocking(), state, output, decrypt)
349    }
350
351    /// Waits until the operation completes, and returns its result.
352    #[instability::unstable]
353    pub async fn wait(&mut self) -> Result<(), GcmError> {
354        let status = self.handle.wait().await;
355        result(status, self.state, self.output, self.decrypt)
356    }
357
358    /// Cancels the operation, and waits until it leaves the work queue.
359    ///
360    /// An operation that has not started ends with [`GcmError::Cancelled`]. One that the backend
361    /// has started runs to completion, because the hardware cannot stop part way, and keeps its
362    /// result. [`Self::wait`] or [`Self::wait_blocking`] then returns the result.
363    #[instability::unstable]
364    pub fn cancel(&mut self) -> impl Future<Output = ()> {
365        self.handle.cancel()
366    }
367}
368
369fn result(
370    status: Status,
371    state: NonNull<GcmState>,
372    output: NonNull<[u8]>,
373    decrypt: bool,
374) -> Result<(), GcmError> {
375    match status {
376        // The backend writes the result before it completes the item.
377        Status::Completed => unsafe { state.as_ref() }.result,
378        Status::Cancelled => {
379            // Plaintext that no tag has verified must not reach the caller.
380            if decrypt {
381                unsafe { zero(output) };
382            }
383            Err(GcmError::Cancelled)
384        }
385    }
386}
387
388unsafe fn zero(buffer: NonNull<[u8]>) {
389    unsafe { buffer.cast::<u8>().write_bytes(0, buffer.len()) };
390}
391
392/// Compares two tags in constant time.
393///
394/// It must not return early: the time it takes must not depend on where the tags differ.
395fn tags_equal(a: &[u8; 16], b: &[u8; 16]) -> bool {
396    let mut difference = 0u8;
397    for (x, y) in a.iter().zip(b) {
398        difference |= black_box(x ^ y);
399    }
400    black_box(difference) == 0
401}
402
403/// The GCM register values of one operation.
404struct GcmRegisters {
405    text_blocks: usize,
406    aad_blocks: usize,
407    remainder_bits: usize,
408    j0: [u8; BLOCK_SIZE],
409}
410
411impl<'d> AesDma<'d> {
412    /// Sets the number of additional-data blocks, and the number of valid bits in the text's last
413    /// block. A full block sets 0.
414    fn set_gcm_lengths(&self, aad_blocks: usize, remainder_bits: usize) {
415        let regs = self.aes.regs();
416        regs.aad_block_num()
417            .write(|w| unsafe { w.aad_block_num().bits(aad_blocks as _) });
418        regs.remainder_bit_num()
419            .write(|w| unsafe { w.remainder_bit_num().bits(remainder_bits as _) });
420    }
421
422    fn write_j0(&self, j0: &[u8; BLOCK_SIZE]) {
423        for (word, reg) in read_words(j0).zip(self.aes.regs().j0_mem_iter()) {
424            reg.write(|w| unsafe { w.bits(word) });
425        }
426    }
427
428    /// Starts the data phase of the operation that computed H.
429    fn continue_transform(&self) {
430        self.aes
431            .regs()
432            .continue_()
433            .write(|w| w.continue_().set_bit());
434    }
435
436    fn read_tag(&self, tag: &mut [u8; 16]) {
437        write_words(tag, |i| self.aes.regs().t0_mem(i).read().bits());
438    }
439
440    fn start_gcm_transfer(
441        mut self,
442        registers: &GcmRegisters,
443        mut output: NoBuffer,
444        mut input: NoBuffer,
445        mode: Mode,
446        key: &Key,
447    ) -> AesTransfer<'d, NoBuffer, NoBuffer> {
448        // AES has to be restarted after each calculation
449        self.reset_aes();
450
451        // Both channels are prepared before the key goes in, so that a DMA error leaves no key in
452        // the registers.
453        let peri = self.aes.aes.dma_peripheral();
454        unwrap!(
455            unsafe { self.channel.tx.prepare_transfer(peri, &mut input) }
456                .and_then(|_| unsafe { self.channel.rx.prepare_transfer(peri, &mut output) })
457        );
458
459        self.aes.write_mode(mode);
460        self.write_key(key);
461        self.set_cipher_mode(CipherMode::Gcm);
462        self.set_gcm_lengths(registers.aad_blocks, registers.remainder_bits);
463
464        // In DMA mode, the first start computes H before the DMA delivers any data.
465        self.enable_dma(true);
466        self.start_transform();
467        while !self.aes.is_idle() {}
468
469        self.write_j0(&registers.j0);
470
471        if let Err(error) = self
472            .channel
473            .tx
474            .start_transfer()
475            .and_then(|_| self.channel.rx.start_transfer())
476        {
477            // The key must not stay in the registers after a panic.
478            self.reset_aes();
479            panic!("AES-GCM DMA transfer failed to start: {:?}", error);
480        }
481
482        self.listen();
483        self.set_num_block(registers.text_blocks as u32);
484        self.continue_transform();
485
486        DmaRxBuffer::into_view(output);
487        DmaTxBuffer::into_view(input);
488        AesTransfer {
489            aes_dma: ManuallyDrop::new(self),
490            rx_view: ManuallyDrop::new(()),
491            tx_view: ManuallyDrop::new(()),
492        }
493    }
494}
495
496impl<'d> AesDmaBackend<'d> {
497    /// Starts a GCM operation on the DMA.
498    pub(super) fn start_gcm(
499        &mut self,
500        driver: AesDma<'d>,
501        item: &mut AesOperation,
502        state: NonNull<GcmState>,
503    ) -> Poll {
504        let len = item.buffers.input.len();
505
506        if !crate::soc::is_valid_memory_address(item.buffers.input.addr().get()) {
507            // The DMA cannot read the input, so the text is copied to the output in RAM and
508            // processed in place.
509            unsafe {
510                item.buffers
511                    .output
512                    .cast::<u8>()
513                    .copy_from_nonoverlapping(item.buffers.input.cast::<u8>(), len);
514            }
515            item.buffers.input = item.buffers.output;
516        }
517
518        let state = unsafe { state.as_ref() };
519        let aad = state.aad;
520        let text_full = len - len % BLOCK_SIZE;
521        let aad_full = aad.len() - aad.len() % BLOCK_SIZE;
522
523        let mut dma = self.gcm.get_mut();
524        dma.clear();
525        unsafe {
526            copy_tail(&mut dma.aad_tail, aad, aad_full);
527            copy_tail(&mut dma.text_tail, item.buffers.input, text_full);
528        }
529        // The length block holds the bit lengths of the additional data and the text, each as a
530        // 64-bit big-endian number.
531        dma.lengths.get_mut()[..8].copy_from_slice(&((aad.len() as u64) * 8).to_be_bytes());
532        dma.lengths.get_mut()[8..].copy_from_slice(&((len as u64) * 8).to_be_bytes());
533
534        let GcmDma {
535            aad_tail,
536            text_tail,
537            lengths,
538            output_tail,
539        } = &mut *dma;
540        let mut descriptors = self.descriptors.get_mut();
541        let (input_descriptors, output_descriptors) =
542            descriptors[..INPUT_DESCRIPTORS + OUTPUT_DESCRIPTORS].split_at_mut(INPUT_DESCRIPTORS);
543
544        let mut input = Chain::new(input_descriptors);
545        if aad_full > 0 {
546            input.tx(get_range(aad, 0, aad_full));
547        }
548        if aad.len() > aad_full {
549            input.tx_block(aad_tail);
550        }
551        if text_full > 0 {
552            input.tx(get_range(item.buffers.input, 0, text_full));
553        }
554        if len > text_full {
555            input.tx_block(text_tail);
556        }
557        input.tx_block(lengths);
558        let input = input.finish(false);
559
560        let mut output = Chain::new(output_descriptors);
561        if text_full > 0 {
562            output.rx(
563                #[cfg(dma_can_access_psram)]
564                &mut self.unaligned_data_buffers,
565                get_range(item.buffers.output, 0, text_full),
566            );
567        }
568        if len > text_full {
569            output.rx_block(output_tail);
570        }
571        let output = output.finish(true);
572
573        #[cfg(soc_internal_memory_cached)]
574        {
575            dma.writeback();
576            descriptors.writeback();
577        }
578
579        let mode = if item.mode == Operation::Encrypt {
580            item.key.encrypt_mode()
581        } else {
582            item.key.decrypt_mode()
583        };
584
585        let registers = GcmRegisters {
586            text_blocks: len.div_ceil(BLOCK_SIZE),
587            aad_blocks: aad.len().div_ceil(BLOCK_SIZE),
588            remainder_bits: (len * 8) % 128,
589            j0: state.j0,
590        };
591        let transfer = driver.start_gcm_transfer(&registers, output, input, mode, &item.key);
592
593        self.state = DriverState::WaitingForDma {
594            transfer,
595            remaining_bytes: 0,
596        };
597
598        Poll::Pending(false)
599    }
600
601    /// Completes a GCM operation once the DMA is done.
602    ///
603    /// It runs in the AES interrupt when the queue is processed there. On a mismatch it zeroes up
604    /// to 20,400 bytes of output.
605    pub(super) fn finish_gcm(
606        &mut self,
607        item: &mut AesOperation,
608        mut state: NonNull<GcmState>,
609    ) -> Poll {
610        let DriverState::WaitingForDma { transfer, .. } = &mut self.state else {
611            unreachable!()
612        };
613        if !transfer.is_done() {
614            return Poll::Pending(false);
615        }
616
617        let mut tag = [0; 16];
618        // `wait` resets the block, which clears the key, H, J0 and tag registers, so the tag is
619        // read first. The work queue never stops the backend during an operation, so every
620        // operation the backend starts reaches this reset.
621        transfer.aes_dma.read_tag(&mut tag);
622
623        let DriverState::WaitingForDma { transfer, .. } =
624            core::mem::replace(&mut self.state, DriverState::None)
625        else {
626            unreachable!()
627        };
628        let (driver, _, _) = transfer.wait();
629        driver.clear_interrupt();
630
631        // The CPU must not read a stale line of the output the DMA wrote.
632        #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
633        {
634            let len = item.buffers.output.len();
635            crate::dma::invalidate_received(get_range(
636                item.buffers.output,
637                0,
638                len - len % BLOCK_SIZE,
639            ));
640        }
641
642        #[cfg(dma_can_access_psram)]
643        for buffer in self.unaligned_data_buffers.iter_mut() {
644            if let Some(buffer) = buffer.as_mut() {
645                buffer.write_back();
646                // The buffer held part of the output, which can be plaintext.
647                buffer.clear();
648            }
649            *buffer = None;
650        }
651
652        let output = item.buffers.output;
653        let len = output.len();
654        let text_full = len - len % BLOCK_SIZE;
655
656        let mut dma = self.gcm.get_mut();
657        #[cfg(soc_internal_memory_cached)]
658        dma.invalidate();
659        if len > text_full {
660            unsafe {
661                output.cast::<u8>().add(text_full).copy_from_nonoverlapping(
662                    NonNull::from(&*dma.output_tail.get_ref()).cast::<u8>(),
663                    len - text_full,
664                );
665            }
666        }
667        dma.clear();
668        #[cfg(soc_internal_memory_cached)]
669        dma.writeback();
670
671        let state = unsafe { state.as_mut() };
672        state.result = if item.mode == Operation::Encrypt {
673            unsafe { state.tag.write(tag) };
674            Ok(())
675        } else if tags_equal(&tag, unsafe { state.tag.as_ref() }) {
676            Ok(())
677        } else {
678            unsafe { zero(output) };
679            Err(GcmError::TagMismatch)
680        };
681        unsafe { core::ptr::write_volatile(&mut tag, [0; 16]) };
682
683        self.state = DriverState::Idle(driver);
684
685        Poll::Ready(Status::Completed)
686    }
687}
688
689/// Copies the bytes of `data` from `start` to its end into the start of `block`. The rest of
690/// `block` stays zero.
691unsafe fn copy_tail(block: &mut Block, data: NonNull<[u8]>, start: usize) {
692    let len = data.len() - start;
693    unsafe {
694        block
695            .get_mut()
696            .as_mut_ptr()
697            .copy_from_nonoverlapping(data.cast::<u8>().add(start).as_ptr(), len);
698    }
699}
700
701fn get_range(data: NonNull<[u8]>, start: usize, end: usize) -> NonNull<[u8]> {
702    NonNull::slice_from_raw_parts(unsafe { data.cast::<u8>().add(start) }, end - start)
703}
704
705/// Builds one descriptor chain from several segments.
706struct Chain<'a> {
707    descriptors: &'a mut [DmaDescriptor],
708    used: usize,
709    #[cfg(dma_can_access_psram)]
710    accesses_psram: bool,
711}
712
713impl<'a> Chain<'a> {
714    fn new(descriptors: &'a mut [DmaDescriptor]) -> Self {
715        Self {
716            descriptors,
717            used: 0,
718            #[cfg(dma_can_access_psram)]
719            accesses_psram: false,
720        }
721    }
722
723    /// Links `count` descriptors from `first` after the chain so far.
724    fn append(&mut self, first: usize, count: usize) {
725        if first > 0 {
726            let next = &raw mut self.descriptors[first];
727            self.descriptors[first - 1].next = next;
728        }
729        self.used = first + count;
730    }
731
732    /// Counts the descriptors a prepared segment linked, from `first`.
733    fn linked_from(&self, first: usize) -> usize {
734        let mut count = 1;
735        while !self.descriptors[first + count - 1].next.is_null() {
736            count += 1;
737        }
738        count
739    }
740
741    fn tx(&mut self, data: NonNull<[u8]>) {
742        let first = self.used;
743        let (buffer, len) =
744            unwrap!(unsafe { prepare_for_tx(&mut self.descriptors[first..], data, BLOCK_SIZE) });
745        // `validate` bounds the lengths and the descriptor counts cover them in PSRAM, so the
746        // chain always takes all of `data`.
747        assert_eq!(len, data.len());
748        cfg_select! {
749            dma_can_access_psram => {
750                self.accesses_psram |= buffer.0.accesses_psram;
751            }
752            _ => {
753                let _ = buffer;
754            }
755        }
756        let count = self.linked_from(first);
757        self.append(first, count);
758    }
759
760    fn tx_block(&mut self, block: &mut Block) {
761        let first = self.used;
762        let descriptor = &mut self.descriptors[first];
763        *descriptor = DmaDescriptor::EMPTY;
764        descriptor.buffer = block.get_mut().as_mut_ptr();
765        descriptor.set_size(BLOCK_SIZE);
766        descriptor.set_length(BLOCK_SIZE);
767        self.append(first, 1);
768    }
769
770    fn rx(
771        &mut self,
772        #[cfg(dma_can_access_psram)] align_buffers: &mut [Option<crate::dma::ManualWritebackBuffer>;
773                 2],
774        data: NonNull<[u8]>,
775    ) {
776        let first = self.used;
777        let (buffer, len) = unsafe {
778            prepare_for_rx(
779                &mut self.descriptors[first..],
780                #[cfg(dma_can_access_psram)]
781                align_buffers,
782                data,
783            )
784        };
785        assert_eq!(len, data.len());
786        cfg_select! {
787            dma_can_access_psram => {
788                self.accesses_psram |= buffer.0.accesses_psram;
789            }
790            _ => {
791                let _ = buffer;
792            }
793        }
794        let count = self.linked_from(first);
795        self.append(first, count);
796    }
797
798    fn rx_block(&mut self, block: &mut Block) {
799        let first = self.used;
800        let descriptor = &mut self.descriptors[first];
801        *descriptor = DmaDescriptor::EMPTY;
802        descriptor.buffer = block.get_mut().as_mut_ptr();
803        descriptor.set_size(BLOCK_SIZE);
804        self.append(first, 1);
805    }
806
807    /// Sets the owner and the end-of-frame flags of the chain, and returns it as a DMA buffer.
808    fn finish(self, rx: bool) -> NoBuffer {
809        let last = self.used - 1;
810        for (i, descriptor) in self.descriptors[..self.used].iter_mut().enumerate() {
811            if rx {
812                descriptor.reset_for_rx();
813            } else {
814                descriptor.reset_for_tx(i == last);
815            }
816        }
817        NoBuffer(Preparation {
818            start: self.descriptors.as_mut_ptr(),
819            burst_transfer: BurstConfig::DEFAULT,
820            check_owner: None,
821            auto_write_back: rx,
822            #[cfg(dma_can_access_psram)]
823            accesses_psram: self.accesses_psram,
824        })
825    }
826}