1use core::{hint::black_box, mem::ManuallyDrop, ptr::NonNull};
15
16use super::{AesDma, AesDmaBackend, AesTransfer, CipherMode, DriverState};
17use crate::{
18 aes::{
19 AES_WORK_QUEUE,
20 AesOperation,
21 BLOCK_SIZE,
22 Error,
23 Key,
24 Mode,
25 Operation,
26 cipher_modes::{CryptoBuffers, UnsafeCryptoBuffers},
27 read_words,
28 write_words,
29 },
30 dma::{
31 BurstConfig,
32 DmaDescriptor,
33 DmaEligiblePeripheral,
34 DmaRxBuffer,
35 DmaTxBuffer,
36 NoBuffer,
37 Preparation,
38 aligned::InternalMemory,
39 prepare_for_rx,
40 prepare_for_tx,
41 },
42 work_queue::{Handle, Poll, Status, WorkQueueFrontend},
43};
44
45const MAX_AAD_LEN: usize = 4_080;
47
48const MAX_TEXT_LEN: usize = 5 * 4_080;
51
52pub(super) const INPUT_DESCRIPTORS: usize = 2 + 6 + 3;
56
57pub(super) const OUTPUT_DESCRIPTORS: usize = 6 + 2 + 1;
60
61type Block = InternalMemory<[u8; BLOCK_SIZE]>;
63
64pub(super) struct GcmDma {
66 aad_tail: Block,
69 text_tail: Block,
70 lengths: Block,
71 output_tail: Block,
72}
73
74impl GcmDma {
75 pub(super) const fn new() -> Self {
76 Self {
77 aad_tail: InternalMemory::new([0; BLOCK_SIZE]),
78 text_tail: InternalMemory::new([0; BLOCK_SIZE]),
79 lengths: InternalMemory::new([0; BLOCK_SIZE]),
80 output_tail: InternalMemory::new([0; BLOCK_SIZE]),
81 }
82 }
83
84 fn clear(&mut self) {
86 for block in [
87 &mut self.aad_tail,
88 &mut self.text_tail,
89 &mut self.lengths,
90 &mut self.output_tail,
91 ] {
92 unsafe { core::ptr::write_volatile(&mut *block.get_mut(), [0; BLOCK_SIZE]) };
93 }
94 }
95}
96
97pub(crate) struct GcmState {
99 j0: [u8; BLOCK_SIZE],
100 aad: NonNull<[u8]>,
101 tag: NonNull<[u8; 16]>,
103 result: Result<(), GcmError>,
104}
105
106unsafe impl Send for GcmState {}
109unsafe impl Sync for GcmState {}
110
111impl GcmState {
112 pub(crate) fn set_unsupported(&mut self) {
114 self.result = Err(GcmError::Unsupported);
115 }
116}
117
118#[derive(Clone, Copy, PartialEq, Eq, Debug)]
120#[cfg_attr(feature = "defmt", derive(defmt::Format))]
121#[non_exhaustive]
122#[instability::unstable]
123pub enum GcmError {
124 TagMismatch,
126
127 Unsupported,
129
130 Cancelled,
134}
135
136#[instability::unstable]
142pub struct AesGcmContext {
143 state: GcmState,
144 frontend: WorkQueueFrontend<AesOperation>,
145}
146
147impl AesGcmContext {
148 #[instability::unstable]
150 pub fn new(key: impl Into<Key>) -> Self {
151 Self {
152 state: GcmState {
153 j0: [0; BLOCK_SIZE],
154 aad: NonNull::slice_from_raw_parts(NonNull::dangling(), 0),
155 tag: NonNull::dangling(),
156 result: Ok(()),
157 },
158 frontend: WorkQueueFrontend::new(AesOperation {
159 mode: Operation::Encrypt,
160 cipher_mode: NonNull::dangling(),
161 buffers: unsafe { CryptoBuffers::new_in_place(&mut []).into_inner() },
162 key: key.into(),
163 gcm: None,
164 }),
165 }
166 }
167
168 #[cfg_attr(
183 esp32s2,
184 doc = "- [`Error::OutputInPsram`] when `output` is in PSRAM. `input` and `aad` can be in PSRAM."
185 )]
186 #[instability::unstable]
187 pub fn encrypt<'t>(
188 &'t mut self,
189 nonce: &[u8; 12],
190 aad: &'t [u8],
191 input: &'t [u8],
192 output: &'t mut [u8],
193 tag: &'t mut [u8; 16],
194 ) -> Result<AesGcmHandle<'t>, Error> {
195 validate(aad, input.len(), output)?;
196 let buffers = unsafe { CryptoBuffers::new(input, output)?.into_inner() };
197 Ok(self.post(Operation::Encrypt, nonce, aad, buffers, NonNull::from(tag)))
198 }
199
200 #[instability::unstable]
209 pub fn encrypt_in_place<'t>(
210 &'t mut self,
211 nonce: &[u8; 12],
212 aad: &'t [u8],
213 buffer: &'t mut [u8],
214 tag: &'t mut [u8; 16],
215 ) -> Result<AesGcmHandle<'t>, Error> {
216 validate(aad, buffer.len(), buffer)?;
217 let buffers = unsafe { CryptoBuffers::new_in_place(buffer).into_inner() };
218 Ok(self.post(Operation::Encrypt, nonce, aad, buffers, NonNull::from(tag)))
219 }
220
221 #[instability::unstable]
230 pub fn decrypt<'t>(
231 &'t mut self,
232 nonce: &[u8; 12],
233 aad: &'t [u8],
234 input: &'t [u8],
235 output: &'t mut [u8],
236 tag: &'t [u8; 16],
237 ) -> Result<AesGcmHandle<'t>, Error> {
238 validate(aad, input.len(), output)?;
239 let buffers = unsafe { CryptoBuffers::new(input, output)?.into_inner() };
240 Ok(self.post(Operation::Decrypt, nonce, aad, buffers, NonNull::from(tag)))
241 }
242
243 #[instability::unstable]
251 pub fn decrypt_in_place<'t>(
252 &'t mut self,
253 nonce: &[u8; 12],
254 aad: &'t [u8],
255 buffer: &'t mut [u8],
256 tag: &'t [u8; 16],
257 ) -> Result<AesGcmHandle<'t>, Error> {
258 validate(aad, buffer.len(), buffer)?;
259 let buffers = unsafe { CryptoBuffers::new_in_place(buffer).into_inner() };
260 Ok(self.post(Operation::Decrypt, nonce, aad, buffers, NonNull::from(tag)))
261 }
262
263 fn post<'t>(
264 &'t mut self,
265 mode: Operation,
266 nonce: &[u8; 12],
267 aad: &'t [u8],
268 buffers: UnsafeCryptoBuffers,
269 tag: NonNull<[u8; 16]>,
270 ) -> AesGcmHandle<'t> {
271 self.state.j0[..12].copy_from_slice(nonce);
273 self.state.j0[12..].copy_from_slice(&1u32.to_be_bytes());
274 self.state.aad = NonNull::from(aad);
275 self.state.tag = tag;
276 self.state.result = Ok(());
277 let state = NonNull::from(&mut self.state);
278
279 let data = self.frontend.data_mut();
280 data.mode = mode;
281 data.buffers = buffers;
282 data.gcm = Some(state);
283
284 AesGcmHandle {
285 handle: self.frontend.post(&AES_WORK_QUEUE),
286 state,
287 output: buffers.output,
288 decrypt: mode == Operation::Decrypt,
289 }
290 }
291}
292
293fn validate(aad: &[u8], text_len: usize, output: &[u8]) -> Result<(), Error> {
294 if text_len == 0 || text_len > MAX_TEXT_LEN || aad.len() > MAX_AAD_LEN {
295 return Err(Error::IncorrectBufferLength);
296 }
297
298 if !aad.is_empty() && !crate::soc::is_valid_memory_address(aad.as_ptr() as usize) {
301 return Err(Error::AadNotInRam);
302 }
303
304 cfg_select! {
308 esp32s2 => {
309 if crate::psram::psram_range().contains(&(output.as_ptr() as usize)) {
310 return Err(Error::OutputInPsram);
311 }
312 }
313 _ => {
314 let _ = output;
315 }
316 }
317
318 Ok(())
319}
320
321#[instability::unstable]
325pub struct AesGcmHandle<'t> {
326 handle: Handle<'t, AesOperation>,
327 state: NonNull<GcmState>,
328 output: NonNull<[u8]>,
329 decrypt: bool,
330}
331
332impl AesGcmHandle<'_> {
333 #[instability::unstable]
335 pub fn poll(&mut self) -> bool {
336 self.handle.poll()
337 }
338
339 #[instability::unstable]
341 pub fn wait_blocking(self) -> Result<(), GcmError> {
342 let Self {
343 handle,
344 state,
345 output,
346 decrypt,
347 } = self;
348 result(handle.wait_blocking(), state, output, decrypt)
349 }
350
351 #[instability::unstable]
353 pub async fn wait(&mut self) -> Result<(), GcmError> {
354 let status = self.handle.wait().await;
355 result(status, self.state, self.output, self.decrypt)
356 }
357
358 #[instability::unstable]
364 pub fn cancel(&mut self) -> impl Future<Output = ()> {
365 self.handle.cancel()
366 }
367}
368
369fn result(
370 status: Status,
371 state: NonNull<GcmState>,
372 output: NonNull<[u8]>,
373 decrypt: bool,
374) -> Result<(), GcmError> {
375 match status {
376 Status::Completed => unsafe { state.as_ref() }.result,
378 Status::Cancelled => {
379 if decrypt {
381 unsafe { zero(output) };
382 }
383 Err(GcmError::Cancelled)
384 }
385 }
386}
387
388unsafe fn zero(buffer: NonNull<[u8]>) {
389 unsafe { buffer.cast::<u8>().write_bytes(0, buffer.len()) };
390}
391
392fn tags_equal(a: &[u8; 16], b: &[u8; 16]) -> bool {
396 let mut difference = 0u8;
397 for (x, y) in a.iter().zip(b) {
398 difference |= black_box(x ^ y);
399 }
400 black_box(difference) == 0
401}
402
403struct GcmRegisters {
405 text_blocks: usize,
406 aad_blocks: usize,
407 remainder_bits: usize,
408 j0: [u8; BLOCK_SIZE],
409}
410
411impl<'d> AesDma<'d> {
412 fn set_gcm_lengths(&self, aad_blocks: usize, remainder_bits: usize) {
415 let regs = self.aes.regs();
416 regs.aad_block_num()
417 .write(|w| unsafe { w.aad_block_num().bits(aad_blocks as _) });
418 regs.remainder_bit_num()
419 .write(|w| unsafe { w.remainder_bit_num().bits(remainder_bits as _) });
420 }
421
422 fn write_j0(&self, j0: &[u8; BLOCK_SIZE]) {
423 for (word, reg) in read_words(j0).zip(self.aes.regs().j0_mem_iter()) {
424 reg.write(|w| unsafe { w.bits(word) });
425 }
426 }
427
428 fn continue_transform(&self) {
430 self.aes
431 .regs()
432 .continue_()
433 .write(|w| w.continue_().set_bit());
434 }
435
436 fn read_tag(&self, tag: &mut [u8; 16]) {
437 write_words(tag, |i| self.aes.regs().t0_mem(i).read().bits());
438 }
439
440 fn start_gcm_transfer(
441 mut self,
442 registers: &GcmRegisters,
443 mut output: NoBuffer,
444 mut input: NoBuffer,
445 mode: Mode,
446 key: &Key,
447 ) -> AesTransfer<'d, NoBuffer, NoBuffer> {
448 self.reset_aes();
450
451 let peri = self.aes.aes.dma_peripheral();
454 unwrap!(
455 unsafe { self.channel.tx.prepare_transfer(peri, &mut input) }
456 .and_then(|_| unsafe { self.channel.rx.prepare_transfer(peri, &mut output) })
457 );
458
459 self.aes.write_mode(mode);
460 self.write_key(key);
461 self.set_cipher_mode(CipherMode::Gcm);
462 self.set_gcm_lengths(registers.aad_blocks, registers.remainder_bits);
463
464 self.enable_dma(true);
466 self.start_transform();
467 while !self.aes.is_idle() {}
468
469 self.write_j0(®isters.j0);
470
471 if let Err(error) = self
472 .channel
473 .tx
474 .start_transfer()
475 .and_then(|_| self.channel.rx.start_transfer())
476 {
477 self.reset_aes();
479 panic!("AES-GCM DMA transfer failed to start: {:?}", error);
480 }
481
482 self.listen();
483 self.set_num_block(registers.text_blocks as u32);
484 self.continue_transform();
485
486 DmaRxBuffer::into_view(output);
487 DmaTxBuffer::into_view(input);
488 AesTransfer {
489 aes_dma: ManuallyDrop::new(self),
490 rx_view: ManuallyDrop::new(()),
491 tx_view: ManuallyDrop::new(()),
492 }
493 }
494}
495
496impl<'d> AesDmaBackend<'d> {
497 pub(super) fn start_gcm(
499 &mut self,
500 driver: AesDma<'d>,
501 item: &mut AesOperation,
502 state: NonNull<GcmState>,
503 ) -> Poll {
504 let len = item.buffers.input.len();
505
506 if !crate::soc::is_valid_memory_address(item.buffers.input.addr().get()) {
507 unsafe {
510 item.buffers
511 .output
512 .cast::<u8>()
513 .copy_from_nonoverlapping(item.buffers.input.cast::<u8>(), len);
514 }
515 item.buffers.input = item.buffers.output;
516 }
517
518 let state = unsafe { state.as_ref() };
519 let aad = state.aad;
520 let text_full = len - len % BLOCK_SIZE;
521 let aad_full = aad.len() - aad.len() % BLOCK_SIZE;
522
523 let mut dma = self.gcm.get_mut();
524 dma.clear();
525 unsafe {
526 copy_tail(&mut dma.aad_tail, aad, aad_full);
527 copy_tail(&mut dma.text_tail, item.buffers.input, text_full);
528 }
529 dma.lengths.get_mut()[..8].copy_from_slice(&((aad.len() as u64) * 8).to_be_bytes());
532 dma.lengths.get_mut()[8..].copy_from_slice(&((len as u64) * 8).to_be_bytes());
533
534 let GcmDma {
535 aad_tail,
536 text_tail,
537 lengths,
538 output_tail,
539 } = &mut *dma;
540 let mut descriptors = self.descriptors.get_mut();
541 let (input_descriptors, output_descriptors) =
542 descriptors[..INPUT_DESCRIPTORS + OUTPUT_DESCRIPTORS].split_at_mut(INPUT_DESCRIPTORS);
543
544 let mut input = Chain::new(input_descriptors);
545 if aad_full > 0 {
546 input.tx(get_range(aad, 0, aad_full));
547 }
548 if aad.len() > aad_full {
549 input.tx_block(aad_tail);
550 }
551 if text_full > 0 {
552 input.tx(get_range(item.buffers.input, 0, text_full));
553 }
554 if len > text_full {
555 input.tx_block(text_tail);
556 }
557 input.tx_block(lengths);
558 let input = input.finish(false);
559
560 let mut output = Chain::new(output_descriptors);
561 if text_full > 0 {
562 output.rx(
563 #[cfg(dma_can_access_psram)]
564 &mut self.unaligned_data_buffers,
565 get_range(item.buffers.output, 0, text_full),
566 );
567 }
568 if len > text_full {
569 output.rx_block(output_tail);
570 }
571 let output = output.finish(true);
572
573 #[cfg(soc_internal_memory_cached)]
574 {
575 dma.writeback();
576 descriptors.writeback();
577 }
578
579 let mode = if item.mode == Operation::Encrypt {
580 item.key.encrypt_mode()
581 } else {
582 item.key.decrypt_mode()
583 };
584
585 let registers = GcmRegisters {
586 text_blocks: len.div_ceil(BLOCK_SIZE),
587 aad_blocks: aad.len().div_ceil(BLOCK_SIZE),
588 remainder_bits: (len * 8) % 128,
589 j0: state.j0,
590 };
591 let transfer = driver.start_gcm_transfer(®isters, output, input, mode, &item.key);
592
593 self.state = DriverState::WaitingForDma {
594 transfer,
595 remaining_bytes: 0,
596 };
597
598 Poll::Pending(false)
599 }
600
601 pub(super) fn finish_gcm(
606 &mut self,
607 item: &mut AesOperation,
608 mut state: NonNull<GcmState>,
609 ) -> Poll {
610 let DriverState::WaitingForDma { transfer, .. } = &mut self.state else {
611 unreachable!()
612 };
613 if !transfer.is_done() {
614 return Poll::Pending(false);
615 }
616
617 let mut tag = [0; 16];
618 transfer.aes_dma.read_tag(&mut tag);
622
623 let DriverState::WaitingForDma { transfer, .. } =
624 core::mem::replace(&mut self.state, DriverState::None)
625 else {
626 unreachable!()
627 };
628 let (driver, _, _) = transfer.wait();
629 driver.clear_interrupt();
630
631 #[cfg(any(soc_internal_memory_cached, dma_can_access_psram))]
633 {
634 let len = item.buffers.output.len();
635 crate::dma::invalidate_received(get_range(
636 item.buffers.output,
637 0,
638 len - len % BLOCK_SIZE,
639 ));
640 }
641
642 #[cfg(dma_can_access_psram)]
643 for buffer in self.unaligned_data_buffers.iter_mut() {
644 if let Some(buffer) = buffer.as_mut() {
645 buffer.write_back();
646 buffer.clear();
648 }
649 *buffer = None;
650 }
651
652 let output = item.buffers.output;
653 let len = output.len();
654 let text_full = len - len % BLOCK_SIZE;
655
656 let mut dma = self.gcm.get_mut();
657 #[cfg(soc_internal_memory_cached)]
658 dma.invalidate();
659 if len > text_full {
660 unsafe {
661 output.cast::<u8>().add(text_full).copy_from_nonoverlapping(
662 NonNull::from(&*dma.output_tail.get_ref()).cast::<u8>(),
663 len - text_full,
664 );
665 }
666 }
667 dma.clear();
668 #[cfg(soc_internal_memory_cached)]
669 dma.writeback();
670
671 let state = unsafe { state.as_mut() };
672 state.result = if item.mode == Operation::Encrypt {
673 unsafe { state.tag.write(tag) };
674 Ok(())
675 } else if tags_equal(&tag, unsafe { state.tag.as_ref() }) {
676 Ok(())
677 } else {
678 unsafe { zero(output) };
679 Err(GcmError::TagMismatch)
680 };
681 unsafe { core::ptr::write_volatile(&mut tag, [0; 16]) };
682
683 self.state = DriverState::Idle(driver);
684
685 Poll::Ready(Status::Completed)
686 }
687}
688
689unsafe fn copy_tail(block: &mut Block, data: NonNull<[u8]>, start: usize) {
692 let len = data.len() - start;
693 unsafe {
694 block
695 .get_mut()
696 .as_mut_ptr()
697 .copy_from_nonoverlapping(data.cast::<u8>().add(start).as_ptr(), len);
698 }
699}
700
701fn get_range(data: NonNull<[u8]>, start: usize, end: usize) -> NonNull<[u8]> {
702 NonNull::slice_from_raw_parts(unsafe { data.cast::<u8>().add(start) }, end - start)
703}
704
705struct Chain<'a> {
707 descriptors: &'a mut [DmaDescriptor],
708 used: usize,
709 #[cfg(dma_can_access_psram)]
710 accesses_psram: bool,
711}
712
713impl<'a> Chain<'a> {
714 fn new(descriptors: &'a mut [DmaDescriptor]) -> Self {
715 Self {
716 descriptors,
717 used: 0,
718 #[cfg(dma_can_access_psram)]
719 accesses_psram: false,
720 }
721 }
722
723 fn append(&mut self, first: usize, count: usize) {
725 if first > 0 {
726 let next = &raw mut self.descriptors[first];
727 self.descriptors[first - 1].next = next;
728 }
729 self.used = first + count;
730 }
731
732 fn linked_from(&self, first: usize) -> usize {
734 let mut count = 1;
735 while !self.descriptors[first + count - 1].next.is_null() {
736 count += 1;
737 }
738 count
739 }
740
741 fn tx(&mut self, data: NonNull<[u8]>) {
742 let first = self.used;
743 let (buffer, len) =
744 unwrap!(unsafe { prepare_for_tx(&mut self.descriptors[first..], data, BLOCK_SIZE) });
745 assert_eq!(len, data.len());
748 cfg_select! {
749 dma_can_access_psram => {
750 self.accesses_psram |= buffer.0.accesses_psram;
751 }
752 _ => {
753 let _ = buffer;
754 }
755 }
756 let count = self.linked_from(first);
757 self.append(first, count);
758 }
759
760 fn tx_block(&mut self, block: &mut Block) {
761 let first = self.used;
762 let descriptor = &mut self.descriptors[first];
763 *descriptor = DmaDescriptor::EMPTY;
764 descriptor.buffer = block.get_mut().as_mut_ptr();
765 descriptor.set_size(BLOCK_SIZE);
766 descriptor.set_length(BLOCK_SIZE);
767 self.append(first, 1);
768 }
769
770 fn rx(
771 &mut self,
772 #[cfg(dma_can_access_psram)] align_buffers: &mut [Option<crate::dma::ManualWritebackBuffer>;
773 2],
774 data: NonNull<[u8]>,
775 ) {
776 let first = self.used;
777 let (buffer, len) = unsafe {
778 prepare_for_rx(
779 &mut self.descriptors[first..],
780 #[cfg(dma_can_access_psram)]
781 align_buffers,
782 data,
783 )
784 };
785 assert_eq!(len, data.len());
786 cfg_select! {
787 dma_can_access_psram => {
788 self.accesses_psram |= buffer.0.accesses_psram;
789 }
790 _ => {
791 let _ = buffer;
792 }
793 }
794 let count = self.linked_from(first);
795 self.append(first, count);
796 }
797
798 fn rx_block(&mut self, block: &mut Block) {
799 let first = self.used;
800 let descriptor = &mut self.descriptors[first];
801 *descriptor = DmaDescriptor::EMPTY;
802 descriptor.buffer = block.get_mut().as_mut_ptr();
803 descriptor.set_size(BLOCK_SIZE);
804 self.append(first, 1);
805 }
806
807 fn finish(self, rx: bool) -> NoBuffer {
809 let last = self.used - 1;
810 for (i, descriptor) in self.descriptors[..self.used].iter_mut().enumerate() {
811 if rx {
812 descriptor.reset_for_rx();
813 } else {
814 descriptor.reset_for_tx(i == last);
815 }
816 }
817 NoBuffer(Preparation {
818 start: self.descriptors.as_mut_ptr(),
819 burst_transfer: BurstConfig::DEFAULT,
820 check_owner: None,
821 auto_write_back: rx,
822 #[cfg(dma_can_access_psram)]
823 accesses_psram: self.accesses_psram,
824 })
825 }
826}