1use core::sync::atomic::{Ordering, fence};
8
9use crate::{dma::aligned::InternalMemory, reg_access::VolatileCell};
10
11pub const MAX_FRAME_SIZE: usize = 1524;
13pub const MIN_RX_FRAME_SIZE: usize = 14;
15
16const TX_CHECKSUM_OFFLOAD: bool = property!("ethernet.tx_checksum_offload");
17
18pub const TDES0_OWN: u32 = 1 << 31;
22pub const TDES0_IC: u32 = 1 << 30;
24pub const TDES0_LS: u32 = 1 << 29;
26pub const TDES0_FS: u32 = 1 << 28;
28pub const TDES0_CHAINED: u32 = 1 << 20;
30pub const TDES0_CIC_SHIFT: u32 = 22;
33pub const TDES0_CIC_FULL: u32 = 0b11 << TDES0_CIC_SHIFT;
35
36pub const RDES0_OWN: u32 = 1 << 31;
40pub const RDES0_FL_SHIFT: u32 = 16;
42pub const RDES0_FL_MASK: u32 = 0x3fff << RDES0_FL_SHIFT;
44pub const RDES0_ES: u32 = 1 << 15;
46pub const RDES0_FS: u32 = 1 << 9;
48pub const RDES0_LS: u32 = 1 << 8;
50
51pub const RDES1_BUF1_SIZE_MASK: u32 = 0x1fff;
53pub const RDES1_CHAINED: u32 = 1 << 14;
55
56cfg_select! {
61 ethernet_rx_checksum_offload => {
62 pub const RDES0_ESA: u32 = 1 << 0;
64 pub const RDES4_IP_HEADER_ERROR: u32 = 1 << 3;
66 pub const RDES4_IP_PAYLOAD_ERROR: u32 = 1 << 4;
68 pub const RDES4_IP_CHECKSUM_BYPASSED: u32 = 1 << 5;
70 pub const RDES4_IPV4_PACKET: u32 = 1 << 6;
72 pub const RDES4_IPV6_PACKET: u32 = 1 << 7;
74 }
75 _ => {}
76}
77
78#[derive(Clone, Copy, Debug, Eq, PartialEq)]
82pub enum OwnedBy {
83 Cpu,
85 Dma,
87}
88
89#[repr(C)]
94#[cfg_attr(ethernet_dma_alignment = "8", repr(align(8)))]
95pub struct TDes {
96 pub(super) tdes0: VolatileCell<u32>,
97 pub(super) tdes1: VolatileCell<u32>,
98 pub(super) buf_addr: VolatileCell<u32>,
99 pub(super) next_desc: VolatileCell<u32>,
100 _tdes4: VolatileCell<u32>,
102 _tdes5: VolatileCell<u32>,
103 _tdes6: VolatileCell<u32>,
104 _tdes7: VolatileCell<u32>,
105}
106
107impl TDes {
108 pub const fn new_zeroed() -> Self {
110 Self {
111 tdes0: VolatileCell::new(0),
112 tdes1: VolatileCell::new(0),
113 buf_addr: VolatileCell::new(0),
114 next_desc: VolatileCell::new(0),
115 _tdes4: VolatileCell::new(0),
116 _tdes5: VolatileCell::new(0),
117 _tdes6: VolatileCell::new(0),
118 _tdes7: VolatileCell::new(0),
119 }
120 }
121
122 pub fn owned_by(&self) -> OwnedBy {
124 if self.tdes0.get() & TDES0_OWN != 0 {
125 OwnedBy::Dma
126 } else {
127 OwnedBy::Cpu
128 }
129 }
130
131 pub fn set_owned_by(&mut self, owner: OwnedBy) {
133 let v = self.tdes0.get();
134 self.tdes0.set(match owner {
135 OwnedBy::Cpu => v & !TDES0_OWN,
136 OwnedBy::Dma => v | TDES0_OWN,
137 });
138 }
139
140 fn set_chained(&mut self) {
141 self.tdes0.set(self.tdes0.get() | TDES0_CHAINED);
142 }
143
144 fn set_len_and_flags(&mut self, len: usize) {
150 self.tdes1.set(len as u32 & RDES1_BUF1_SIZE_MASK);
151 let cic = if TX_CHECKSUM_OFFLOAD {
152 TDES0_CIC_FULL
153 } else {
154 0
155 };
156 self.tdes0
157 .set((self.tdes0.get() & TDES0_CHAINED) | TDES0_FS | TDES0_LS | TDES0_IC | cic);
158 }
159
160 fn set_buffer_addr(&mut self, addr: *const u8) {
161 self.buf_addr.set(addr as u32);
162 }
163
164 fn set_next_desc(&mut self, addr: *const TDes) {
165 self.next_desc.set(addr as u32);
166 }
167}
168
169#[repr(C)]
173#[cfg_attr(ethernet_dma_alignment = "8", repr(align(8)))]
174pub struct RDes {
175 pub(super) rdes0: VolatileCell<u32>,
176 pub(super) rdes1: VolatileCell<u32>,
177 pub(super) buf_addr: VolatileCell<u32>,
178 pub(super) next_desc: VolatileCell<u32>,
179 #[cfg_attr(
181 not(ethernet_rx_checksum_offload),
182 allow(dead_code, reason = "only read when RX checksum offload is enabled")
183 )]
184 rdes4: VolatileCell<u32>,
185 _rdes5: VolatileCell<u32>,
186 _rdes6: VolatileCell<u32>,
187 _rdes7: VolatileCell<u32>,
188}
189
190impl RDes {
191 pub const fn new_zeroed() -> Self {
193 Self {
194 rdes0: VolatileCell::new(0),
195 rdes1: VolatileCell::new(0),
196 buf_addr: VolatileCell::new(0),
197 next_desc: VolatileCell::new(0),
198 rdes4: VolatileCell::new(0),
199 _rdes5: VolatileCell::new(0),
200 _rdes6: VolatileCell::new(0),
201 _rdes7: VolatileCell::new(0),
202 }
203 }
204
205 pub fn owned_by(&self) -> OwnedBy {
207 if self.rdes0.get() & RDES0_OWN != 0 {
208 OwnedBy::Dma
209 } else {
210 OwnedBy::Cpu
211 }
212 }
213
214 fn set_rdes0(&mut self, value: u32) {
215 self.rdes0.set(value);
216 }
217
218 fn set_owned_by(&mut self, owner: OwnedBy) {
219 let v = self.rdes0.get();
220 self.set_rdes0(match owner {
221 OwnedBy::Cpu => v & !RDES0_OWN,
222 OwnedBy::Dma => v | RDES0_OWN,
223 });
224 }
225
226 fn is_complete_frame(&self) -> bool {
227 let s = self.rdes0.get();
228 s & RDES0_FS != 0 && s & RDES0_LS != 0
229 }
230
231 fn frame_len(&self) -> usize {
232 ((self.rdes0.get() & RDES0_FL_MASK) >> RDES0_FL_SHIFT) as usize
233 }
234
235 #[cfg(ethernet_rx_checksum_offload)]
243 fn checksum_error(&self) -> bool {
244 if self.rdes0.get() & RDES0_ESA == 0 {
246 return false;
247 }
248
249 let ext = self.rdes4.get();
250
251 let is_ip = ext & (RDES4_IPV4_PACKET | RDES4_IPV6_PACKET) != 0;
253 let bypassed = ext & RDES4_IP_CHECKSUM_BYPASSED != 0;
255
256 is_ip && !bypassed && ext & (RDES4_IP_HEADER_ERROR | RDES4_IP_PAYLOAD_ERROR) != 0
257 }
258
259 fn configure_buffer(&mut self, size: usize) {
260 self.rdes1.set(
261 (self.rdes1.get() & !RDES1_BUF1_SIZE_MASK)
262 | (size as u32 & RDES1_BUF1_SIZE_MASK)
263 | RDES1_CHAINED,
264 );
265 }
266
267 fn set_buffer_addr(&mut self, addr: *const u8) {
268 self.buf_addr.set(addr as u32);
269 }
270
271 fn set_next_desc(&mut self, addr: *const RDes) {
272 self.next_desc.set(addr as u32);
273 }
274}
275
276#[repr(C)]
280#[cfg_attr(ethernet_dma_alignment = "8", repr(align(8)))]
281pub(crate) struct Buffer([u8; MAX_FRAME_SIZE]);
282
283cfg_select! {
284 any(ethernet_dma_alignment = "4", ethernet_dma_alignment = "8") => {}
285 _ => {
286 compile_error!("unsupported Ethernet DMA alignment");
287 }
288}
289
290pub struct EthernetDmaStorage<const RX: usize, const TX: usize> {
295 pub(super) rx_descs: [InternalMemory<RDes>; RX],
296 pub(super) tx_descs: [InternalMemory<TDes>; TX],
297 pub(super) rx_bufs: [InternalMemory<Buffer>; RX],
298 pub(super) tx_bufs: [InternalMemory<Buffer>; TX],
299}
300
301impl<const RX: usize, const TX: usize> Default for EthernetDmaStorage<RX, TX> {
302 fn default() -> Self {
303 Self::new()
304 }
305}
306
307impl<const RX: usize, const TX: usize> EthernetDmaStorage<RX, TX> {
308 pub const fn new() -> Self {
310 Self {
311 rx_descs: [const { InternalMemory::new(RDes::new_zeroed()) }; RX],
312 tx_descs: [const { InternalMemory::new(TDes::new_zeroed()) }; TX],
313 rx_bufs: [const { InternalMemory::new(Buffer([0u8; MAX_FRAME_SIZE])) }; RX],
314 tx_bufs: [const { InternalMemory::new(Buffer([0u8; MAX_FRAME_SIZE])) }; TX],
315 }
316 }
317}
318
319unsafe impl<const RX: usize, const TX: usize> Send for EthernetDmaStorage<RX, TX> {}
322unsafe impl<const RX: usize, const TX: usize> Sync for EthernetDmaStorage<RX, TX> {}
323
324pub struct TDesRing<'a> {
328 descriptors: &'a mut [InternalMemory<TDes>],
329 buffers: &'a mut [InternalMemory<Buffer>],
330 index: usize,
331}
332
333impl<'a> TDesRing<'a> {
334 pub fn new(
336 descriptors: &'a mut [InternalMemory<TDes>],
337 buffers: &'a mut [InternalMemory<Buffer>],
338 ) -> Self {
339 assert!(!descriptors.is_empty());
340 assert_eq!(descriptors.len(), buffers.len());
341
342 let mut ring = Self {
343 descriptors,
344 buffers,
345 index: 0,
346 };
347 ring.reset();
348 ring
349 }
350
351 pub(crate) fn len(&self) -> usize {
352 self.descriptors.len()
353 }
354
355 pub fn reset(&mut self) {
359 let n = self.descriptors.len();
360 for i in 0..n {
361 let next = self.descriptors[(i + 1) % n].as_ptr();
362 let buf_addr = self.buffers[i].as_ptr().cast::<u8>();
363
364 let mut desc = self.descriptors[i].get_mut();
365 desc.tdes0.set(0);
366 desc.tdes1.set(0);
367 desc.set_chained();
368 desc.set_buffer_addr(buf_addr);
369 desc.set_next_desc(next);
370 desc.set_owned_by(OwnedBy::Cpu);
371 #[cfg(soc_internal_memory_cached)]
372 self.descriptors[i].get_mut().writeback();
373 }
374 self.index = 0;
375 fence(Ordering::Release);
376 }
377
378 pub fn base_ptr(&self) -> *const TDes {
380 self.descriptors[0].as_ptr()
381 }
382
383 pub fn transmit(&mut self, frame: &[u8]) -> Result<(), TxError> {
388 if frame.len() > MAX_FRAME_SIZE {
389 return Err(TxError::FrameTooLarge);
390 }
391
392 if let Some(buf) = self.available_buf() {
393 buf[..frame.len()].copy_from_slice(frame);
394 self.commit(frame.len());
395 Ok(())
396 } else {
397 Err(TxError::RingFull)
398 }
399 }
400
401 pub fn has_capacity(&self) -> bool {
403 let desc = self.descriptors[self.index].get_ref();
404 #[cfg(soc_internal_memory_cached)]
405 desc.invalidate();
406 fence(Ordering::Acquire);
407 desc.owned_by() == OwnedBy::Cpu
408 }
409
410 pub fn available_buf(&mut self) -> Option<&mut [u8; MAX_FRAME_SIZE]> {
416 if self.has_capacity() {
417 let idx = self.index;
418 Some(&mut self.buffers[idx].get_mut().into_inner().0)
419 } else {
420 None
421 }
422 }
423
424 pub fn commit(&mut self, len: usize) {
430 let idx = self.index;
431 let n = self.descriptors.len();
432
433 #[cfg(soc_internal_memory_cached)]
434 self.buffers[idx].get_mut().writeback();
435
436 let mut desc = self.descriptors[idx].get_mut();
437 desc.set_len_and_flags(len);
438 desc.set_owned_by(OwnedBy::Dma);
439
440 #[cfg(soc_internal_memory_cached)]
441 desc.writeback();
442
443 fence(Ordering::Release);
444 self.index = (idx + 1) % n;
445 }
446}
447
448#[derive(Clone, Copy, Debug, Eq, PartialEq)]
450pub enum TxError {
451 RingFull,
453 FrameTooLarge,
455}
456
457pub struct RDesRing<'a> {
461 descriptors: &'a mut [InternalMemory<RDes>],
462 buffers: &'a mut [InternalMemory<Buffer>],
463 index: usize,
464}
465
466impl<'a> RDesRing<'a> {
467 pub fn new(
469 descriptors: &'a mut [InternalMemory<RDes>],
470 buffers: &'a mut [InternalMemory<Buffer>],
471 ) -> Self {
472 assert!(!descriptors.is_empty());
473 assert_eq!(descriptors.len(), buffers.len());
474
475 let mut ring = Self {
476 descriptors,
477 buffers,
478 index: 0,
479 };
480 ring.reset();
481 ring
482 }
483
484 pub fn reset(&mut self) {
488 let n = self.descriptors.len();
489 for i in 0..n {
490 let next = self.descriptors[(i + 1) % n].as_ptr();
491 let buf_addr = self.buffers[i].as_ptr().cast::<u8>();
492
493 #[cfg(soc_internal_memory_cached)]
494 self.buffers[i].get_mut().invalidate();
495
496 let mut desc = self.descriptors[i].get_mut();
497 desc.rdes0.set(0);
498 desc.configure_buffer(core::mem::size_of::<Buffer>());
499 desc.set_buffer_addr(buf_addr);
500 desc.set_next_desc(next);
501 desc.set_owned_by(OwnedBy::Dma);
502 #[cfg(soc_internal_memory_cached)]
503 desc.writeback();
504 }
505 self.index = 0;
506 fence(Ordering::Release);
507 }
508
509 pub fn base_ptr(&self) -> *const RDes {
511 self.descriptors[0].as_ptr()
512 }
513
514 pub fn receive(&mut self) -> Option<&mut [u8]> {
521 loop {
522 let idx = self.index;
523
524 let len = {
528 let desc = self.descriptors[idx].get_ref();
529 #[cfg(soc_internal_memory_cached)]
530 desc.invalidate();
531 fence(Ordering::Acquire);
532 if desc.owned_by() != OwnedBy::Cpu {
533 return None;
534 }
535
536 let status = desc.rdes0.get();
537 let is_complete = desc.is_complete_frame();
538 let frame_len = desc.frame_len();
539
540 let checksum_bad = cfg_select! {
543 ethernet_rx_checksum_offload => desc.checksum_error(),
544 _ => false,
545 };
546
547 if status & RDES0_ES != 0 || !is_complete || checksum_bad {
548 None
549 } else {
550 let len = frame_len.saturating_sub(4);
552 if (MIN_RX_FRAME_SIZE..=MAX_FRAME_SIZE).contains(&len) {
553 Some(len)
554 } else {
555 None
556 }
557 }
558 };
559
560 let Some(len) = len else {
561 self.recycle_current();
562 continue;
563 };
564
565 #[cfg(soc_internal_memory_cached)]
566 self.buffers[idx].get_mut().invalidate();
567 fence(Ordering::Acquire);
568 return Some(&mut self.buffers[idx].get_mut().into_inner().0[..len]);
569 }
570 }
571
572 pub fn pop(&mut self) {
576 self.recycle_current();
577 }
578
579 pub fn has_packet(&self) -> bool {
581 let desc = self.descriptors[self.index].get_ref();
582 #[cfg(soc_internal_memory_cached)]
583 desc.invalidate();
584 fence(Ordering::Acquire);
585 desc.owned_by() == OwnedBy::Cpu
586 }
587
588 fn recycle_current(&mut self) {
589 let idx = self.index;
590 let n = self.descriptors.len();
591 let mut desc = self.descriptors[idx].get_mut();
592 desc.set_rdes0(RDES0_OWN);
593 #[cfg(soc_internal_memory_cached)]
594 desc.writeback();
595 fence(Ordering::Release);
596 self.index = (idx + 1) % n;
597 }
598}