1pub(crate) use esp_riscv_rt::riscv::interrupt::free;
16pub use esp_riscv_rt::TrapFrame;
17use riscv::register::{mcause, mtvec};
18
19#[cfg(not(plic))]
20pub use self::classic::*;
21#[cfg(plic)]
22pub use self::plic::*;
23pub use self::vectored::*;
24use super::InterruptStatus;
25use crate::{
26 pac,
27 peripherals::{Interrupt, INTERRUPT_CORE0},
28 system::Cpu,
29};
30
31#[derive(Copy, Clone, Debug, PartialEq, Eq)]
33#[cfg_attr(feature = "defmt", derive(defmt::Format))]
34pub enum Error {
35 InvalidInterruptPriority,
37 CpuInterruptReserved,
39}
40
41#[cfg_attr(feature = "defmt", derive(defmt::Format))]
43pub enum InterruptKind {
44 Level,
46 Edge,
48}
49
50#[repr(u32)]
54#[derive(Debug, Copy, Clone)]
55#[cfg_attr(feature = "defmt", derive(defmt::Format))]
56pub enum CpuInterrupt {
57 Interrupt1 = 1,
59 Interrupt2,
61 Interrupt3,
63 Interrupt4,
65 Interrupt5,
67 Interrupt6,
69 Interrupt7,
71 Interrupt8,
73 Interrupt9,
75 Interrupt10,
77 Interrupt11,
79 Interrupt12,
81 Interrupt13,
83 Interrupt14,
85 Interrupt15,
87 Interrupt16,
89 Interrupt17,
91 Interrupt18,
93 Interrupt19,
95 Interrupt20,
97 Interrupt21,
99 Interrupt22,
101 Interrupt23,
103 Interrupt24,
105 Interrupt25,
107 Interrupt26,
109 Interrupt27,
111 Interrupt28,
113 Interrupt29,
115 Interrupt30,
117 Interrupt31,
119}
120
121#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
123#[cfg_attr(feature = "defmt", derive(defmt::Format))]
124#[repr(u8)]
125pub enum Priority {
126 None = 0,
128 Priority1,
130 Priority2,
132 Priority3,
134 Priority4,
136 Priority5,
138 Priority6,
140 Priority7,
142 Priority8,
144 Priority9,
146 Priority10,
148 Priority11,
150 Priority12,
152 Priority13,
154 Priority14,
156 Priority15,
158}
159
160impl Priority {
161 pub const fn max() -> Priority {
163 Priority::Priority15
164 }
165
166 pub const fn min() -> Priority {
168 Priority::Priority1
169 }
170}
171
172impl TryFrom<u8> for Priority {
173 type Error = Error;
174
175 fn try_from(value: u8) -> Result<Self, Self::Error> {
176 match value {
177 0 => Ok(Priority::None),
178 1 => Ok(Priority::Priority1),
179 2 => Ok(Priority::Priority2),
180 3 => Ok(Priority::Priority3),
181 4 => Ok(Priority::Priority4),
182 5 => Ok(Priority::Priority5),
183 6 => Ok(Priority::Priority6),
184 7 => Ok(Priority::Priority7),
185 8 => Ok(Priority::Priority8),
186 9 => Ok(Priority::Priority9),
187 10 => Ok(Priority::Priority10),
188 11 => Ok(Priority::Priority11),
189 12 => Ok(Priority::Priority12),
190 13 => Ok(Priority::Priority13),
191 14 => Ok(Priority::Priority14),
192 15 => Ok(Priority::Priority15),
193 _ => Err(Error::InvalidInterruptPriority),
194 }
195 }
196}
197
198#[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
200pub static RESERVED_INTERRUPTS: &[usize] = PRIORITY_TO_INTERRUPT;
201
202#[doc(hidden)]
206#[link_section = ".trap.rust"]
207#[export_name = "_start_trap_rust_hal"]
208pub unsafe extern "C" fn start_trap_rust_hal(trap_frame: *mut TrapFrame) {
209 assert!(
210 mcause::read().is_exception(),
211 "Arrived into _start_trap_rust_hal but mcause is not an exception!"
212 );
213 extern "C" {
214 fn ExceptionHandler(tf: *mut TrapFrame);
215 }
216 ExceptionHandler(trap_frame);
217}
218
219#[doc(hidden)]
220#[no_mangle]
221pub fn _setup_interrupts() {
222 extern "C" {
223 static _vector_table: *const u32;
224 }
225
226 unsafe {
227 for peripheral_interrupt in 0..255 {
231 crate::peripherals::Interrupt::try_from(peripheral_interrupt)
232 .map(|intr| {
233 #[cfg(multi_core)]
234 disable(Cpu::AppCpu, intr);
235 disable(Cpu::ProCpu, intr);
236 })
237 .ok();
238 }
239
240 let vec_table = &_vector_table as *const _ as usize;
241 mtvec::write(vec_table, mtvec::TrapMode::Vectored);
242
243 crate::interrupt::init_vectoring();
244 };
245
246 #[cfg(plic)]
247 unsafe {
248 core::arch::asm!("csrw mie, {0}", in(reg) u32::MAX);
249 }
250}
251
252pub fn enable_direct(
259 interrupt: Interrupt,
260 level: Priority,
261 cpu_interrupt: CpuInterrupt,
262) -> Result<(), Error> {
263 if RESERVED_INTERRUPTS.contains(&(cpu_interrupt as _)) {
264 return Err(Error::CpuInterruptReserved);
265 }
266 if matches!(level, Priority::None) {
267 return Err(Error::InvalidInterruptPriority);
268 }
269 unsafe {
270 map(Cpu::current(), interrupt, cpu_interrupt);
271 set_priority(Cpu::current(), cpu_interrupt, level);
272 enable_cpu_interrupt(cpu_interrupt);
273 }
274 Ok(())
275}
276
277pub fn disable(_core: Cpu, interrupt: Interrupt) {
279 unsafe {
280 let interrupt_number = interrupt as isize;
281 let intr_map_base = crate::soc::registers::INTERRUPT_MAP_BASE as *mut u32;
282
283 intr_map_base
286 .offset(interrupt_number)
287 .write_volatile(DISABLED_CPU_INTERRUPT);
288 }
289}
290
291#[inline]
293pub fn status(_core: Cpu) -> InterruptStatus {
294 cfg_if::cfg_if! {
295 if #[cfg(large_intr_status)] {
296 InterruptStatus::from(
297 INTERRUPT_CORE0::regs().intr_status_reg_0().read().bits(),
298 INTERRUPT_CORE0::regs().intr_status_reg_1().read().bits(),
299 INTERRUPT_CORE0::regs().int_status_reg_2().read().bits(),
300 )
301 } else if #[cfg(very_large_intr_status)] {
302 InterruptStatus::from(
303 INTERRUPT_CORE0::regs().intr_status_reg_0().read().bits(),
304 INTERRUPT_CORE0::regs().intr_status_reg_1().read().bits(),
305 INTERRUPT_CORE0::regs().intr_status_reg_2().read().bits(),
306 INTERRUPT_CORE0::regs().intr_status_reg_3().read().bits(),
307 )
308 } else {
309 InterruptStatus::from(
310 INTERRUPT_CORE0::regs().intr_status_reg_0().read().bits(),
311 INTERRUPT_CORE0::regs().intr_status_reg_1().read().bits(),
312 )
313 }
314 }
315}
316
317pub unsafe fn map(_core: Cpu, interrupt: Interrupt, which: CpuInterrupt) {
323 let interrupt_number = interrupt as isize;
324 let cpu_interrupt_number = which as isize;
325 #[cfg(not(multi_core))]
326 let intr_map_base = crate::soc::registers::INTERRUPT_MAP_BASE as *mut u32;
327 #[cfg(multi_core)]
328 let intr_map_base = match _core {
329 Cpu::ProCpu => crate::soc::registers::INTERRUPT_MAP_BASE as *mut u32,
330 Cpu::AppCpu => crate::soc::registers::INTERRUPT_MAP_BASE_APP_CPU as *mut u32,
331 };
332
333 intr_map_base
334 .offset(interrupt_number)
335 .write_volatile(cpu_interrupt_number as u32 + EXTERNAL_INTERRUPT_OFFSET);
336}
337
338#[inline]
340unsafe fn assigned_cpu_interrupt(interrupt: Interrupt) -> Option<CpuInterrupt> {
341 let interrupt_number = interrupt as isize;
342 let intr_map_base = crate::soc::registers::INTERRUPT_MAP_BASE as *mut u32;
343
344 let cpu_intr = intr_map_base.offset(interrupt_number).read_volatile();
345 if cpu_intr > 0 && cpu_intr != DISABLED_CPU_INTERRUPT {
346 Some(core::mem::transmute::<u32, CpuInterrupt>(
347 cpu_intr - EXTERNAL_INTERRUPT_OFFSET,
348 ))
349 } else {
350 None
351 }
352}
353
354pub(crate) fn bound_cpu_interrupt_for(_cpu: Cpu, interrupt: Interrupt) -> Option<CpuInterrupt> {
355 unsafe { assigned_cpu_interrupt(interrupt) }
356}
357
358mod vectored {
359 use procmacros::ram;
360
361 use super::*;
362
363 #[doc(hidden)]
365 pub(crate) unsafe fn init_vectoring() {
366 for (prio, num) in PRIORITY_TO_INTERRUPT.iter().enumerate() {
367 set_kind(
368 Cpu::current(),
369 core::mem::transmute::<u32, CpuInterrupt>(*num as u32),
370 InterruptKind::Level,
371 );
372 set_priority(
373 Cpu::current(),
374 core::mem::transmute::<u32, CpuInterrupt>(*num as u32),
375 core::mem::transmute::<u8, Priority>((prio as u8) + 1),
376 );
377 enable_cpu_interrupt(core::mem::transmute::<u32, CpuInterrupt>(*num as u32));
378 }
379 }
380
381 #[inline]
384 fn configured_interrupts(
385 core: Cpu,
386 status: InterruptStatus,
387 priority: Priority,
388 ) -> InterruptStatus {
389 unsafe {
390 let mut res = InterruptStatus::empty();
391
392 for interrupt_nr in status.iterator() {
393 if let Some(cpu_interrupt) =
395 assigned_cpu_interrupt(core::mem::transmute::<u16, Interrupt>(
396 interrupt_nr as u16,
397 ))
398 {
399 if priority_by_core(core, cpu_interrupt) == priority {
400 res.set(interrupt_nr as u16);
401 }
402 }
403 }
404 res
405 }
406 }
407
408 pub fn enable(interrupt: Interrupt, level: Priority) -> Result<(), Error> {
413 if matches!(level, Priority::None) {
414 return Err(Error::InvalidInterruptPriority);
415 }
416 unsafe {
417 let cpu_interrupt = core::mem::transmute::<u32, CpuInterrupt>(
418 PRIORITY_TO_INTERRUPT[(level as usize) - 1] as u32,
419 );
420 map(Cpu::current(), interrupt, cpu_interrupt);
421 enable_cpu_interrupt(cpu_interrupt);
422 }
423 Ok(())
424 }
425
426 pub unsafe fn bind_interrupt(interrupt: Interrupt, handler: unsafe extern "C" fn()) {
432 let ptr = &pac::__EXTERNAL_INTERRUPTS[interrupt as usize]._handler as *const _
433 as *mut unsafe extern "C" fn();
434 ptr.write_volatile(handler);
435 }
436
437 pub fn bound_handler(interrupt: Interrupt) -> Option<unsafe extern "C" fn()> {
439 unsafe {
440 let addr = pac::__EXTERNAL_INTERRUPTS[interrupt as usize]._handler;
441 if addr as usize == 0 {
442 return None;
443 }
444
445 Some(addr)
446 }
447 }
448
449 #[no_mangle]
450 #[ram]
451 unsafe fn handle_interrupts(cpu_intr: CpuInterrupt, context: &mut TrapFrame) {
452 let core = Cpu::current();
453 let status = status(core);
454
455 clear(core, cpu_intr);
458 let prio: Priority =
459 unsafe { core::mem::transmute(INTERRUPT_TO_PRIORITY[cpu_intr as usize - 1] as u8) };
460 let configured_interrupts = configured_interrupts(core, status, prio);
461
462 for interrupt_nr in configured_interrupts.iterator() {
463 let interrupt: Interrupt = unsafe { core::mem::transmute(interrupt_nr as u16) };
465 handle_interrupt(interrupt, context);
466 }
467 }
468
469 #[inline(always)]
470 unsafe fn handle_interrupt(interrupt: Interrupt, save_frame: &mut TrapFrame) {
471 extern "C" {
472 fn EspDefaultHandler(interrupt: Interrupt);
474 }
475
476 let handler = pac::__EXTERNAL_INTERRUPTS[interrupt as usize]._handler;
477
478 if core::ptr::eq(
479 handler as *const _,
480 EspDefaultHandler as *const unsafe extern "C" fn(),
481 ) {
482 EspDefaultHandler(interrupt);
483 } else {
484 let handler: fn(&mut TrapFrame) =
485 core::mem::transmute::<unsafe extern "C" fn(), fn(&mut TrapFrame)>(handler);
486 handler(save_frame);
487 }
488 }
489
490 macro_rules! interrupt_handler {
517 ($num:literal) => {
518 core::arch::global_asm! {
519 concat!(
520 r#"
521 .section .rwtext, "ax"
522 .global interrupt"#,$num,r#"
523
524 interrupt"#,$num,r#":
525 mv a1, a0
526 li a0,"#,$num,r#"
527 j handle_interrupts
528 "#
529 )
530 }
531 };
532 }
533
534 interrupt_handler!(1);
535 interrupt_handler!(2);
536 interrupt_handler!(3);
537 interrupt_handler!(4);
538 interrupt_handler!(5);
539 interrupt_handler!(6);
540 interrupt_handler!(7);
541 interrupt_handler!(8);
542 interrupt_handler!(9);
543 interrupt_handler!(10);
544 interrupt_handler!(11);
545 interrupt_handler!(12);
546 interrupt_handler!(13);
547 interrupt_handler!(14);
548 interrupt_handler!(15);
549
550 #[cfg(plic)]
551 interrupt_handler!(16);
552 #[cfg(plic)]
553 interrupt_handler!(17);
554 #[cfg(plic)]
555 interrupt_handler!(18);
556 #[cfg(plic)]
557 interrupt_handler!(19);
558}
559
560#[cfg(not(plic))]
561mod classic {
562 use super::{CpuInterrupt, InterruptKind, Priority};
563 use crate::{peripherals::INTERRUPT_CORE0, system::Cpu};
564
565 #[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
566 pub(super) static DISABLED_CPU_INTERRUPT: u32 = 0;
567
568 #[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
569 pub(super) static EXTERNAL_INTERRUPT_OFFSET: u32 = 0;
570
571 #[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
572 pub(super) static PRIORITY_TO_INTERRUPT: &[usize] =
573 &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
574
575 #[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
576 pub(super) static INTERRUPT_TO_PRIORITY: &[usize] =
577 &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
578
579 pub unsafe fn enable_cpu_interrupt(which: CpuInterrupt) {
585 let cpu_interrupt_number = which as isize;
586 let intr = INTERRUPT_CORE0::regs();
587 intr.cpu_int_enable()
588 .modify(|r, w| w.bits((1 << cpu_interrupt_number) | r.bits()));
589 }
590
591 pub fn set_kind(_core: Cpu, which: CpuInterrupt, kind: InterruptKind) {
596 unsafe {
597 let intr = INTERRUPT_CORE0::regs();
598 let cpu_interrupt_number = which as isize;
599
600 let interrupt_type = match kind {
601 InterruptKind::Level => 0,
602 InterruptKind::Edge => 1,
603 };
604 intr.cpu_int_type().modify(|r, w| {
605 w.bits(
606 r.bits() & !(1 << cpu_interrupt_number)
607 | (interrupt_type << cpu_interrupt_number),
608 )
609 });
610 }
611 }
612
613 pub unsafe fn set_priority(_core: Cpu, which: CpuInterrupt, priority: Priority) {
620 let intr = INTERRUPT_CORE0::regs();
621 intr.cpu_int_pri(which as usize)
622 .write(|w| w.map().bits(priority as u8));
623 }
624
625 #[inline]
627 pub fn clear(_core: Cpu, which: CpuInterrupt) {
628 unsafe {
629 let cpu_interrupt_number = which as isize;
630 let intr = INTERRUPT_CORE0::regs();
631 intr.cpu_int_clear()
632 .write(|w| w.bits(1 << cpu_interrupt_number));
633 }
634 }
635
636 #[inline]
638 pub(super) fn priority_by_core(_core: Cpu, cpu_interrupt: CpuInterrupt) -> Priority {
639 unsafe { priority(cpu_interrupt) }
640 }
641
642 #[inline]
644 pub(super) unsafe extern "C" fn priority(cpu_interrupt: CpuInterrupt) -> Priority {
645 let intr = INTERRUPT_CORE0::regs();
646 core::mem::transmute::<u8, Priority>(
647 intr.cpu_int_pri(cpu_interrupt as usize).read().map().bits(),
648 )
649 }
650 #[no_mangle]
651 #[link_section = ".trap"]
652 pub(super) unsafe extern "C" fn _handle_priority() -> u32 {
653 use super::mcause;
654 let interrupt_id: usize = mcause::read().code(); let intr = INTERRUPT_CORE0::regs();
656 let interrupt_priority = intr
657 .cpu_int_pri(0)
658 .as_ptr()
659 .add(interrupt_id)
660 .read_volatile();
661
662 let prev_interrupt_priority = intr.cpu_int_thresh().read().bits();
663 if interrupt_priority < 15 {
664 intr.cpu_int_thresh()
666 .write(|w| w.bits(interrupt_priority + 1)); unsafe {
668 riscv::interrupt::enable();
669 }
670 }
671 prev_interrupt_priority
672 }
673 #[no_mangle]
674 #[link_section = ".trap"]
675 pub(super) unsafe extern "C" fn _restore_priority(stored_prio: u32) {
676 riscv::interrupt::disable();
677 let intr = INTERRUPT_CORE0::regs();
678 intr.cpu_int_thresh().write(|w| w.bits(stored_prio));
679 }
680
681 pub(crate) fn current_runlevel() -> Priority {
683 let intr = INTERRUPT_CORE0::regs();
684 let prev_interrupt_priority = intr.cpu_int_thresh().read().bits().saturating_sub(1) as u8;
685
686 unwrap!(Priority::try_from(prev_interrupt_priority))
687 }
688
689 pub(crate) unsafe fn change_current_runlevel(level: Priority) -> Priority {
698 let prev_interrupt_priority = current_runlevel();
699
700 INTERRUPT_CORE0::regs()
703 .cpu_int_thresh()
704 .write(|w| w.bits(level as u32 + 1));
705
706 prev_interrupt_priority
707 }
708}
709
710#[cfg(plic)]
711mod plic {
712 use super::{CpuInterrupt, InterruptKind, Priority};
713 use crate::{peripherals::PLIC_MX, system::Cpu};
714
715 #[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
716 pub(super) static DISABLED_CPU_INTERRUPT: u32 = 31;
717
718 #[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
719 pub(super) static EXTERNAL_INTERRUPT_OFFSET: u32 = 0;
720
721 #[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
725 pub(super) static PRIORITY_TO_INTERRUPT: &[usize] =
726 &[1, 2, 5, 6, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19];
727
728 #[cfg_attr(place_switch_tables_in_ram, link_section = ".rwtext")]
729 pub(super) static INTERRUPT_TO_PRIORITY: &[usize] = &[
730 1, 2, 0, 0, 3, 4, 0, 0, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
731 ];
732
733 pub unsafe fn enable_cpu_interrupt(which: CpuInterrupt) {
739 unsafe {
740 PLIC_MX::regs().mxint_enable().modify(|r, w| {
741 let old = r.cpu_mxint_enable().bits();
742 let new = old | (1 << (which as isize));
743 w.cpu_mxint_enable().bits(new)
744 });
745 }
746 }
747
748 pub fn set_kind(_core: Cpu, which: CpuInterrupt, kind: InterruptKind) {
753 let interrupt_type = match kind {
754 InterruptKind::Level => 0,
755 InterruptKind::Edge => 1,
756 };
757
758 unsafe {
759 PLIC_MX::regs().mxint_type().modify(|r, w| {
760 let old = r.cpu_mxint_type().bits();
761 let new = old & !(1 << (which as isize)) | (interrupt_type << (which as isize));
762 w.cpu_mxint_type().bits(new)
763 });
764 }
765 }
766
767 pub unsafe fn set_priority(_core: Cpu, which: CpuInterrupt, priority: Priority) {
774 unsafe {
775 PLIC_MX::regs()
776 .mxint_pri(which as usize)
777 .modify(|_, w| w.cpu_mxint_pri().bits(priority as u8));
778 }
779 }
780
781 #[inline]
783 pub fn clear(_core: Cpu, which: CpuInterrupt) {
784 unsafe {
785 PLIC_MX::regs().mxint_clear().modify(|r, w| {
786 let old = r.cpu_mxint_clear().bits();
787 let new = old | (1 << (which as isize));
788 w.cpu_mxint_clear().bits(new)
789 });
790 }
791 }
792
793 #[inline]
795 pub(super) unsafe extern "C" fn priority_by_core(
796 _core: Cpu,
797 cpu_interrupt: CpuInterrupt,
798 ) -> Priority {
799 unsafe { priority(cpu_interrupt) }
800 }
801
802 #[inline]
804 pub(super) unsafe extern "C" fn priority(cpu_interrupt: CpuInterrupt) -> Priority {
805 let prio = PLIC_MX::regs()
806 .mxint_pri(cpu_interrupt as usize)
807 .read()
808 .cpu_mxint_pri()
809 .bits();
810 core::mem::transmute::<u8, Priority>(prio)
811 }
812 #[no_mangle]
813 #[link_section = ".trap"]
814 pub(super) unsafe extern "C" fn _handle_priority() -> u32 {
815 let interrupt_id: usize = super::mcause::read().code(); let interrupt_priority = PLIC_MX::regs()
817 .mxint_pri(interrupt_id)
818 .read()
819 .cpu_mxint_pri()
820 .bits();
821
822 let prev_interrupt_priority = PLIC_MX::regs()
823 .mxint_thresh()
824 .read()
825 .cpu_mxint_thresh()
826 .bits();
827 if interrupt_priority < 15 {
828 PLIC_MX::regs()
830 .mxint_thresh()
831 .write(|w| w.cpu_mxint_thresh().bits(interrupt_priority + 1));
832 unsafe {
833 riscv::interrupt::enable();
834 }
835 }
836 prev_interrupt_priority as u32
837 }
838 #[no_mangle]
839 #[link_section = ".trap"]
840 pub(super) unsafe extern "C" fn _restore_priority(stored_prio: u32) {
841 riscv::interrupt::disable();
842 PLIC_MX::regs()
843 .mxint_thresh()
844 .write(|w| w.cpu_mxint_thresh().bits(stored_prio as u8));
845 }
846
847 pub(crate) fn current_runlevel() -> Priority {
849 let prev_interrupt_priority = PLIC_MX::regs()
850 .mxint_thresh()
851 .read()
852 .cpu_mxint_thresh()
853 .bits()
854 .saturating_sub(1);
855
856 unwrap!(Priority::try_from(prev_interrupt_priority))
857 }
858
859 pub(crate) unsafe fn change_current_runlevel(level: Priority) -> Priority {
868 let prev_interrupt_priority = current_runlevel();
869
870 PLIC_MX::regs()
873 .mxint_thresh()
874 .write(|w| w.cpu_mxint_thresh().bits(level as u8 + 1));
875
876 prev_interrupt_priority
877 }
878}