1#[cfg(feature = "rt")]
13#[instability::unstable]
14pub use esp_riscv_rt::TrapFrame;
15
16#[cfg_attr(interrupt_controller = "riscv_basic", path = "riscv/basic.rs")]
17#[cfg_attr(interrupt_controller = "plic", path = "riscv/plic.rs")]
18#[cfg_attr(interrupt_controller = "clic", path = "riscv/clic.rs")]
19mod cpu_int;
20
21#[inline]
23pub(crate) fn free<F, R>(f: F) -> R
24where
25 F: FnOnce() -> R,
26{
27 use esp_sync::raw::{RawLock, SingleCoreInterruptLock};
28 let irq_token = unsafe { SingleCoreInterruptLock.enter() };
29
30 let r = f();
31
32 unsafe { SingleCoreInterruptLock.exit(irq_token) };
33
34 r
35}
36
37use crate::{
38 interrupt::{PriorityError, RunLevel},
39 peripherals::Interrupt,
40 system::Cpu,
41};
42
43#[cfg_attr(feature = "defmt", derive(defmt::Format))]
45#[instability::unstable]
46pub enum InterruptKind {
47 Level,
49 Edge,
51}
52
53for_each_interrupt!(
54 (all $( ([$class:ident $idx_in_class:literal] $n:literal) ),*) => {
55 paste::paste! {
56 #[repr(u32)]
58 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
59 #[cfg_attr(feature = "defmt", derive(defmt::Format))]
60 #[instability::unstable]
61 pub enum CpuInterrupt {
62 $(
63 #[doc = concat!(" Interrupt number ", stringify!($n), ".")]
64 [<Interrupt $n>] = $n,
65 )*
66 }
67
68 impl CpuInterrupt {
69 #[inline]
70 pub(crate) fn from_u32(n: u32) -> Option<Self> {
71 match n {
72 $(n if n == $n && n != DISABLED_CPU_INTERRUPT => Some(Self:: [<Interrupt $n>]),)*
73 _ => None
74 }
75 }
76 }
77 }
78 };
79);
80
81for_each_classified_interrupt!(
82 (context_switch ([$class:ident $idx_in_class:literal] $n:literal)) => {
83 paste::paste! {
84 pub(crate) const IPC_INTERRUPT: CpuInterrupt = CpuInterrupt::[<Interrupt $n>];
85 }
86 };
87
88 (direct_bindable $( ([$class:ident $idx_in_class:literal] $n:literal) ),* ) => {
89 paste::paste! {
90 #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
92 #[cfg_attr(feature = "defmt", derive(defmt::Format))]
93 #[instability::unstable]
94 pub enum DirectBindableCpuInterrupt {
95 $(
96 #[doc = concat!(" Direct bindable CPU interrupt number ", stringify!($idx_in_class), ".")]
97 #[doc = " "]
98 #[doc = concat!(" Corresponds to CPU interrupt ", stringify!($n), ".")]
99 [<Interrupt $idx_in_class>] = $n,
100 )*
101 }
102
103 impl From<DirectBindableCpuInterrupt> for CpuInterrupt {
104 fn from(bindable: DirectBindableCpuInterrupt) -> CpuInterrupt {
105 match bindable {
106 $(
107 DirectBindableCpuInterrupt::[<Interrupt $idx_in_class>] => CpuInterrupt::[<Interrupt $n>],
108 )*
109 }
110 }
111 }
112 }
113 };
114);
115
116impl CpuInterrupt {
117 #[inline]
118 #[cfg(feature = "rt")]
119 pub(crate) fn is_vectored(self) -> bool {
120 const VECTORED_CPU_INTERRUPT_RANGE: core::ops::RangeInclusive<u32> = PRIORITY_TO_INTERRUPT
122 [0] as u32
123 ..=PRIORITY_TO_INTERRUPT[PRIORITY_TO_INTERRUPT.len() - 1] as u32;
124 VECTORED_CPU_INTERRUPT_RANGE.contains(&(self as u32))
125 }
126
127 #[inline]
129 #[instability::unstable]
130 pub fn enable(self) {
131 cpu_int::enable_cpu_interrupt_raw(self as u32);
132 }
133
134 #[inline]
136 #[instability::unstable]
137 pub fn clear(self) {
138 cpu_int::clear_raw(self as u32);
139 }
140
141 #[inline]
147 #[instability::unstable]
148 pub fn set_kind(self, kind: InterruptKind) {
149 cpu_int::set_kind_raw(self as u32, kind);
150 }
151
152 #[inline]
154 #[instability::unstable]
155 pub fn set_priority(self, priority: Priority) {
156 cpu_int::set_priority_raw(self as u32, priority);
157 }
158
159 #[inline]
161 #[instability::unstable]
162 pub fn priority(self) -> Priority {
163 unwrap!(Priority::try_from_u32(self.level()))
164 }
165
166 #[inline]
167 pub(crate) fn level(self) -> u32 {
168 cpu_int::cpu_interrupt_priority_raw(self as u32) as u32
169 }
170}
171
172for_each_interrupt_priority!(
173 (all $( ($idx:literal, $n:literal, $ident:ident, $level:ident) ),*) => {
174 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
179 #[cfg_attr(feature = "defmt", derive(defmt::Format))]
180 #[repr(u8)]
181 pub enum Priority {
182 $(
183 #[doc = concat!(" Priority level ", stringify!($n), ".")]
184 $ident = $n,
185 )*
186 }
187
188 impl Priority {
189 fn iter() -> impl Iterator<Item = Priority> {
190 [$(Priority::$ident,)*].into_iter()
191 }
192 }
193
194 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
196 #[cfg_attr(feature = "defmt", derive(defmt::Format))]
197 #[repr(u8)]
198 pub enum ElevatedRunLevel {
199 $(
200 #[doc = concat!("Run level ", stringify!($n), ".")]
201 $level = $n,
202 )*
203 }
204
205 impl ElevatedRunLevel {
206 pub const fn from_priority(priority: Priority) -> Self {
208 match priority {
209 $(Priority::$ident => Self::$level,)*
210 }
211 }
212 }
213 };
214);
215
216impl Priority {
217 #[allow(unused_assignments)]
219 #[instability::unstable]
220 pub const fn max() -> Priority {
221 const {
222 let mut last = Self::min();
223 for_each_interrupt_priority!(
224 ($_idx:literal, $_n:literal, $ident:ident, $_level:ident) => {
225 last = Self::$ident;
226 };
227 );
228 last
229 }
230 }
231
232 pub const fn min() -> Priority {
234 Priority::Priority1
235 }
236
237 pub(crate) fn try_from_u32(priority: u32) -> Result<Self, PriorityError> {
238 let result;
239 for_each_interrupt_priority!(
240 (all $( ($idx:literal, $n:literal, $ident:ident, $_level:ident) ),*) => {
241 result = match priority {
242 $($n => Ok(Priority::$ident),)*
243 _ => Err(PriorityError::InvalidInterruptPriority),
244 }
245 };
246 );
247 result
248 }
249}
250
251impl ElevatedRunLevel {
252 #[instability::unstable]
254 pub const fn max() -> ElevatedRunLevel {
255 Self::from_priority(Priority::max())
256 }
257
258 pub const fn min() -> ElevatedRunLevel {
260 Self::from_priority(Priority::min())
261 }
262
263 pub(crate) fn try_from_u32(level: u32) -> Result<Self, PriorityError> {
264 Priority::try_from_u32(level).map(Self::from_priority)
265 }
266}
267
268#[instability::unstable]
269impl TryFrom<u32> for ElevatedRunLevel {
270 type Error = PriorityError;
271
272 fn try_from(value: u32) -> Result<Self, Self::Error> {
273 Self::try_from_u32(value)
274 }
275}
276
277impl From<Priority> for ElevatedRunLevel {
278 fn from(priority: Priority) -> Self {
279 Self::from_priority(priority)
280 }
281}
282
283#[cfg_attr(place_switch_tables_in_ram, unsafe(link_section = ".rwtext"))]
284pub(super) static DISABLED_CPU_INTERRUPT: u32 = property!("interrupts.disabled_interrupt");
285
286const VECTOR_COUNT: usize = const {
288 let mut count = 0;
289 for_each_interrupt!(([vector $n:tt] $_:literal) => { count += 1; };);
290
291 core::assert!(count == Priority::max() as usize);
292
293 count
294};
295
296#[cfg_attr(place_switch_tables_in_ram, unsafe(link_section = ".rwtext"))]
298pub(super) static PRIORITY_TO_INTERRUPT: [CpuInterrupt; VECTOR_COUNT] = const {
299 let mut counter = 0;
300 let mut vector = [CpuInterrupt::Interrupt0; VECTOR_COUNT];
301
302 for_each_interrupt!(
303 ([vector $_n:tt] $interrupt:literal) => {
304 vector[counter] = paste::paste! { CpuInterrupt::[<Interrupt $interrupt>] };
305 counter += 1;
306 };
307 );
308 vector
309};
310
311#[instability::unstable]
327pub fn enable_direct(
328 interrupt: Interrupt,
329 level: Priority,
330 cpu_interrupt: DirectBindableCpuInterrupt,
331 handler: unsafe extern "C" fn(),
332) {
333 enable_direct_inner(interrupt, level, cpu_interrupt.into(), handler)
334}
335
336pub(crate) fn enable_direct_inner(
337 interrupt: Interrupt,
338 level: Priority,
339 cpu_interrupt: CpuInterrupt,
340 handler: unsafe extern "C" fn(),
341) {
342 bind_cpu_interrupt(cpu_interrupt, handler);
343 super::map_raw(Cpu::current(), interrupt, cpu_interrupt as u32);
344 enable_cpu_interrupt(cpu_interrupt, level);
345}
346
347pub(crate) fn bind_cpu_interrupt(cpu_interrupt: CpuInterrupt, handler: unsafe extern "C" fn()) {
349 cfg_select! {
350 interrupt_controller = "clic" => {
351 let clic = unsafe { crate::soc::pac::CLIC::steal() };
352
353 clic.int_attr(cpu_interrupt as usize).modify(|_, w| {
355 w.shv().hardware();
356 w.trig().positive_level()
357 });
358
359 let mtvt_table: *mut [u32; 48];
360 unsafe { core::arch::asm!("csrr {0}, 0x307", out(reg) mtvt_table) };
361
362 let int_slot = mtvt_table
363 .cast::<u32>()
364 .wrapping_add(cpu_interrupt as usize);
365
366 let instr = handler as usize as u32;
367 }
368 _ => {
369 use riscv::register::mtvec;
370 let mt = mtvec::read();
371
372 assert_eq!(
373 mt.trap_mode().into_usize(),
374 mtvec::TrapMode::Vectored.into_usize()
375 );
376
377 let base_addr = mt.address() as usize;
378
379 let int_slot = base_addr.wrapping_add((cpu_interrupt as usize) * 4) as *mut u32;
380
381 let instr = encode_jal_x0(handler as usize, int_slot as usize);
382 }
383 }
384
385 if crate::debugger::debugger_connected() {
386 unsafe { core::ptr::write_volatile(int_slot, instr) };
387 } else {
388 crate::debugger::DEBUGGER_LOCK.lock(|| unsafe {
389 let wp = crate::debugger::clear_watchpoint(1);
390 core::ptr::write_volatile(int_slot, instr);
391 crate::debugger::restore_watchpoint(1, wp);
392 });
393 }
394 unsafe {
395 core::arch::asm!("fence.i");
396 }
397
398 #[cfg(esp32p4)]
399 unsafe {
400 crate::soc::cache_writeback_addr(mtvt_table as u32, 48 * 4);
402 crate::soc::cache_invalidate_icache_addr(mtvt_table as u32, 48 * 4);
404 }
405}
406
407pub(crate) fn enable_cpu_interrupt(cpu_interrupt: CpuInterrupt, level: Priority) {
409 cpu_int::set_priority_raw(cpu_interrupt as u32, level);
410 cpu_int::set_kind_raw(cpu_interrupt as u32, InterruptKind::Level);
411 cpu_int::enable_cpu_interrupt_raw(cpu_interrupt as u32);
412}
413
414#[cfg(not(interrupt_controller = "clic"))]
416fn encode_jal_x0(target: usize, pc: usize) -> u32 {
417 let offset = (target as isize) - (pc as isize);
418
419 const MIN: isize = -(1isize << 20);
420 const MAX: isize = (1isize << 20) - 1;
421
422 assert!(offset % 2 == 0 && (MIN..=MAX).contains(&offset));
423
424 let imm = offset as u32;
425 let imm20 = (imm >> 20) & 0x1;
426 let imm10_1 = (imm >> 1) & 0x3ff;
427 let imm11 = (imm >> 11) & 0x1;
428 let imm19_12 = (imm >> 12) & 0xff;
429
430 (imm20 << 31)
431 | (imm19_12 << 12)
432 | (imm11 << 20)
433 | (imm10_1 << 21)
434 | 0b1101111u32
436}
437
438pub(crate) fn current_raw_runlevel() -> u32 {
442 cpu_int::current_runlevel() as u32
443}
444
445#[crate::ram]
453pub(crate) unsafe fn change_current_runlevel(level: RunLevel) -> RunLevel {
454 let previous = cpu_int::change_current_runlevel(level);
455 unwrap!(RunLevel::try_from_u32(previous as u32))
456}
457
458fn cpu_wait_mode_on() -> bool {
459 cfg_select! {
460 esp32h4 => {
462 let conf = crate::peripherals::PCR::regs().cpu_waiti_conf().read();
463 match Cpu::current() {
464 Cpu::ProCpu => conf.cpu0_wait_mode_force_on().bit_is_set(),
465 Cpu::AppCpu => conf.cpu1_wait_mode_force_on().bit_is_set(),
466 }
467 }
468 soc_has_pcr => crate::peripherals::PCR::regs()
469 .cpu_waiti_conf()
470 .read()
471 .cpu_wait_mode_force_on()
472 .bit_is_set(),
473 soc_has_hp_sys => crate::peripherals::HP_SYS::regs()
474 .cpu_waiti_conf()
475 .read()
476 .cpu_wait_mode_force_on()
477 .bit_is_set(),
478 _ => crate::peripherals::SYSTEM::regs()
479 .cpu_per_conf()
480 .read()
481 .cpu_wait_mode_force_on()
482 .bit_is_set(),
483 }
484}
485
486#[inline(always)]
499#[instability::unstable]
500pub fn wait_for_interrupt() {
501 if crate::debugger::debugger_connected() && !cpu_wait_mode_on() {
502 return;
506 }
507 cfg_select! {
508 idle_frequency_scaling => crate::clock::wait_for_interrupt(),
509 _ => unsafe { core::arch::asm!("wfi") },
510 }
511}
512
513pub(crate) fn priority_to_cpu_interrupt(_interrupt: Interrupt, level: Priority) -> CpuInterrupt {
514 PRIORITY_TO_INTERRUPT[(level as usize) - 1]
515}
516
517#[cfg(any(feature = "rt", multi_core))]
523pub(crate) unsafe fn init_vectoring() {
524 use riscv::register::mtvec;
525
526 unsafe extern "C" {
527 static _vector_table: u32;
528 }
529
530 #[cfg(soc_has_clint)]
533 crate::peripherals::CLINT::regs()
534 .msip()
535 .write(|w| w.msip().clear_bit());
536
537 unsafe {
538 let vec_table = (&raw const _vector_table).addr();
539
540 #[cfg(not(interrupt_controller = "clic"))]
541 {
542 mtvec::write({
543 let mut mtvec = mtvec::Mtvec::from_bits(0);
544 mtvec.set_trap_mode(mtvec::TrapMode::Vectored);
545 mtvec.set_address(vec_table);
546 mtvec
547 });
548 }
549
550 #[cfg(interrupt_controller = "clic")]
551 {
552 mtvec::write({
553 let mut mtvec = mtvec::Mtvec::from_bits(0x03); mtvec.set_address(vec_table);
555 mtvec
556 });
557
558 let mtvt_table = match Cpu::current() {
560 Cpu::ProCpu => {
561 unsafe extern "C" {
562 static _mtvt_table: u32;
563 }
564 (&raw const _mtvt_table).addr()
565 }
566 #[cfg(multi_core)]
567 Cpu::AppCpu => {
568 unsafe extern "C" {
569 static _mtvt_table2: u32;
570 }
571 (&raw const _mtvt_table2).addr()
572 }
573 };
574 core::arch::asm!("csrw 0x307, {0}", in(reg) mtvt_table);
575 }
576 };
577
578 #[cfg(feature = "rt")]
581 cpu_int::init();
582
583 for (int, prio) in PRIORITY_TO_INTERRUPT.iter().copied().zip(Priority::iter()) {
585 let num = int as u32;
586 cpu_int::set_kind_raw(num, InterruptKind::Level);
587 cpu_int::set_priority_raw(num, prio);
588 cpu_int::enable_cpu_interrupt_raw(num);
589 }
590}
591
592#[cfg(feature = "rt")]
593pub(crate) mod rt {
594 use esp_riscv_rt::TrapFrame;
595 use riscv::register::mcause;
596
597 use super::*;
598 use crate::interrupt::InterruptStatus;
599
600 #[cfg(not(interrupt_controller = "clic"))]
602 const INTERRUPT_COUNT: usize = const {
603 let mut count = 0;
604 for_each_interrupt!(([$_class:tt $n:tt] $_:literal) => { count += 1; };);
605 count
606 };
607
608 #[cfg(not(interrupt_controller = "clic"))]
610 #[cfg_attr(place_switch_tables_in_ram, unsafe(link_section = ".rwtext"))]
611 pub(super) static INTERRUPT_TO_PRIORITY: [Option<Priority>; INTERRUPT_COUNT] = const {
612 let mut priorities = [None; INTERRUPT_COUNT];
613
614 for_each_interrupt!(
615 ([vector $n:tt] $int:literal) => {
616 for_each_interrupt_priority!(($n, $__:tt, $ident:ident, $_level:ident) => { priorities[$int] = Some(Priority::$ident); };);
617 };
618 );
619
620 priorities
621 };
622
623 #[doc(hidden)]
627 #[unsafe(link_section = ".trap.rust")]
628 #[unsafe(export_name = "_start_trap_rust_hal")]
629 unsafe extern "C" fn start_trap_rust_hal(trap_frame: *mut TrapFrame) {
630 assert!(
631 mcause::read().is_exception(),
632 "Arrived into _start_trap_rust_hal but mcause is not an exception!"
633 );
634 unsafe extern "C" {
635 fn ExceptionHandler(tf: *mut TrapFrame);
636 }
637 unsafe {
638 ExceptionHandler(trap_frame);
639 }
640 }
641
642 #[doc(hidden)]
643 #[unsafe(no_mangle)]
644 #[unsafe(link_section = ".init")]
645 unsafe fn _setup_interrupts() {
646 crate::soc::riscv_preinit();
647 crate::interrupt::setup_interrupts();
648
649 #[cfg(interrupt_controller = "plic")]
650 unsafe {
651 core::arch::asm!("csrw mie, {0}", in(reg) u32::MAX);
652 }
653 }
654
655 #[unsafe(no_mangle)]
656 #[crate::ram]
657 unsafe fn handle_interrupts(cpu_intr: CpuInterrupt) {
658 let status = InterruptStatus::current();
659
660 cpu_intr.clear();
663
664 cfg_select! {
665 interrupt_controller = "clic" => {
666 let prio = cpu_int::current_runlevel();
667 let mcause = riscv::register::mcause::read();
668 }
669 _ => {
670 let prio = unwrap!(
673 INTERRUPT_TO_PRIORITY
674 .get(cpu_intr as usize)
675 .copied()
676 .flatten()
677 );
678 let level = unsafe {
679 change_current_runlevel(RunLevel::Interrupt(ElevatedRunLevel::from(prio)))
680 };
681 let prio = prio as u8;
682 }
683 }
684
685 let handle_interrupts = || unsafe {
686 for interrupt in status.iterator().filter(|&interrupt| {
687 crate::interrupt::should_handle(Cpu::current(), interrupt as u32, prio as u32)
688 }) {
689 let handler = crate::soc::pac::__EXTERNAL_INTERRUPTS[interrupt as usize]._handler;
690
691 handler();
692 }
693 };
694
695 if prio != Priority::max() as u8 {
699 unsafe {
700 #[cfg(interrupt_controller = "clic")]
704 let triggers_enabled = crate::debugger::triggers_enabled_on_mret();
705 riscv::interrupt::nested(handle_interrupts);
706 #[cfg(interrupt_controller = "clic")]
707 if triggers_enabled {
708 crate::debugger::enable_triggers_on_mret();
709 }
710 }
711 } else {
712 handle_interrupts();
713 }
714
715 cfg_select! {
716 interrupt_controller = "clic" => unsafe {
717 core::arch::asm!("csrw 0x342, {}", in(reg) mcause.bits())
722 },
723 _ => {
724 unsafe { change_current_runlevel(level) };
725 }
726 }
727 }
728}