Skip to main content

esp_hal/system/
multi_core.rs

1//! Multi-core support
2
3use core::{
4    marker::PhantomData,
5    mem::{ManuallyDrop, MaybeUninit},
6    sync::atomic::{AtomicPtr, Ordering},
7};
8
9#[instability::unstable]
10pub use crate::soc::cpu_control::is_running;
11use crate::{peripherals::CPU_CTRL, soc::cpu_control::internal_park_core, system::Cpu};
12
13/// Data type for a properly aligned stack of N bytes
14// Xtensa ISA 10.5: [B]y default, the
15// stack frame is 16-byte aligned. However, the maximal alignment allowed for a
16// TIE ctype is 64-bytes. If a function has any wide-aligned (>16-byte aligned)
17// data type for their arguments or the return values, the caller has to ensure
18// that the SP is aligned to the largest alignment right before the call.
19//
20// ^ this means that we should be able to get away with 16 bytes of alignment
21// because our root stack frame has no arguments and no return values.
22//
23// This alignment also doesn't align the stack frames, only the end of stack.
24// Stack frame alignment depends on the SIZE as well as the placement of the
25// array.
26#[repr(C, align(16))]
27#[instability::unstable]
28pub struct Stack<const SIZE: usize> {
29    /// Memory to be used for the stack
30    pub mem: MaybeUninit<[u8; SIZE]>,
31}
32
33impl<const SIZE: usize> Default for Stack<SIZE> {
34    fn default() -> Self {
35        Self::new()
36    }
37}
38
39#[allow(clippy::len_without_is_empty)]
40impl<const SIZE: usize> Stack<SIZE> {
41    /// Creates a new stack of length SIZE, uninitialized.
42    #[instability::unstable]
43    pub const fn new() -> Stack<SIZE> {
44        const {
45            // Make sure stack top is aligned, too.
46            ::core::assert!(SIZE.is_multiple_of(16));
47        }
48
49        Stack {
50            mem: MaybeUninit::uninit(),
51        }
52    }
53
54    /// Returns the length of the stack in bytes.
55    #[instability::unstable]
56    pub const fn len(&self) -> usize {
57        SIZE
58    }
59
60    /// Returns a mutable pointer to the bottom of the stack.
61    #[instability::unstable]
62    pub fn bottom(&mut self) -> *mut u32 {
63        self.mem.as_mut_ptr() as *mut u32
64    }
65
66    /// Returns a mutable pointer to the top of the stack.
67    #[instability::unstable]
68    pub fn top(&mut self) -> *mut u32 {
69        unsafe { self.bottom().add(SIZE / 4) }
70    }
71}
72
73// Pointer to the closure that will be executed on the second core. The closure
74// is copied to the core's stack.
75pub(crate) static START_CORE1_FUNCTION: AtomicPtr<()> = AtomicPtr::new(core::ptr::null_mut());
76pub(crate) static APP_CORE_STACK_TOP: AtomicPtr<u32> = AtomicPtr::new(core::ptr::null_mut());
77pub(crate) static APP_CORE_STACK_GUARD: AtomicPtr<u32> = AtomicPtr::new(core::ptr::null_mut());
78
79/// Will park the APP (second) core when dropped.
80///
81/// Parking stops the APP core immediately, also when it holds a lock. On the ESP32-S3, ESP32-S31,
82/// ESP32-H4 and ESP32-P4, DMA buffer operations lock the cache. If the APP core is parked during
83/// such an operation, the next cache operation on the PRO core waits forever. Make sure that the
84/// APP core does no DMA or cache operation before you drop this guard.
85///
86/// The parked APP core runs no [IPC requests][crate::interrupt::ipc::IpcRequest]. A request that
87/// the APP core posted or took at the park can stay busy.
88#[must_use = "Dropping this guard will park the APP core"]
89#[instability::unstable]
90pub struct AppCoreGuard<'a> {
91    phantom: PhantomData<&'a ()>,
92}
93
94impl Drop for AppCoreGuard<'_> {
95    fn drop(&mut self) {
96        unsafe { internal_park_core(Cpu::AppCpu, true) };
97    }
98}
99
100/// Suspends the scheduler of the current core until the guard drops.
101///
102/// The current core defers its context switches while a guard is alive. The current thread thus
103/// keeps the current core, and cannot move to the other core. Interrupt handlers and
104/// [IPC requests][crate::interrupt::ipc::IpcRequest] still run. The deferred context switch runs
105/// when the last guard drops.
106///
107/// The scheduler of the other core continues to run. Do not keep a guard for a long time,
108/// because other threads cannot run on the current core.
109#[must_use = "The scheduler continues when the guard drops"]
110#[instability::unstable]
111pub struct SchedulerSuspendGuard {
112    // The guard must drop on the core that made the guard.
113    _not_send: PhantomData<*const ()>,
114}
115
116impl SchedulerSuspendGuard {
117    /// Creates a new guard, and suspends the scheduler of the current core.
118    #[instability::unstable]
119    #[inline(always)]
120    #[allow(clippy::new_without_default)]
121    pub fn new() -> Self {
122        defer_context_switch();
123        Self {
124            _not_send: PhantomData,
125        }
126    }
127}
128
129impl Drop for SchedulerSuspendGuard {
130    #[inline(always)]
131    fn drop(&mut self) {
132        unsafe { resume_context_switch() };
133    }
134}
135
136/// Represents errors that can occur while working with the core.
137#[derive(Debug, Clone, Copy, PartialEq)]
138#[cfg_attr(feature = "defmt", derive(defmt::Format))]
139#[instability::unstable]
140pub enum Error {
141    /// The core is already running.
142    CoreAlreadyRunning,
143}
144
145#[procmacros::doc_replace]
146/// Control CPU Cores
147///
148/// # Examples
149///
150/// ```rust, no_run
151/// # {before_snippet}
152/// # use esp_hal::delay::Delay;
153/// # use esp_hal::system::{CpuControl, Stack};
154/// # use core::{cell::RefCell, ptr::addr_of_mut};
155/// # use critical_section::Mutex;
156/// # let delay = Delay::new();
157/// static mut APP_CORE_STACK: Stack<8192> = Stack::new();
158///
159/// let counter = Mutex::new(RefCell::new(0));
160///
161/// let mut cpu_control = CpuControl::new(peripherals.CPU_CTRL);
162/// let cpu1_fnctn = || {
163///     cpu1_task(&delay, &counter);
164/// };
165/// let _guard =
166///     cpu_control.start_app_core(unsafe { &mut *addr_of_mut!(APP_CORE_STACK) }, cpu1_fnctn)?;
167///
168/// loop {
169///     delay.delay(Duration::from_secs(1));
170///     let count = critical_section::with(|cs| *counter.borrow_ref(cs));
171/// }
172/// # }
173///
174/// // Where `cpu1_task()` may be defined as:
175/// # use esp_hal::delay::Delay;
176/// # use core::cell::RefCell;
177///
178/// fn cpu1_task(delay: &Delay, counter: &critical_section::Mutex<RefCell<i32>>) -> ! {
179///     loop {
180///         delay.delay(Duration::from_millis(500));
181///
182///         critical_section::with(|cs| {
183///             let mut val = counter.borrow_ref_mut(cs);
184///             *val = val.wrapping_add(1);
185///         });
186///     }
187/// }
188/// ```
189#[instability::unstable]
190pub struct CpuControl<'d> {
191    _cpu_control: CPU_CTRL<'d>,
192}
193
194impl<'d> CpuControl<'d> {
195    /// Creates a new instance of `CpuControl`.
196    #[instability::unstable]
197    pub fn new(cpu_control: CPU_CTRL<'d>) -> CpuControl<'d> {
198        CpuControl {
199            _cpu_control: cpu_control,
200        }
201    }
202
203    /// Parks the given core.
204    ///
205    /// Parking stops the core immediately, also when it holds a lock. On the ESP32-S3, ESP32-S31,
206    /// ESP32-H4 and ESP32-P4, DMA buffer operations lock the cache. If the core is parked during
207    /// such an operation, the next cache operation on the other core waits forever. Make sure that
208    /// the parked core does no DMA or cache operation when you call this function.
209    ///
210    /// IPC requests to the parked core run after [`Self::unpark_core`].
211    ///
212    /// # Safety
213    ///
214    /// The caller must ensure that the core being parked is not the core which is
215    /// currently executing this code.
216    #[instability::unstable]
217    pub unsafe fn park_core(&mut self, core: Cpu) {
218        unsafe { internal_park_core(core, true) };
219    }
220
221    /// Unparks the given core.
222    #[instability::unstable]
223    pub fn unpark_core(&mut self, core: Cpu) {
224        unsafe { internal_park_core(core, false) };
225    }
226
227    /// Runs the core1 closure.
228    #[inline(never)]
229    pub(crate) unsafe fn start_core1_run<F>() -> !
230    where
231        F: FnOnce(),
232    {
233        let entry = START_CORE1_FUNCTION.load(Ordering::Acquire);
234        debug_assert!(!entry.is_null());
235
236        unsafe {
237            let entry = ManuallyDrop::take(&mut *entry.cast::<ManuallyDrop<F>>());
238            entry();
239            loop {
240                internal_park_core(Cpu::current(), true);
241            }
242        }
243    }
244
245    /// Starts the APP (second) core.
246    ///
247    /// The second core starts running the closure `entry`. The core enables interrupts before
248    /// `entry` runs. If the closure exits, the core is parked.
249    ///
250    /// Dropping the returned guard will park the core.
251    #[instability::unstable]
252    pub fn start_app_core<'a, const SIZE: usize, F>(
253        &mut self,
254        stack: &'static mut Stack<SIZE>,
255        entry: F,
256    ) -> Result<AppCoreGuard<'a>, Error>
257    where
258        F: FnOnce(),
259        F: Send + 'a,
260    {
261        cfg_select! {
262            all(stack_guard_monitoring) => {
263                let stack_guard_offset = Some(esp_config::esp_config_int!(
264                    usize,
265                    "ESP_HAL_CONFIG_STACK_GUARD_OFFSET"
266                ));
267            }
268            _ => {
269                let stack_guard_offset = None;
270            }
271        };
272
273        self.start_app_core_with_stack_guard_offset(stack, stack_guard_offset, entry)
274    }
275
276    /// Starts the APP (second) core.
277    ///
278    /// The second core starts running the closure `entry`. The core enables interrupts before
279    /// `entry` runs. If the closure exits, the core is parked.
280    ///
281    /// Dropping the returned guard will park the core.
282    #[instability::unstable]
283    pub fn start_app_core_with_stack_guard_offset<'a, const SIZE: usize, F>(
284        &mut self,
285        stack: &'static mut Stack<SIZE>,
286        stack_guard_offset: Option<usize>,
287        entry: F,
288    ) -> Result<AppCoreGuard<'a>, Error>
289    where
290        F: FnOnce(),
291        F: Send + 'a,
292    {
293        if !crate::debugger::debugger_connected() && is_running(Cpu::AppCpu) {
294            return Err(Error::CoreAlreadyRunning);
295        }
296
297        setup_second_core_stack(stack, stack_guard_offset, entry);
298
299        app_core_starting();
300        crate::soc::cpu_control::start_core1(start_core1_init::<F> as *const u32);
301
302        self.unpark_core(Cpu::AppCpu);
303
304        // The APP core can hold a lock during startup. The flash driver must not park the APP
305        // core before startup ends.
306        while !app_core_takes_ipc() {
307            core::hint::spin_loop();
308        }
309
310        Ok(AppCoreGuard {
311            phantom: PhantomData,
312        })
313    }
314}
315
316fn setup_second_core_stack<'a, F, const SIZE: usize>(
317    stack: &'static mut Stack<SIZE>,
318    stack_guard_offset: Option<usize>,
319    entry: F,
320) where
321    F: FnOnce(),
322    F: Send + 'a,
323{
324    // We don't want to drop this, since it's getting moved to the other core.
325    let entry = ManuallyDrop::new(entry);
326
327    unsafe {
328        let stack_bottom = stack.bottom().cast::<u8>();
329        let (stack_guard, stack_bottom_above_guard) =
330            if let Some(stack_guard_offset) = stack_guard_offset {
331                assert!(stack_guard_offset.is_multiple_of(4));
332                assert!(stack_guard_offset <= stack.len() - 4);
333                (
334                    stack_bottom.byte_add(stack_guard_offset),
335                    stack_bottom.byte_add(stack_guard_offset).byte_add(4),
336                )
337            } else {
338                (core::ptr::null_mut(), stack_bottom)
339            };
340
341        // Push `entry` to an aligned address at the (physical) bottom of the stack, but above
342        // the stack guard. The second core will copy it into its proper place, then
343        // calls it.
344        let align_offset = stack_bottom_above_guard.align_offset(core::mem::align_of::<F>());
345        let entry_dst = stack_bottom_above_guard
346            .add(align_offset)
347            .cast::<ManuallyDrop<F>>();
348
349        entry_dst.write(entry);
350
351        let entry_fn = entry_dst.cast::<()>();
352        #[cfg(soc_cpu_pma_needs_core1_setup)]
353        pma::save_core0_pma(&mut *core0_pma_slot::<F>(entry_fn));
354        START_CORE1_FUNCTION.store(entry_fn, Ordering::Release);
355        APP_CORE_STACK_TOP.store(stack.top(), Ordering::Release);
356        APP_CORE_STACK_GUARD.store(stack_guard.cast(), Ordering::Release);
357    }
358}
359
360#[cfg(soc_cpu_pma_needs_core1_setup)]
361fn core0_pma_slot<F>(entry: *mut ()) -> *mut [u32; 32] {
362    let end = entry.cast::<u8>().wrapping_add(core::mem::size_of::<F>());
363    end.wrapping_add(end.align_offset(core::mem::align_of::<u32>()))
364        .cast()
365}
366
367#[cfg(soc_cpu_pma_needs_core1_setup)]
368mod pma {
369    macro_rules! read_csr {
370        ($csr:literal) => {{
371            let value: u32;
372            unsafe { core::arch::asm!(concat!("csrr {0}, ", $csr), out(reg) value) };
373            value
374        }};
375    }
376
377    macro_rules! write_csr {
378        ($csr:literal, $value:expr) => {
379            unsafe { core::arch::asm!(concat!("csrw ", $csr, ", {0}"), in(reg) $value) }
380        };
381    }
382
383    macro_rules! pma_entries {
384        ($($i:literal: $cfg:literal, $addr:literal;)*) => {
385            pub(super) fn save_core0_pma(dst: &mut [u32; 32]) {
386                $(
387                    dst[$i] = read_csr!($cfg);
388                    dst[$i + 16] = read_csr!($addr);
389                )*
390            }
391
392            pub(super) fn set_core1_pma(src: &[u32; 32]) {
393                $(
394                    write_csr!($addr, src[$i + 16]);
395                    write_csr!($cfg, src[$i]);
396                )*
397            }
398        };
399    }
400
401    pma_entries! {
402        0: "0xbc0", "0xbd0";
403        1: "0xbc1", "0xbd1";
404        2: "0xbc2", "0xbd2";
405        3: "0xbc3", "0xbd3";
406        4: "0xbc4", "0xbd4";
407        5: "0xbc5", "0xbd5";
408        6: "0xbc6", "0xbd6";
409        7: "0xbc7", "0xbd7";
410        8: "0xbc8", "0xbd8";
411        9: "0xbc9", "0xbd9";
412        10: "0xbca", "0xbda";
413        11: "0xbcb", "0xbdb";
414        12: "0xbcc", "0xbdc";
415        13: "0xbcd", "0xbdd";
416        14: "0xbce", "0xbde";
417        15: "0xbcf", "0xbdf";
418    }
419}
420
421/// Core 1 entry point, set as the boot address.
422#[cfg(xtensa)]
423pub(crate) fn start_core1_init<F>() -> !
424where
425    F: FnOnce(),
426{
427    // disables interrupts
428    unsafe {
429        xtensa_lx::interrupt::set_mask(0);
430    }
431
432    // reset cycle compare registers
433    xtensa_lx::timer::set_ccompare0(0);
434    xtensa_lx::timer::set_ccompare1(0);
435    xtensa_lx::timer::set_ccompare2(0);
436
437    unsafe extern "C" {
438        static mut _init_start: u32;
439    }
440
441    // set vector table and stack pointer
442    unsafe {
443        xtensa_lx::set_vecbase(&raw const _init_start);
444        xtensa_lx::set_stack_pointer(APP_CORE_STACK_TOP.load(Ordering::Acquire));
445
446        #[cfg(all(feature = "rt", stack_guard_monitoring))]
447        {
448            let stack_guard = APP_CORE_STACK_GUARD.load(Ordering::Acquire);
449            stack_guard.write_volatile(esp_config::esp_config_int!(
450                u32,
451                "ESP_HAL_CONFIG_STACK_GUARD_VALUE"
452            ));
453            // setting 0 effectively disables the functionality
454            crate::debugger::set_stack_watchpoint(stack_guard as usize);
455        }
456    }
457
458    // The ROM has already handed off to us; clear the AppCpu boot address so a
459    // subsequent reset doesn't see a stale entry point. Matches IDF's
460    // `call_start_cpu1`.
461    cfg_select! {
462        esp32 => {
463            crate::peripherals::DPORT::regs()
464                .appcpu_ctrl_d()
465                .write(|w| unsafe { w.appcpu_boot_addr().bits(0) });
466        }
467        _ => crate::rom::ets_set_appcpu_boot_addr(0),
468    }
469
470    // Do not call setup_interrupts as that would disable peripheral interrupts, too.
471    unsafe { crate::interrupt::init_vectoring() };
472
473    enable_app_core_interrupts();
474
475    // Trampoline to run from the new stack.
476    // start_core1_run should _NEVER_ be inlined
477    // as we rely on the function call to use
478    // the new stack.
479    unsafe { CpuControl::start_core1_run::<F>() }
480}
481
482/// Core 1 entry point, set as the boot address.
483///
484/// ROM jumps here directly, bypassing `_start`, so `gp`, the FPU and the stack pointer need to be
485/// set up before entering regular Rust.
486#[cfg(riscv)]
487#[unsafe(naked)]
488pub(crate) extern "C" fn start_core1_init<F>() -> !
489where
490    F: FnOnce(),
491{
492    core::arch::naked_asm!(
493        ".option push",
494        ".option norelax",
495        "la gp, __global_pointer$",
496        "li ra, 0", // ensure probe-rs stops unwinding
497        ".option pop",
498        // Follow IDF's FPU initialization (rv_utils_enable_fpu, rv_utils_clear_fpu): enable the
499        // FPU in Initial state, touch fcsr (which makes the state Dirty), then clear the dirty
500        // bit to leave mstatus.FS in Clean state (0b10), which is what the interrupt handler
501        // expects.
502        "li t0, 0x2000",
503        "csrs mstatus, t0",
504        "li t0, 1",
505        "csrw fcsr, t0",
506        "li t0, 0x2000",
507        "csrc mstatus, t0",
508        // Switch to Core 1's stack (stored by Core 0 before releasing this core).
509        "la t0, {stack_top}",
510        "lw sp, 0(t0)",
511        "j {init}",
512        stack_top = sym APP_CORE_STACK_TOP,
513        init = sym start_core1_init_impl::<F>,
514    )
515}
516
517#[cfg(riscv)]
518fn start_core1_init_impl<F>() -> !
519where
520    F: FnOnce(),
521{
522    #[cfg(soc_cpu_pma_needs_core1_setup)]
523    pma::set_core1_pma(unsafe {
524        &*core0_pma_slot::<F>(START_CORE1_FUNCTION.load(Ordering::Acquire))
525    });
526
527    crate::soc::branch_predictor::enable();
528
529    // The ROM has already handed off to us; clear the AppCpu boot address so a subsequent software
530    // reset doesn't re-enter this stale entry point. Matches IDF's `call_start_cpu1`.
531    crate::rom::ets_set_appcpu_boot_addr(0);
532
533    unsafe {
534        #[cfg(all(feature = "rt", stack_guard_monitoring))]
535        {
536            let guard = APP_CORE_STACK_GUARD.load(Ordering::Acquire);
537            guard.write_volatile(esp_config::esp_config_int!(
538                u32,
539                "ESP_HAL_CONFIG_STACK_GUARD_VALUE"
540            ));
541            crate::debugger::set_stack_watchpoint(guard as usize);
542        }
543        crate::interrupt::init_vectoring();
544    }
545
546    enable_app_core_interrupts();
547
548    unsafe { CpuControl::start_core1_run::<F>() }
549}
550
551fn enable_app_core_interrupts() {
552    unsafe {
553        crate::interrupt::RunLevel::change(crate::interrupt::RunLevel::ThreadMode);
554        #[cfg(riscv)]
555        core::arch::asm!("csrsi mstatus, 8", options(nomem, nostack));
556    }
557    app_core_started();
558}
559
560// A build can hold esp-hal versions without the `rt` feature. Those versions call the functions
561// below by symbol name. The names and signatures must stay stable.
562cfg_select! {
563    feature = "rt" => {
564        use core::sync::atomic::AtomicBool;
565
566        use crate::interrupt::ipc::{defer_context_switch, resume_context_switch};
567
568        static APP_CORE_TAKES_IPC: AtomicBool = AtomicBool::new(false);
569
570        /// Marks the APP core as not running code of this program before the APP core starts.
571        #[unsafe(export_name = "esp_hal_app_core_starting")]
572        fn app_core_starting() {
573            APP_CORE_TAKES_IPC.store(false, Ordering::Release);
574            // Posts to the APP core return `Offline` after the store, so no post adds a request
575            // to the APP core while this call fails the requests that the APP core did not run.
576            unsafe { crate::interrupt::ipc::fail_stranded(Cpu::AppCpu) };
577            // The APP core can stop while a `SchedulerSuspendGuard` is alive on the APP core.
578            crate::interrupt::ipc::reset_context_switch_deferral(Cpu::AppCpu);
579        }
580
581        /// Lets the APP core take IPC interrupts. The APP core calls this function after it
582        /// enables its interrupts.
583        #[unsafe(export_name = "esp_hal_app_core_started")]
584        fn app_core_started() {
585            debug_assert_eq!(Cpu::current(), Cpu::AppCpu);
586            #[cfg(riscv)]
587            crate::interrupt::ipc::install_app();
588            APP_CORE_TAKES_IPC.store(true, Ordering::Release);
589        }
590
591        /// Returns `true` if [`CpuControl`] started the APP core in this boot. That core takes IPC
592        /// interrupts unless it masks them.
593        #[unsafe(export_name = "esp_hal_app_core_takes_ipc")]
594        pub(crate) fn app_core_takes_ipc() -> bool {
595            APP_CORE_TAKES_IPC.load(Ordering::Acquire)
596        }
597    }
598    _ => {
599        unsafe extern "Rust" {
600            #[link_name = "esp_hal_app_core_starting"]
601            safe fn app_core_starting();
602            #[link_name = "esp_hal_app_core_started"]
603            safe fn app_core_started();
604            #[link_name = "esp_hal_app_core_takes_ipc"]
605            pub(crate) safe fn app_core_takes_ipc() -> bool;
606            #[link_name = "esp_hal_defer_context_switch"]
607            safe fn defer_context_switch();
608            #[link_name = "esp_hal_resume_context_switch"]
609            fn resume_context_switch();
610        }
611    }
612}