1use core::{
4 marker::PhantomData,
5 mem::{ManuallyDrop, MaybeUninit},
6 sync::atomic::{AtomicPtr, Ordering},
7};
8
9#[instability::unstable]
10pub use crate::soc::cpu_control::is_running;
11use crate::{peripherals::CPU_CTRL, soc::cpu_control::internal_park_core, system::Cpu};
12
13#[repr(C, align(16))]
27#[instability::unstable]
28pub struct Stack<const SIZE: usize> {
29 pub mem: MaybeUninit<[u8; SIZE]>,
31}
32
33impl<const SIZE: usize> Default for Stack<SIZE> {
34 fn default() -> Self {
35 Self::new()
36 }
37}
38
39#[allow(clippy::len_without_is_empty)]
40impl<const SIZE: usize> Stack<SIZE> {
41 #[instability::unstable]
43 pub const fn new() -> Stack<SIZE> {
44 const {
45 ::core::assert!(SIZE.is_multiple_of(16));
47 }
48
49 Stack {
50 mem: MaybeUninit::uninit(),
51 }
52 }
53
54 #[instability::unstable]
56 pub const fn len(&self) -> usize {
57 SIZE
58 }
59
60 #[instability::unstable]
62 pub fn bottom(&mut self) -> *mut u32 {
63 self.mem.as_mut_ptr() as *mut u32
64 }
65
66 #[instability::unstable]
68 pub fn top(&mut self) -> *mut u32 {
69 unsafe { self.bottom().add(SIZE / 4) }
70 }
71}
72
73pub(crate) static START_CORE1_FUNCTION: AtomicPtr<()> = AtomicPtr::new(core::ptr::null_mut());
76pub(crate) static APP_CORE_STACK_TOP: AtomicPtr<u32> = AtomicPtr::new(core::ptr::null_mut());
77pub(crate) static APP_CORE_STACK_GUARD: AtomicPtr<u32> = AtomicPtr::new(core::ptr::null_mut());
78
79#[must_use = "Dropping this guard will park the APP core"]
89#[instability::unstable]
90pub struct AppCoreGuard<'a> {
91 phantom: PhantomData<&'a ()>,
92}
93
94impl Drop for AppCoreGuard<'_> {
95 fn drop(&mut self) {
96 unsafe { internal_park_core(Cpu::AppCpu, true) };
97 }
98}
99
100#[must_use = "The scheduler continues when the guard drops"]
110#[instability::unstable]
111pub struct SchedulerSuspendGuard {
112 _not_send: PhantomData<*const ()>,
114}
115
116impl SchedulerSuspendGuard {
117 #[instability::unstable]
119 #[inline(always)]
120 #[allow(clippy::new_without_default)]
121 pub fn new() -> Self {
122 defer_context_switch();
123 Self {
124 _not_send: PhantomData,
125 }
126 }
127}
128
129impl Drop for SchedulerSuspendGuard {
130 #[inline(always)]
131 fn drop(&mut self) {
132 unsafe { resume_context_switch() };
133 }
134}
135
136#[derive(Debug, Clone, Copy, PartialEq)]
138#[cfg_attr(feature = "defmt", derive(defmt::Format))]
139#[instability::unstable]
140pub enum Error {
141 CoreAlreadyRunning,
143}
144
145#[procmacros::doc_replace]
146#[instability::unstable]
190pub struct CpuControl<'d> {
191 _cpu_control: CPU_CTRL<'d>,
192}
193
194impl<'d> CpuControl<'d> {
195 #[instability::unstable]
197 pub fn new(cpu_control: CPU_CTRL<'d>) -> CpuControl<'d> {
198 CpuControl {
199 _cpu_control: cpu_control,
200 }
201 }
202
203 #[instability::unstable]
217 pub unsafe fn park_core(&mut self, core: Cpu) {
218 unsafe { internal_park_core(core, true) };
219 }
220
221 #[instability::unstable]
223 pub fn unpark_core(&mut self, core: Cpu) {
224 unsafe { internal_park_core(core, false) };
225 }
226
227 #[inline(never)]
229 pub(crate) unsafe fn start_core1_run<F>() -> !
230 where
231 F: FnOnce(),
232 {
233 let entry = START_CORE1_FUNCTION.load(Ordering::Acquire);
234 debug_assert!(!entry.is_null());
235
236 unsafe {
237 let entry = ManuallyDrop::take(&mut *entry.cast::<ManuallyDrop<F>>());
238 entry();
239 loop {
240 internal_park_core(Cpu::current(), true);
241 }
242 }
243 }
244
245 #[instability::unstable]
252 pub fn start_app_core<'a, const SIZE: usize, F>(
253 &mut self,
254 stack: &'static mut Stack<SIZE>,
255 entry: F,
256 ) -> Result<AppCoreGuard<'a>, Error>
257 where
258 F: FnOnce(),
259 F: Send + 'a,
260 {
261 cfg_select! {
262 all(stack_guard_monitoring) => {
263 let stack_guard_offset = Some(esp_config::esp_config_int!(
264 usize,
265 "ESP_HAL_CONFIG_STACK_GUARD_OFFSET"
266 ));
267 }
268 _ => {
269 let stack_guard_offset = None;
270 }
271 };
272
273 self.start_app_core_with_stack_guard_offset(stack, stack_guard_offset, entry)
274 }
275
276 #[instability::unstable]
283 pub fn start_app_core_with_stack_guard_offset<'a, const SIZE: usize, F>(
284 &mut self,
285 stack: &'static mut Stack<SIZE>,
286 stack_guard_offset: Option<usize>,
287 entry: F,
288 ) -> Result<AppCoreGuard<'a>, Error>
289 where
290 F: FnOnce(),
291 F: Send + 'a,
292 {
293 if !crate::debugger::debugger_connected() && is_running(Cpu::AppCpu) {
294 return Err(Error::CoreAlreadyRunning);
295 }
296
297 setup_second_core_stack(stack, stack_guard_offset, entry);
298
299 app_core_starting();
300 crate::soc::cpu_control::start_core1(start_core1_init::<F> as *const u32);
301
302 self.unpark_core(Cpu::AppCpu);
303
304 while !app_core_takes_ipc() {
307 core::hint::spin_loop();
308 }
309
310 Ok(AppCoreGuard {
311 phantom: PhantomData,
312 })
313 }
314}
315
316fn setup_second_core_stack<'a, F, const SIZE: usize>(
317 stack: &'static mut Stack<SIZE>,
318 stack_guard_offset: Option<usize>,
319 entry: F,
320) where
321 F: FnOnce(),
322 F: Send + 'a,
323{
324 let entry = ManuallyDrop::new(entry);
326
327 unsafe {
328 let stack_bottom = stack.bottom().cast::<u8>();
329 let (stack_guard, stack_bottom_above_guard) =
330 if let Some(stack_guard_offset) = stack_guard_offset {
331 assert!(stack_guard_offset.is_multiple_of(4));
332 assert!(stack_guard_offset <= stack.len() - 4);
333 (
334 stack_bottom.byte_add(stack_guard_offset),
335 stack_bottom.byte_add(stack_guard_offset).byte_add(4),
336 )
337 } else {
338 (core::ptr::null_mut(), stack_bottom)
339 };
340
341 let align_offset = stack_bottom_above_guard.align_offset(core::mem::align_of::<F>());
345 let entry_dst = stack_bottom_above_guard
346 .add(align_offset)
347 .cast::<ManuallyDrop<F>>();
348
349 entry_dst.write(entry);
350
351 let entry_fn = entry_dst.cast::<()>();
352 #[cfg(soc_cpu_pma_needs_core1_setup)]
353 pma::save_core0_pma(&mut *core0_pma_slot::<F>(entry_fn));
354 START_CORE1_FUNCTION.store(entry_fn, Ordering::Release);
355 APP_CORE_STACK_TOP.store(stack.top(), Ordering::Release);
356 APP_CORE_STACK_GUARD.store(stack_guard.cast(), Ordering::Release);
357 }
358}
359
360#[cfg(soc_cpu_pma_needs_core1_setup)]
361fn core0_pma_slot<F>(entry: *mut ()) -> *mut [u32; 32] {
362 let end = entry.cast::<u8>().wrapping_add(core::mem::size_of::<F>());
363 end.wrapping_add(end.align_offset(core::mem::align_of::<u32>()))
364 .cast()
365}
366
367#[cfg(soc_cpu_pma_needs_core1_setup)]
368mod pma {
369 macro_rules! read_csr {
370 ($csr:literal) => {{
371 let value: u32;
372 unsafe { core::arch::asm!(concat!("csrr {0}, ", $csr), out(reg) value) };
373 value
374 }};
375 }
376
377 macro_rules! write_csr {
378 ($csr:literal, $value:expr) => {
379 unsafe { core::arch::asm!(concat!("csrw ", $csr, ", {0}"), in(reg) $value) }
380 };
381 }
382
383 macro_rules! pma_entries {
384 ($($i:literal: $cfg:literal, $addr:literal;)*) => {
385 pub(super) fn save_core0_pma(dst: &mut [u32; 32]) {
386 $(
387 dst[$i] = read_csr!($cfg);
388 dst[$i + 16] = read_csr!($addr);
389 )*
390 }
391
392 pub(super) fn set_core1_pma(src: &[u32; 32]) {
393 $(
394 write_csr!($addr, src[$i + 16]);
395 write_csr!($cfg, src[$i]);
396 )*
397 }
398 };
399 }
400
401 pma_entries! {
402 0: "0xbc0", "0xbd0";
403 1: "0xbc1", "0xbd1";
404 2: "0xbc2", "0xbd2";
405 3: "0xbc3", "0xbd3";
406 4: "0xbc4", "0xbd4";
407 5: "0xbc5", "0xbd5";
408 6: "0xbc6", "0xbd6";
409 7: "0xbc7", "0xbd7";
410 8: "0xbc8", "0xbd8";
411 9: "0xbc9", "0xbd9";
412 10: "0xbca", "0xbda";
413 11: "0xbcb", "0xbdb";
414 12: "0xbcc", "0xbdc";
415 13: "0xbcd", "0xbdd";
416 14: "0xbce", "0xbde";
417 15: "0xbcf", "0xbdf";
418 }
419}
420
421#[cfg(xtensa)]
423pub(crate) fn start_core1_init<F>() -> !
424where
425 F: FnOnce(),
426{
427 unsafe {
429 xtensa_lx::interrupt::set_mask(0);
430 }
431
432 xtensa_lx::timer::set_ccompare0(0);
434 xtensa_lx::timer::set_ccompare1(0);
435 xtensa_lx::timer::set_ccompare2(0);
436
437 unsafe extern "C" {
438 static mut _init_start: u32;
439 }
440
441 unsafe {
443 xtensa_lx::set_vecbase(&raw const _init_start);
444 xtensa_lx::set_stack_pointer(APP_CORE_STACK_TOP.load(Ordering::Acquire));
445
446 #[cfg(all(feature = "rt", stack_guard_monitoring))]
447 {
448 let stack_guard = APP_CORE_STACK_GUARD.load(Ordering::Acquire);
449 stack_guard.write_volatile(esp_config::esp_config_int!(
450 u32,
451 "ESP_HAL_CONFIG_STACK_GUARD_VALUE"
452 ));
453 crate::debugger::set_stack_watchpoint(stack_guard as usize);
455 }
456 }
457
458 cfg_select! {
462 esp32 => {
463 crate::peripherals::DPORT::regs()
464 .appcpu_ctrl_d()
465 .write(|w| unsafe { w.appcpu_boot_addr().bits(0) });
466 }
467 _ => crate::rom::ets_set_appcpu_boot_addr(0),
468 }
469
470 unsafe { crate::interrupt::init_vectoring() };
472
473 enable_app_core_interrupts();
474
475 unsafe { CpuControl::start_core1_run::<F>() }
480}
481
482#[cfg(riscv)]
487#[unsafe(naked)]
488pub(crate) extern "C" fn start_core1_init<F>() -> !
489where
490 F: FnOnce(),
491{
492 core::arch::naked_asm!(
493 ".option push",
494 ".option norelax",
495 "la gp, __global_pointer$",
496 "li ra, 0", ".option pop",
498 "li t0, 0x2000",
503 "csrs mstatus, t0",
504 "li t0, 1",
505 "csrw fcsr, t0",
506 "li t0, 0x2000",
507 "csrc mstatus, t0",
508 "la t0, {stack_top}",
510 "lw sp, 0(t0)",
511 "j {init}",
512 stack_top = sym APP_CORE_STACK_TOP,
513 init = sym start_core1_init_impl::<F>,
514 )
515}
516
517#[cfg(riscv)]
518fn start_core1_init_impl<F>() -> !
519where
520 F: FnOnce(),
521{
522 #[cfg(soc_cpu_pma_needs_core1_setup)]
523 pma::set_core1_pma(unsafe {
524 &*core0_pma_slot::<F>(START_CORE1_FUNCTION.load(Ordering::Acquire))
525 });
526
527 crate::soc::branch_predictor::enable();
528
529 crate::rom::ets_set_appcpu_boot_addr(0);
532
533 unsafe {
534 #[cfg(all(feature = "rt", stack_guard_monitoring))]
535 {
536 let guard = APP_CORE_STACK_GUARD.load(Ordering::Acquire);
537 guard.write_volatile(esp_config::esp_config_int!(
538 u32,
539 "ESP_HAL_CONFIG_STACK_GUARD_VALUE"
540 ));
541 crate::debugger::set_stack_watchpoint(guard as usize);
542 }
543 crate::interrupt::init_vectoring();
544 }
545
546 enable_app_core_interrupts();
547
548 unsafe { CpuControl::start_core1_run::<F>() }
549}
550
551fn enable_app_core_interrupts() {
552 unsafe {
553 crate::interrupt::RunLevel::change(crate::interrupt::RunLevel::ThreadMode);
554 #[cfg(riscv)]
555 core::arch::asm!("csrsi mstatus, 8", options(nomem, nostack));
556 }
557 app_core_started();
558}
559
560cfg_select! {
563 feature = "rt" => {
564 use core::sync::atomic::AtomicBool;
565
566 use crate::interrupt::ipc::{defer_context_switch, resume_context_switch};
567
568 static APP_CORE_TAKES_IPC: AtomicBool = AtomicBool::new(false);
569
570 #[unsafe(export_name = "esp_hal_app_core_starting")]
572 fn app_core_starting() {
573 APP_CORE_TAKES_IPC.store(false, Ordering::Release);
574 unsafe { crate::interrupt::ipc::fail_stranded(Cpu::AppCpu) };
577 crate::interrupt::ipc::reset_context_switch_deferral(Cpu::AppCpu);
579 }
580
581 #[unsafe(export_name = "esp_hal_app_core_started")]
584 fn app_core_started() {
585 debug_assert_eq!(Cpu::current(), Cpu::AppCpu);
586 #[cfg(riscv)]
587 crate::interrupt::ipc::install_app();
588 APP_CORE_TAKES_IPC.store(true, Ordering::Release);
589 }
590
591 #[unsafe(export_name = "esp_hal_app_core_takes_ipc")]
594 pub(crate) fn app_core_takes_ipc() -> bool {
595 APP_CORE_TAKES_IPC.load(Ordering::Acquire)
596 }
597 }
598 _ => {
599 unsafe extern "Rust" {
600 #[link_name = "esp_hal_app_core_starting"]
601 safe fn app_core_starting();
602 #[link_name = "esp_hal_app_core_started"]
603 safe fn app_core_started();
604 #[link_name = "esp_hal_app_core_takes_ipc"]
605 pub(crate) safe fn app_core_takes_ipc() -> bool;
606 #[link_name = "esp_hal_defer_context_switch"]
607 safe fn defer_context_switch();
608 #[link_name = "esp_hal_resume_context_switch"]
609 fn resume_context_switch();
610 }
611 }
612}