From 90df83f885f0bc3eb70d8ef9014c3e75eac7122a Mon Sep 17 00:00:00 2001 From: eupn Date: Wed, 5 Apr 2023 16:35:01 +0400 Subject: [PATCH] kernel: revamp IRQ handling --- kernel/Cargo.toml | 3 +- kernel/bin/armv7a-unknown-xous-elf.a | Bin 3080 -> 3456 bytes kernel/src/arch/arm/asm.S | 96 ++++++++++++++++++++------- kernel/src/arch/arm/irq.rs | 38 +++++------ kernel/src/arch/arm/syscall.rs | 14 +++- 5 files changed, 101 insertions(+), 50 deletions(-) diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 32e1c7915..171d73140 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -26,11 +26,12 @@ xous-kernel = { package = "xous", version = "0.9.37", features = [ riscv = { version = "0.5.6", package = "xous-riscv" } # source is in "../imports/riscv-0.5.6" [target.'cfg(target_arch = "arm")'.dependencies] -armv7 = { git = "https://github.com/Foundation-Devices/armv7.git", branch = "update" } +armv7 = { git = "https://github.com/Foundation-Devices/armv7.git", branch = "update", features = ["critical-section"] } atsama5d27 = { git = "https://github.com/Foundation-Devices/atsama5d27.git", branch = "master" } xous-kernel = { package = "xous", version = "0.9.37", features = [ "v2p", ] } +critical-section = "1.1.1" [features] atsama5d27 = ["utralib/atsama5d27"] diff --git a/kernel/bin/armv7a-unknown-xous-elf.a b/kernel/bin/armv7a-unknown-xous-elf.a index 609d6824d799245a7dd67f8cbd41602ed5f76657..b3ba04fa0684e40c2ca621bad772a22e9bc74d18 100644 GIT binary patch delta 1589 zcmYjRUuaup6hHT0(yncCn@-!A#N4dgh+=bZ+O%7xgCG`*I2$*8Q_|Yrt;MFTH;s(p zNClNWC^)|Q;!7tKd=dwHo&@ntWpxVrFk$VeWEInbpbxHIf9Lk5&4J(f{myqz&iTH3 zzng14%h6As2%fQqh6npI*ORXt_hrT~1}!+F--0tTmNk^k8pgnYWnEtxsr=eXY+BZ% z6m?O=Qw`Tbw5Q*QC8E_Z)Ki6&h1CFsB`$qIjPfNSv4Jje5i>p$vbW?mTMJqgk3)K5B zZ&I%Bp<6lkaAf`5Cfy(3;5fe%uQTgA5sxiDUiA4VOv?Yby2U;2LR0Q`ZE>dqwYO`Q z&-#vB=Z)}M-d_vtE^N4!-;CPG`p28JyjPQ|H#V}>k2Y!IUX7~ZjYPG5ljyuqMhN;x5!u$Rn?nKj{H z>?|>+q>PL?zzYY4_c`E|_$%UyGFo|bHLbL#m9DhXmzHG=HD4>)PRT68o746?rc*90 znU@#JrM{^J$2Q-xoq2nXO3suy=}gVQ6=||mOkbcRHm8xKf_VG;wt@>mxG6-#fV3=i zlHT1ZMZ!ub8af&3ig|gW7_#1p@H85>BI0UV<@w@8HKq)wEitP;E7sJ@(u{~}QEd#B zk3^pqt-Q*r@jJAbfKD~V4)j`6T!j9 z3oF;@^VsW%*ev+sL zjwgtAnGK>5w2miH2afgbA5F%KIF7vBAL*>^B^6LtFQkQ9-R z5k+|cjG18ZVavtPwEbrJQekd(-Y%4uoLk#1_yAVKpRK*E*U;_^3UzVpNJ9G=kK`Y5 z_DFA~iuDw)0|9_<3QxWOVB_BIK=hrmf7@8IagK4xXOL$q8k}qdV6I`C0{E{k2e8#J z1ShX}gE`R{T!L==hzp-W0)Ki3*2sN;m4@bI4s`|#-}n%0?*Yw&dl2wp8SV7^;ouc4 za6|0Mvk5eO!PiWjW2^#r7>gj!zEH)Lwo@J7qMUCUzowY@vMs8e2og<-AKH3*QkeOb KU(UpB*!~Aiv?)CR delta 1325 zcmZ8gO-NKx6h3#}%;-4d_(r3K#-Ep_jDhoJ#+PXrs3o=tt^6q@C=JwCQL1&iLN=@W8p}`_6gyyZ4@ZUotcuoI99v ztiC=PHyY*-ohvtCT+@z5W16PLqZqVUG+JjEU=49&exdQ|bS|;Ej7$hqAq6tX8c^PF z!F_NqJP*z_&gZ~KNn81wPH(6k#akbn7r(0=zO8@h^rq<>sn@bs91#ohfhZFb^0}xG zU2<83+6w8?nnL6aAPPhl!lVsK6|AIEGi$!>YUZ0(UZj>*jMU63UHQDu-;||%tI&}5 z9KSl6l4+uJdKvpts2Za~{PyRQfPuip_*lQ*H`YIhVVJ0Id^pm{c^urqqoN5Nc1Dls zhOWymN;r>sOt01TsH}9hw!XLE?hT%Krq+WnK79}Tj13$CzLEvvTKCy=+oLD04NZ;@ULGEv#B{!y;$otDjwM7X#FA2?Hyq1| zwmGluoYa9s)`4^6{X6y(;Xfe90o=Bk-)F`kM+}h8M9qwtC*OJco;KzQ-24e|g=Y}I zO%&AvdEHy)I*X2ekPp2TiO0xu7YpFL8h4oa1$g!zfJ^Y#R3Tbs)jE;zz?;m%67=|# ztj9W*UO%uk2arxe52r{l&X`r?c3GGU7xAISz%&+Ry9Ku|{0R$ldw5~KR%_w=z^uoI zXPbgv%MQed#&MB-7LM8|LdISx@~5w=<`Z&FXcudT;H=A-*G4I8L3FQOw&qs^*Rhku JY<~AN{{h>Iw$=au diff --git a/kernel/src/arch/arm/asm.S b/kernel/src/arch/arm/asm.S index fd3e361cb..b5159d88a 100644 --- a/kernel/src/arch/arm/asm.S +++ b/kernel/src/arch/arm/asm.S @@ -22,6 +22,10 @@ #define I_BIT 0x80 #define F_BIT 0x40 +#define EXCEPTION_STACK_TOP 0xffff0000 +#define IRQ_STACK_TOP 0xfffd0000 +#define THREAD_CONTEXT_AREA 0xff804000 + .section .text.reset_vector, "ax" .global reset @@ -36,8 +40,8 @@ _exception_vectors_kernel: b trap_handler /* Prefetch Abort */ b trap_handler /* Data Abort */ .word 0 /* Empty */ - b trap_handler /* IRQ : read the AIC */ - b trap_handler /* FIQ */ + b irq_handler /* IRQ : read the AIC */ + b fiq_handler /* FIQ */ undef_vector: bkpt /* Issue debugger breakpoint */ @@ -51,6 +55,21 @@ _branch_main: ldr r4, =_exception_vectors_kernel mcr p15, 0, r4, c12, c0, 0 + /* Clear all faults if any */ + mov ip, #0 + mcr p15, 0, ip, c6, c0, 0 + mcr p15, 0, ip, c6, c0, 2 + mcr p15, 0, ip, c5, c0, 0 + mcr p15, 0, ip, c5, c0, 1 + + /* Configure IRQ/FIQ stack pointers */ + msr cpsr_c, #ARM_MODE_IRQ + ldr sp, =(IRQ_STACK_TOP) + msr cpsr_c, #ARM_MODE_FIQ + ldr sp, =(IRQ_STACK_TOP) + + msr cpsr_c, #ARM_MODE_SVC + /* Note that registers r0-r3 still contain the kernel arguments */ bl init bl kmain @@ -60,18 +79,18 @@ _branch_main: Syscall/IRQ/Abort trap handler. Saves all process context into a structure that is - mapped to each thread at 0xff804000 (THREAD_CONTEXT_AREA). + mapped to each thread at THREAD_CONTEXT_AREA. Return happens elsewhere. */ trap_handler: - ldr ip, =(0xff804000) + ldr ip, =(THREAD_CONTEXT_AREA) # If this is a syscall, then r0 contains a pointer to the syscall arguments structure # Due to it being clobbered by this routine's code, we have save it into the scratch area for future use str r0, [ip] /* Preserve r0 in the scratch field */ ldr r0, [ip, #4] /* Load current context number */ lsl r0, #7 /* Multiply by 32 */ - add ip, ip, r0 /* ip = 0xff804000 + (context_number * 32) */ + add ip, ip, r0 /* ip = THREAD_CONTEXT_AREA + (context_number * 32) */ # If the trap has originated in the user/system mode, then we need to store # the user/system mode registers that are currently banked out @@ -82,7 +101,7 @@ trap_handler: cmp r0, #ARM_MODE_USR /* Compare with 'User' mode */ beq _store_banked /* Save banked out user registers if coming from User mode */ - ldr r0, =(0xff804000) /* Restore r0 from the scratch field */ + ldr r0, =(THREAD_CONTEXT_AREA) /* Restore r0 from the scratch field */ ldr r0, [r0] stm ip, {r0 - r15} /* Save all the registers */ @@ -95,7 +114,7 @@ _store_banked: str r0, [ip] sub ip, ip, #(17 * 4) - ldr r0, =(0xff804000) /* Restore r0 from the scratch field */ + ldr r0, =(THREAD_CONTEXT_AREA) /* Restore r0 from the scratch field */ ldr r0, [r0] stm ip, {r0 - r7} /* Save the common registers */ @@ -108,8 +127,8 @@ _skip_store_banked: str lr, [ip] sub ip, #(4 * 19) - # Set a default exception/irq stack pointer (EXCEPTION_STACK_TOP) - ldr sp, =(0xffff0000) + # Set a default stack pointer + ldr sp, =(EXCEPTION_STACK_TOP) # Choose where to go next depending on the current mode: SVC (syscall), abort or IRQ mrs ip, cpsr /* Access the current mode register */ @@ -118,10 +137,6 @@ _skip_store_banked: bleq _swi_handler_rust /* Jump to SVC handler */ cmp ip, #ARM_MODE_ABT /* Compare with 'Abort' mode */ bleq _abort_handler_rust /* Jump to abort handler */ - cmp ip, #ARM_MODE_IRQ /* Compare with 'IRQ' mode */ - bleq irq_handler /* Jump to IRQ handler */ - cmp ip, #ARM_MODE_FIQ /* Compare with 'FIQ' mode */ - bleq fiq_handler /* Jump to FIQ handler */ # Any other unhandled exception goes here bkpt @@ -129,14 +144,49 @@ _skip_store_banked: irq_handler: fiq_handler: - # Disable further IRQs - mrs ip, cpsr - orr ip, ip, #I_BIT | F_BIT /* Disable interrupts */ - msr cpsr, ip + /* Save interrupt context on stack to allow nesting */ + sub lr, lr, #4 + stmfd sp!, {lr} + stmfd sp, {r0 - r12, sp, lr}^ + sub sp, sp, #(4 * 15) + mrs lr, spsr + stmfd sp!, {lr} + mrc p15, 0, lr, c13, c0, 2 /* Read hardware thread pointer register */ + stmfd sp!, {lr} + mrc p15, 0, lr, c2, c0, 0 /* Read TTBR0 */ + stmfd sp!, {lr} + mrc p15, 0, lr, c13, c0, 1 /* Read CONTEXTIDR */ + stmfd sp!, {lr} + mrc p15, 0, lr, c13, c0, 3 /* Read thread resume address scratch register */ + stmfd sp!, {lr} - bl _irq_handler_rust - bkpt - b . + /* Handle IRQs in supervisor mode (IRQs disabled) */ + msr cpsr_c, #(ARM_MODE_SVC | I_BIT | F_BIT) + # Set a default stack pointer + ldr sp, =(EXCEPTION_STACK_TOP) + + b _irq_handler_rust + +.global irq_handler_return +irq_handler_return: + /* As we're returning from ABORT mode, go back to IRQ mode (but with interrupts disabled) */ + msr cpsr_c, #(ARM_MODE_IRQ | I_BIT | F_BIT) + + /* Restore interrupt context and branch back to calling code */ + ldmia sp!, {lr} + mcr p15, 0, lr, c13, c0, 3 /* Set thread resume address scratch register */ + ldmia sp!, {lr} + mcr p15, 0, lr, c13, c0, 1 /* Set CONTEXTIDR */ + ldmia sp!, {lr} + mcr p15, 0, lr, c2, c0, 0 /* Set TTBR0 */ + bl flush_mmu + ldmia sp!, {lr} + mcr p15, 0, lr, c13, c0, 2 /* Set hardware thread pointer register */ + ldmia sp!, {lr} + msr spsr_cxsf, lr + ldmia sp, {r0 - r12, sp, lr}^ + add sp, sp, #(4 * 15) + ldmia sp!, {pc}^ /* A thread resume routine. @@ -149,11 +199,11 @@ _resume_trampoline: ldm ip, {sp, lr}^ # Switch privilege modes. - # Interrupts have to be disabled if we're in the IRQ mode because the code we're resuming to - # will have to service the interrupt source first before allowing more interrupts + # Keep interrupts disabled if we're going to switch to the user mode IRQ handler (r2 != 0) to prevent + # endless interrupt firing loop mrs ip, cpsr /* Access the current state register */ and ip, ip, #0x1f /* Extract the mode field */ - cmp ip, #ARM_MODE_IRQ + cmp r2, #0 /* r2 contains a "keep IRQs enabled" flag */ mrs ip, spsr bic ip, ip, #I_BIT | F_BIT /* Allow interrupts. They'll stay enabled unless we're in IRQ mode */ blne skip_disabling_interrupts diff --git a/kernel/src/arch/arm/irq.rs b/kernel/src/arch/arm/irq.rs index 2d92a48d6..a443eddfb 100644 --- a/kernel/src/arch/arm/irq.rs +++ b/kernel/src/arch/arm/irq.rs @@ -14,6 +14,7 @@ use crate::SystemServices; extern "Rust" { fn _xous_syscall_return_result(context: &Thread) -> !; + fn irq_handler_return() -> !; } static mut PREVIOUS_PAIR: Option<(PID, TID)> = None; @@ -240,16 +241,7 @@ pub extern "C" fn abort_handler() { // Resume the new thread within the same process. ArchProcess::with_current_mut(|p| { - // Adjust the program counter by the amount returned by the exception handler - /* FIXME let pc_adjust = a0 as isize; - if pc_adjust < 0 { - p.current_thread_mut().sepc -= pc_adjust.abs() as usize; - } else { - p.current_thread_mut().sepc += pc_adjust.abs() as usize; - }*/ - clear_fault(); - crate::arch::syscall::resume(pid.get() == 1, p.current_thread()) }); } @@ -272,7 +264,7 @@ pub extern "C" fn abort_handler() { }); } - RETURN_FROM_ISR if dfar == 0 => { + RETURN_FROM_ISR => { // If we hit this address, then an ISR has just returned. Since // we're in an interrupt context, it is safe to access this // global variable. @@ -294,12 +286,9 @@ pub extern "C" fn abort_handler() { clear_fault(); - ArchProcess::with_current_mut(|process| { - let mut curr_thread = process.current_thread_mut(); - curr_thread.resume_addr -= 4; - curr_thread.ret_addr = 0; - crate::arch::syscall::resume(current_pid().get() == 1, curr_thread) - }); + unsafe { + irq_handler_return(); + } } _ => { @@ -360,15 +349,18 @@ pub extern "C" fn _irq_handler_rust() { }); klog!("Pending irqs mask: {:032b}", irqs_pending); - // Safe to access globals since interrupts are disabled - unsafe { - if PREVIOUS_PAIR.is_none() { - let tid = crate::arch::process::current_tid(); - set_isr_return_pair(pid, tid); + critical_section::with(|_| { + // Safe to access globals since interrupts are disabled (critical section) + unsafe { + if PREVIOUS_PAIR.is_none() { + let tid = crate::arch::process::current_tid(); + set_isr_return_pair(pid, tid); + } } - } + }); + crate::irq::handle(irqs_pending).expect("Couldn't handle IRQ"); ArchProcess::with_current_mut(|process| { - crate::arch::syscall::resume(current_pid().get() == 1, process.current_thread()) + crate::arch::syscall::resume_no_irqs(current_pid().get() == 1, process.current_thread()) }) } diff --git a/kernel/src/arch/arm/syscall.rs b/kernel/src/arch/arm/syscall.rs index a4b01cf50..05cf905c5 100644 --- a/kernel/src/arch/arm/syscall.rs +++ b/kernel/src/arch/arm/syscall.rs @@ -4,7 +4,7 @@ use crate::services::Thread; extern "C" { - fn _resume_trampoline(mode: usize, thread: &Thread) -> !; + fn _resume_trampoline(mode: usize, thread: &Thread, enable_irqs: bool) -> !; } pub fn invoke( @@ -37,8 +37,16 @@ pub fn invoke( } pub fn resume(supervisor: bool, thread: &Thread) -> ! { + resume_inner(supervisor, thread, true) +} + +pub fn resume_no_irqs(supervisor: bool, thread: &Thread) -> ! { + resume_inner(supervisor, thread, false) +} + +fn resume_inner(supervisor: bool, thread: &Thread, enable_irqs: bool) -> ! { // Restore thread stack and PC, pass resume arguments via r0-r4 - klog!("resume: setting sp={:08x}, res_addr={:08x}, lr={:08x} | privileged: {:?} | ret: {:08x}", thread.sp, thread.resume_addr, thread.lr, supervisor, thread.ret_addr); + klog!("resume: setting sp={:08x}, res_addr={:08x}, lr={:08x} | privileged: {:?} | irqs: {:?} | ret: {:08x}", thread.sp, thread.resume_addr, thread.lr, supervisor, enable_irqs, thread.ret_addr); // See ARM ARM // B1.3.1 ARM processor modes @@ -49,6 +57,6 @@ pub fn resume(supervisor: bool, thread: &Thread) -> ! { }; unsafe { - _resume_trampoline(mode, thread); + _resume_trampoline(mode, thread, enable_irqs); } }