Nothing to see here, move along meow
1use core::arch::global_asm;
2
3use spin::Once;
4use x86_64::VirtAddr;
5use x86_64::structures::idt::{InterruptDescriptorTable, InterruptStackFrame};
6
7use super::gdt::DOUBLE_FAULT_IST_INDEX;
8use crate::proc::context::CpuContext;
9
10#[derive(Debug, Clone, Copy, PartialEq, Eq)]
11pub struct IrqVector(u8);
12
13impl IrqVector {
14 pub const fn new(v: u8) -> Self {
15 assert!(
16 v >= 32,
17 "IrqVector must be >= 32 (0-31 reserved for CPU exceptions)"
18 );
19 Self(v)
20 }
21
22 pub const fn try_new(v: u8) -> Option<Self> {
23 if v >= 32 { Some(Self(v)) } else { None }
24 }
25
26 #[inline]
27 pub(crate) const fn raw(self) -> u8 {
28 self.0
29 }
30
31 #[inline]
32 pub const fn as_usize(self) -> usize {
33 self.0 as usize
34 }
35}
36
37impl core::fmt::Display for IrqVector {
38 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
39 write!(f, "{}", self.0)
40 }
41}
42
43#[derive(Debug, Clone, Copy, PartialEq, Eq)]
44pub enum CpuMode {
45 Kernel,
46 User,
47}
48
49impl CpuMode {
50 fn from_interrupt_frame(frame: &InterruptStackFrame) -> Self {
51 if frame.code_segment.0 & 3 != 0 {
52 Self::User
53 } else {
54 Self::Kernel
55 }
56 }
57}
58
59pub const TIMER_VECTOR: IrqVector = IrqVector::new(32);
60pub const ERROR_VECTOR: IrqVector = IrqVector::new(33);
61pub const COM1_VECTOR: IrqVector = IrqVector::new(36);
62pub const KBD_VECTOR: IrqVector = IrqVector::new(37);
63pub const VIRTIO_NET_VECTOR: IrqVector = IrqVector::new(38);
64pub const IOMMU_FAULT_VECTOR: IrqVector = IrqVector::new(39);
65pub const NVME_VECTOR: IrqVector = IrqVector::new(40);
66pub const IPI_RESCHEDULE_VECTOR: IrqVector = IrqVector::new(0xF0);
67pub const IPI_TLB_SHOOTDOWN_VECTOR: IrqVector = IrqVector::new(0xF1);
68pub const IPI_HALT_VECTOR: IrqVector = IrqVector::new(0xF2);
69pub const IPI_FUNC_CALL_VECTOR: IrqVector = IrqVector::new(0xF3);
70pub const SPURIOUS_VECTOR: IrqVector = IrqVector::new(255);
71
72unsafe extern "C" {
73 fn timer_entry();
74 fn dev_irq_entry();
75 fn nvme_irq_entry();
76 fn com1_entry();
77 fn kbd_entry();
78 fn iommu_fault_entry();
79 fn pf_entry();
80 fn gpf_entry();
81 fn errcode_entry_invalid_tss();
82 fn errcode_entry_seg_not_present();
83 fn errcode_entry_stack_segment();
84 fn errcode_entry_alignment_check();
85 fn errcode_entry_security_exception();
86}
87
88static IDT: Once<InterruptDescriptorTable> = Once::new();
89
90fn build_idt() -> InterruptDescriptorTable {
91 let mut idt = InterruptDescriptorTable::new();
92
93 idt.divide_error.set_handler_fn(divide_error_handler);
94 idt.debug.set_handler_fn(debug_handler);
95 unsafe {
96 idt.non_maskable_interrupt
97 .set_handler_fn(nmi_handler)
98 .set_stack_index(super::gdt::NMI_IST_INDEX);
99 }
100 idt.breakpoint.set_handler_fn(breakpoint_handler);
101 idt.overflow.set_handler_fn(overflow_handler);
102 idt.bound_range_exceeded.set_handler_fn(bound_range_handler);
103 idt.invalid_opcode.set_handler_fn(invalid_opcode_handler);
104 idt.device_not_available
105 .set_handler_fn(device_not_available_handler);
106 unsafe {
107 idt.double_fault
108 .set_handler_fn(double_fault_handler)
109 .set_stack_index(DOUBLE_FAULT_IST_INDEX);
110 }
111 unsafe {
112 idt.invalid_tss
113 .set_handler_addr(VirtAddr::new(errcode_entry_invalid_tss as *const () as u64));
114 idt.segment_not_present.set_handler_addr(VirtAddr::new(
115 errcode_entry_seg_not_present as *const () as u64,
116 ));
117 idt.stack_segment_fault.set_handler_addr(VirtAddr::new(
118 errcode_entry_stack_segment as *const () as u64,
119 ));
120 idt.general_protection_fault
121 .set_handler_addr(VirtAddr::new(gpf_entry as *const () as u64));
122 idt.page_fault
123 .set_handler_addr(VirtAddr::new(pf_entry as *const () as u64));
124 }
125 idt.x87_floating_point.set_handler_fn(x87_fp_handler);
126 unsafe {
127 idt.alignment_check.set_handler_addr(VirtAddr::new(
128 errcode_entry_alignment_check as *const () as u64,
129 ));
130 }
131 unsafe {
132 idt.machine_check
133 .set_handler_fn(machine_check_handler)
134 .set_stack_index(super::gdt::MACHINE_CHECK_IST_INDEX);
135 }
136 idt.simd_floating_point.set_handler_fn(simd_fp_handler);
137 idt.virtualization.set_handler_fn(virtualization_handler);
138 unsafe {
139 idt.security_exception.set_handler_addr(VirtAddr::new(
140 errcode_entry_security_exception as *const () as u64,
141 ));
142 }
143
144 unsafe {
145 idt[TIMER_VECTOR.0]
146 .set_handler_addr(VirtAddr::new(timer_entry as *const () as u64))
147 .set_stack_index(super::gdt::GENERAL_IRQ_IST_INDEX);
148 }
149 idt[ERROR_VECTOR.0].set_handler_fn(apic_error_handler);
150 unsafe {
151 idt[COM1_VECTOR.0]
152 .set_handler_addr(VirtAddr::new(com1_entry as *const () as u64))
153 .set_stack_index(super::gdt::GENERAL_IRQ_IST_INDEX);
154 idt[KBD_VECTOR.0]
155 .set_handler_addr(VirtAddr::new(kbd_entry as *const () as u64))
156 .set_stack_index(super::gdt::GENERAL_IRQ_IST_INDEX);
157 }
158 unsafe {
159 idt[VIRTIO_NET_VECTOR.0]
160 .set_handler_addr(VirtAddr::new(dev_irq_entry as *const () as u64))
161 .set_stack_index(super::gdt::GENERAL_IRQ_IST_INDEX);
162 idt[NVME_VECTOR.0]
163 .set_handler_addr(VirtAddr::new(nvme_irq_entry as *const () as u64))
164 .set_stack_index(super::gdt::GENERAL_IRQ_IST_INDEX);
165 idt[IOMMU_FAULT_VECTOR.0]
166 .set_handler_addr(VirtAddr::new(iommu_fault_entry as *const () as u64))
167 .set_stack_index(super::gdt::GENERAL_IRQ_IST_INDEX);
168 }
169 idt[IPI_RESCHEDULE_VECTOR.0].set_handler_fn(ipi_reschedule_handler);
170 idt[IPI_TLB_SHOOTDOWN_VECTOR.0].set_handler_fn(ipi_tlb_shootdown_handler);
171 idt[IPI_HALT_VECTOR.0].set_handler_fn(ipi_halt_handler);
172 idt[IPI_FUNC_CALL_VECTOR.0].set_handler_fn(ipi_func_call_handler);
173 idt[SPURIOUS_VECTOR.0].set_handler_fn(spurious_handler);
174
175 idt
176}
177
178pub fn init() {
179 let idt = IDT.call_once(build_idt);
180 idt.load();
181}
182
183#[unsafe(no_mangle)]
184extern "C" fn timer_handler_inner(ctx: *mut CpuContext) {
185 super::syscall::enter_ist();
186 assert!(
187 !ctx.is_null() && (ctx as u64) >= 0xFFFF_8000_0000_0000,
188 "timer_handler_inner: ctx pointer outside kernel address space"
189 );
190 assert!(
191 (ctx as usize).is_multiple_of(core::mem::align_of::<CpuContext>()),
192 "timer_handler_inner: ctx pointer misaligned"
193 );
194 let ctx = unsafe { &mut *ctx };
195 super::apic::tick();
196 unsafe { super::apic::end_of_interrupt() };
197 crate::sched::timer_tick(ctx);
198 super::syscall::leave_ist();
199}
200
201#[unsafe(no_mangle)]
202extern "C" fn timer_handler_kernel() {
203 super::syscall::enter_ist();
204 super::apic::tick();
205 unsafe { super::apic::end_of_interrupt() };
206 super::syscall::leave_ist();
207}
208
209extern "x86-interrupt" fn divide_error_handler(frame: InterruptStackFrame) {
210 let mode = conditional_swapgs(&frame);
211 crate::klog!("idt", error, "division error\n{:#?}", frame);
212 kill_user_or_halt(&frame, "division error", mode);
213}
214
215extern "x86-interrupt" fn debug_handler(frame: InterruptStackFrame) {
216 let mode = conditional_swapgs(&frame);
217 crate::klog!("idt", "debug\n{:#?}", frame);
218 match mode {
219 CpuMode::User => unsafe { core::arch::asm!("swapgs", options(nomem, nostack)) },
220 CpuMode::Kernel => {}
221 }
222}
223
224extern "x86-interrupt" fn nmi_handler(frame: InterruptStackFrame) {
225 let mode = conditional_swapgs(&frame);
226 crate::klog!("idt", "non-maskable interrupt\n{:#?}", frame);
227 match mode {
228 CpuMode::User => unsafe { core::arch::asm!("swapgs", options(nomem, nostack)) },
229 CpuMode::Kernel => {}
230 }
231}
232
233extern "x86-interrupt" fn breakpoint_handler(frame: InterruptStackFrame) {
234 let mode = conditional_swapgs(&frame);
235 #[cfg(not(lancer_test))]
236 crate::klog!("idt", "breakpoint\n{:#?}", frame);
237 #[cfg(lancer_test)]
238 crate::tests::interrupts::BREAKPOINT_HIT.store(true, core::sync::atomic::Ordering::SeqCst);
239 match mode {
240 CpuMode::User => unsafe { core::arch::asm!("swapgs", options(nomem, nostack)) },
241 CpuMode::Kernel => {}
242 }
243}
244
245extern "x86-interrupt" fn overflow_handler(frame: InterruptStackFrame) {
246 let mode = conditional_swapgs(&frame);
247 crate::klog!("idt", error, "overflow\n{:#?}", frame);
248 kill_user_or_halt(&frame, "overflow", mode);
249}
250
251extern "x86-interrupt" fn bound_range_handler(frame: InterruptStackFrame) {
252 let mode = conditional_swapgs(&frame);
253 crate::klog!("idt", error, "bound range exceeded\n{:#?}", frame);
254 kill_user_or_halt(&frame, "bound range exceeded", mode);
255}
256
257extern "x86-interrupt" fn invalid_opcode_handler(frame: InterruptStackFrame) {
258 let mode = conditional_swapgs(&frame);
259 crate::klog!("idt", error, "invalid opcode\n{:#?}", frame);
260 kill_user_or_halt(&frame, "invalid opcode", mode);
261}
262
263extern "x86-interrupt" fn device_not_available_handler(frame: InterruptStackFrame) {
264 let mode = conditional_swapgs(&frame);
265 crate::klog!("idt", error, "device not available\n{:#?}", frame);
266 kill_user_or_halt(&frame, "device not available", mode);
267}
268
269extern "x86-interrupt" fn double_fault_handler(frame: InterruptStackFrame, error_code: u64) -> ! {
270 crate::klog!(
271 "idt",
272 error,
273 "double fault error_code {} rsp {:#x}\n{:#?}",
274 error_code,
275 frame.stack_pointer.as_u64(),
276 frame
277 );
278 halt_loop();
279}
280
281#[unsafe(no_mangle)]
282extern "C" fn pf_user_handler(ctx: *const CpuContext, error_code: u64, fault_addr: u64) -> ! {
283 let ctx = unsafe { &*ctx };
284 let pid = crate::sched::current();
285 let nbuf = proc_name_buf(pid);
286 let cause = match (
287 error_code & 1 != 0,
288 error_code & 2 != 0,
289 error_code & 0x10 != 0,
290 ) {
291 (false, _, _) => "not-present",
292 (true, true, _) => "write-protect",
293 (true, false, true) => "exec-protect",
294 (true, false, false) => "read-protect",
295 };
296 crate::klog!(
297 "mem",
298 error,
299 "page fault {} pid {} {} rip {:#x} addr {:#x} err {:#x} destroying\n rax={:#018x} rbx={:#018x} rcx={:#018x} rdx={:#018x}\n rsi={:#018x} rdi={:#018x} rbp={:#018x} rsp={:#018x}\n r8 ={:#018x} r9 ={:#018x} r10={:#018x} r11={:#018x}\n r12={:#018x} r13={:#018x} r14={:#018x} r15={:#018x}\n rflags={:#018x} cs={:#x} ss={:#x}",
300 cause,
301 pid.raw(),
302 proc_name_str(&nbuf),
303 ctx.rip,
304 fault_addr,
305 error_code,
306 ctx.rax,
307 ctx.rbx,
308 ctx.rcx,
309 ctx.rdx,
310 ctx.rsi,
311 ctx.rdi,
312 ctx.rbp,
313 ctx.rsp,
314 ctx.r8,
315 ctx.r9,
316 ctx.r10,
317 ctx.r11,
318 ctx.r12,
319 ctx.r13,
320 ctx.r14,
321 ctx.r15,
322 ctx.rflags,
323 ctx.cs,
324 ctx.ss
325 );
326 dump_user_stack(pid, ctx.rsp);
327 kill_and_rescue(pid)
328}
329
330fn dump_user_stack(pid: crate::types::Pid, user_rsp: u64) {
331 let pml4 = match crate::proc::PROCESSES.try_lock() {
332 Some(pt) => match pt.exec(pid) {
333 Some(e) => e.pml4_phys.raw(),
334 None => return,
335 },
336 None => return,
337 };
338
339 let hhdm = crate::mem::addr::hhdm_offset();
340 crate::kprint!(" stack:");
341 (0u64..8).for_each(|i| {
342 let addr = user_rsp.wrapping_add(i * 8);
343 let page_base = addr & !0xFFF;
344 match crate::syscall::resolve_user_page(pml4, page_base, false, false) {
345 Some(frame_phys) => {
346 let offset = addr & 0xFFF;
347 let ptr = (frame_phys + hhdm + offset) as *const u64;
348 let val = unsafe { ptr.read() };
349 crate::kprint!(" {:#x}", val);
350 }
351 None => {
352 crate::kprint!(" ????????");
353 }
354 }
355 });
356 crate::kprintln!();
357}
358
359#[unsafe(no_mangle)]
360extern "C" fn pf_kernel_handler(rip: u64, error_code: u64, fault_addr: u64) -> ! {
361 crate::klog!(
362 "kern",
363 error,
364 "page fault rip {:#x} addr {:#x} err {:#x}",
365 rip,
366 fault_addr,
367 error_code
368 );
369 halt_loop()
370}
371
372#[unsafe(no_mangle)]
373extern "C" fn gpf_user_handler(ctx: *const CpuContext) -> ! {
374 let ctx = unsafe { &*ctx };
375 let pid = crate::sched::current();
376 let nbuf = proc_name_buf(pid);
377 crate::klog!(
378 "mem",
379 error,
380 "gpf pid {} {} rip {:#x} destroying\n rax={:#018x} rbx={:#018x} rcx={:#018x} rdx={:#018x}\n rsi={:#018x} rdi={:#018x} rbp={:#018x} rsp={:#018x}\n r8 ={:#018x} r9 ={:#018x} r10={:#018x} r11={:#018x}\n r12={:#018x} r13={:#018x} r14={:#018x} r15={:#018x}\n rflags={:#018x} cs={:#x} ss={:#x}",
381 pid.raw(),
382 proc_name_str(&nbuf),
383 ctx.rip,
384 ctx.rax,
385 ctx.rbx,
386 ctx.rcx,
387 ctx.rdx,
388 ctx.rsi,
389 ctx.rdi,
390 ctx.rbp,
391 ctx.rsp,
392 ctx.r8,
393 ctx.r9,
394 ctx.r10,
395 ctx.r11,
396 ctx.r12,
397 ctx.r13,
398 ctx.r14,
399 ctx.r15,
400 ctx.rflags,
401 ctx.cs,
402 ctx.ss
403 );
404 kill_and_rescue(pid)
405}
406
407#[unsafe(no_mangle)]
408extern "C" fn gpf_kernel_handler(rip: u64, error_code: u64) -> ! {
409 crate::klog!(
410 "kern",
411 error,
412 "gpf rip {:#x} error_code {}",
413 rip,
414 error_code
415 );
416 halt_loop()
417}
418
419const ERRCODE_NAMES: [&str; 6] = [
420 "invalid tss",
421 "segment not present",
422 "stack segment fault",
423 "alignment check",
424 "security exception",
425 "unknown",
426];
427
428#[unsafe(no_mangle)]
429extern "C" fn errcode_user_handler(rip: u64, error_code: u64, exception_id: u64) -> ! {
430 let exc_name = ERRCODE_NAMES
431 .get(exception_id as usize)
432 .copied()
433 .unwrap_or("unknown");
434 let pid = crate::sched::current();
435 let nbuf = proc_name_buf(pid);
436 crate::klog!(
437 "mem",
438 error,
439 "{} pid {} {} rip {:#x} error_code {} destroying",
440 exc_name,
441 pid.raw(),
442 proc_name_str(&nbuf),
443 rip,
444 error_code
445 );
446 kill_and_rescue(pid)
447}
448
449#[unsafe(no_mangle)]
450extern "C" fn errcode_kernel_handler(rip: u64, error_code: u64, exception_id: u64) -> ! {
451 let name = ERRCODE_NAMES
452 .get(exception_id as usize)
453 .copied()
454 .unwrap_or("unknown");
455 crate::klog!(
456 "kern",
457 error,
458 "{} rip {:#x} error_code {}",
459 name,
460 rip,
461 error_code
462 );
463 halt_loop()
464}
465
466fn proc_name_buf(pid: crate::types::Pid) -> [u8; 32] {
467 let mut buf = [0u8; 32];
468 match crate::proc::PROCESSES.try_lock() {
469 Some(ptable) => match ptable.exec(pid) {
470 Some(e) => {
471 let name = e.name_str().as_bytes();
472 let len = name.len().min(32);
473 buf[..len].copy_from_slice(&name[..len]);
474 }
475 None => {
476 buf[0] = b'?';
477 }
478 },
479 None => {
480 buf[0] = b'?';
481 }
482 }
483 buf
484}
485
486fn proc_name_str(buf: &[u8; 32]) -> &str {
487 let len = buf.iter().position(|&b| b == 0).unwrap_or(32);
488 core::str::from_utf8(&buf[..len]).unwrap_or("?")
489}
490
491fn kill_and_rescue(pid: crate::types::Pid) -> ! {
492 match crate::proc::PROCESSES.try_lock() {
493 Some(mut ptable) => {
494 ptable.zombify(pid);
495 crate::sched::rescue_with_ptable(ptable)
496 }
497 None => {
498 crate::klog!("kern", error, "process table already locked, halting");
499 halt_loop()
500 }
501 }
502}
503
504extern "x86-interrupt" fn x87_fp_handler(frame: InterruptStackFrame) {
505 let mode = conditional_swapgs(&frame);
506 crate::klog!("idt", error, "x87 floating point\n{:#?}", frame);
507 kill_user_or_halt(&frame, "x87 floating point", mode);
508}
509
510extern "x86-interrupt" fn machine_check_handler(frame: InterruptStackFrame) -> ! {
511 crate::klog!("idt", error, "machine check\n{:#?}", frame);
512 halt_loop();
513}
514
515extern "x86-interrupt" fn simd_fp_handler(frame: InterruptStackFrame) {
516 let mode = conditional_swapgs(&frame);
517 crate::klog!("idt", error, "simd floating point\n{:#?}", frame);
518 kill_user_or_halt(&frame, "simd floating point", mode);
519}
520
521extern "x86-interrupt" fn virtualization_handler(frame: InterruptStackFrame) {
522 let mode = conditional_swapgs(&frame);
523 crate::klog!("idt", error, "virtualization\n{:#?}", frame);
524 kill_user_or_halt(&frame, "virtualization", mode);
525}
526
527#[unsafe(no_mangle)]
528extern "C" fn com1_handler_inner() {
529 super::syscall::enter_ist();
530 crate::irq::handle_irq(COM1_VECTOR);
531 unsafe { super::apic::end_of_interrupt() };
532 super::syscall::leave_ist();
533}
534
535#[unsafe(no_mangle)]
536extern "C" fn kbd_handler_inner() {
537 super::syscall::enter_ist();
538 crate::irq::handle_irq(KBD_VECTOR);
539 unsafe { super::apic::end_of_interrupt() };
540 super::syscall::leave_ist();
541}
542
543#[unsafe(no_mangle)]
544extern "C" fn iommu_fault_handler_inner() {
545 super::syscall::enter_ist();
546 crate::iommu::fault::handle_fault();
547 unsafe { super::apic::end_of_interrupt() };
548 super::syscall::leave_ist();
549}
550
551#[unsafe(no_mangle)]
552extern "C" fn dev_irq_handler_inner(ctx: *mut CpuContext) {
553 super::syscall::enter_ist();
554 let ctx = unsafe { &mut *ctx };
555 crate::irq::handle_irq(VIRTIO_NET_VECTOR);
556 unsafe { super::apic::end_of_interrupt() };
557 crate::sched::timer_tick(ctx);
558 super::syscall::leave_ist();
559}
560
561#[unsafe(no_mangle)]
562extern "C" fn dev_irq_handler_kernel() {
563 super::syscall::enter_ist();
564 crate::irq::handle_irq(VIRTIO_NET_VECTOR);
565 unsafe { super::apic::end_of_interrupt() };
566 super::syscall::leave_ist();
567}
568
569#[unsafe(no_mangle)]
570extern "C" fn nvme_irq_handler_inner(ctx: *mut CpuContext) {
571 super::syscall::enter_ist();
572 let ctx = unsafe { &mut *ctx };
573 crate::irq::handle_irq(NVME_VECTOR);
574 unsafe { super::apic::end_of_interrupt() };
575 crate::sched::timer_tick(ctx);
576 super::syscall::leave_ist();
577}
578
579#[unsafe(no_mangle)]
580extern "C" fn nvme_irq_handler_kernel() {
581 super::syscall::enter_ist();
582 crate::irq::handle_irq(NVME_VECTOR);
583 unsafe { super::apic::end_of_interrupt() };
584 super::syscall::leave_ist();
585}
586
587extern "x86-interrupt" fn apic_error_handler(_frame: InterruptStackFrame) {
588 let esr = unsafe { super::apic::read_and_clear_esr() };
589 crate::klog!("apic", error, "error interrupt esr {:#x}", esr);
590 unsafe { super::apic::end_of_interrupt() };
591}
592
593extern "x86-interrupt" fn ipi_reschedule_handler(_frame: InterruptStackFrame) {
594 unsafe { super::apic::end_of_interrupt() };
595}
596
597extern "x86-interrupt" fn ipi_tlb_shootdown_handler(_frame: InterruptStackFrame) {
598 unsafe { super::apic::end_of_interrupt() };
599}
600
601extern "x86-interrupt" fn ipi_halt_handler(_frame: InterruptStackFrame) {
602 unsafe { super::apic::end_of_interrupt() };
603 x86_64::instructions::interrupts::disable();
604 loop {
605 x86_64::instructions::hlt();
606 }
607}
608
609extern "x86-interrupt" fn ipi_func_call_handler(_frame: InterruptStackFrame) {
610 unsafe { super::apic::end_of_interrupt() };
611}
612
613extern "x86-interrupt" fn spurious_handler(_frame: InterruptStackFrame) {}
614
615fn conditional_swapgs(frame: &InterruptStackFrame) -> CpuMode {
616 let mode = CpuMode::from_interrupt_frame(frame);
617 match mode {
618 CpuMode::User => unsafe { core::arch::asm!("swapgs", options(nomem, nostack)) },
619 CpuMode::Kernel => {}
620 }
621 mode
622}
623
624fn kill_user_or_halt(frame: &InterruptStackFrame, exc_name: &str, mode: CpuMode) -> ! {
625 match mode {
626 CpuMode::Kernel => {
627 crate::klog!("kern", error, "{} halting", exc_name);
628 halt_loop()
629 }
630 CpuMode::User => {
631 let pid = crate::sched::current();
632 let nbuf = proc_name_buf(pid);
633 crate::klog!(
634 "mem",
635 error,
636 "{} pid {} {} at {:#x} destroying",
637 exc_name,
638 pid.raw(),
639 proc_name_str(&nbuf),
640 frame.instruction_pointer.as_u64()
641 );
642 kill_and_rescue(pid)
643 }
644 }
645}
646
647fn halt_loop() -> ! {
648 x86_64::instructions::interrupts::disable();
649 loop {
650 x86_64::instructions::hlt();
651 }
652}
653
654global_asm!(
655 r#"
656.global timer_entry
657.type timer_entry, @function
658timer_entry:
659 // the cpu already pushed RIP +0, CS +8, RFLAGS +16, RSP +24, SS +32
660 // test CPL bits in CS to detect user-mode origin
661 test qword ptr [rsp + 8], 3
662 jz .kernel_timer
663
664 // swap to kernel GS base for the user-mode path
665 swapgs
666
667 // rearrange interrupt frame to match CpuContext tail layout
668 // CPU: [RIP(0), CS(8), RFLAGS(16), RSP_user(24), SS(32)]
669 // CpuContext: [rip(0), rsp_user(8), rflags(16), cs(24), ss(32)]
670 // swap CS(+8) and RSP_user(+24); offsets shifted +8 by push
671 push rax
672 mov rax, [rsp + 16]
673 xchg rax, [rsp + 32]
674 mov [rsp + 16], rax
675 pop rax
676
677 // push all 15 GPRs to build full CpuContext on stack
678 push r15
679 push r14
680 push r13
681 push r12
682 push r11
683 push r10
684 push r9
685 push r8
686 push rbp
687 push rdi
688 push rsi
689 push rdx
690 push rcx
691 push rbx
692 push rax
693
694 mov rdi, gs:[{fpu_ptr}]
695 mov eax, dword ptr gs:[{xsave_lo}]
696 mov edx, dword ptr gs:[{xsave_hi}]
697 test eax, eax
698 jz .tmr_fxsave
699 xsave64 [rdi]
700 jmp .tmr_fpu_saved
701.tmr_fxsave:
702 fxsave64 [rdi]
703.tmr_fpu_saved:
704
705 cld
706 mov rdi, rsp
707 call timer_handler_inner
708
709 mov rdi, gs:[{fpu_ptr}]
710 mov eax, dword ptr gs:[{xsave_lo}]
711 mov edx, dword ptr gs:[{xsave_hi}]
712 test eax, eax
713 jz .tmr_fxrstor
714 xrstor64 [rdi]
715 jmp .tmr_fpu_restored
716.tmr_fxrstor:
717 fxrstor64 [rdi]
718.tmr_fpu_restored:
719
720 pop rax
721 pop rbx
722 pop rcx
723 pop rdx
724 pop rsi
725 pop rdi
726 pop rbp
727 pop r8
728 pop r9
729 pop r10
730 pop r11
731 pop r12
732 pop r13
733 pop r14
734 pop r15
735
736 // reverse swap to restore iretq layout
737 // current: [rip(0), rsp_user(8), rflags(16), cs(24), ss(32)]
738 // iretq: [RIP(0), CS(8), RFLAGS(16), RSP(24), SS(32)]
739 // offsets shifted +8 by push
740 push rax
741 mov rax, [rsp + 16]
742 xchg rax, [rsp + 32]
743 mov [rsp + 16], rax
744 pop rax
745
746 swapgs
747 iretq
748
749.kernel_timer:
750 push rax
751 push rcx
752 push rdx
753 push rsi
754 push rdi
755 push r8
756 push r9
757 push r10
758 push r11
759 sub rsp, 256
760 movups [rsp], xmm0
761 movups [rsp + 16], xmm1
762 movups [rsp + 32], xmm2
763 movups [rsp + 48], xmm3
764 movups [rsp + 64], xmm4
765 movups [rsp + 80], xmm5
766 movups [rsp + 96], xmm6
767 movups [rsp + 112], xmm7
768 movups [rsp + 128], xmm8
769 movups [rsp + 144], xmm9
770 movups [rsp + 160], xmm10
771 movups [rsp + 176], xmm11
772 movups [rsp + 192], xmm12
773 movups [rsp + 208], xmm13
774 movups [rsp + 224], xmm14
775 movups [rsp + 240], xmm15
776 cld
777 call timer_handler_kernel
778 movups xmm0, [rsp]
779 movups xmm1, [rsp + 16]
780 movups xmm2, [rsp + 32]
781 movups xmm3, [rsp + 48]
782 movups xmm4, [rsp + 64]
783 movups xmm5, [rsp + 80]
784 movups xmm6, [rsp + 96]
785 movups xmm7, [rsp + 112]
786 movups xmm8, [rsp + 128]
787 movups xmm9, [rsp + 144]
788 movups xmm10, [rsp + 160]
789 movups xmm11, [rsp + 176]
790 movups xmm12, [rsp + 192]
791 movups xmm13, [rsp + 208]
792 movups xmm14, [rsp + 224]
793 movups xmm15, [rsp + 240]
794 add rsp, 256
795 pop r11
796 pop r10
797 pop r9
798 pop r8
799 pop rdi
800 pop rsi
801 pop rdx
802 pop rcx
803 pop rax
804 iretq
805
806.size timer_entry, . - timer_entry
807
808.macro DEV_IRQ_ENTRY name, user_handler, kernel_handler
809.global \name
810.type \name, @function
811\name:
812 test qword ptr [rsp + 8], 3
813 jz .L_kernel_\name
814
815 swapgs
816
817 push rax
818 mov rax, [rsp + 16]
819 xchg rax, [rsp + 32]
820 mov [rsp + 16], rax
821 pop rax
822
823 push r15
824 push r14
825 push r13
826 push r12
827 push r11
828 push r10
829 push r9
830 push r8
831 push rbp
832 push rdi
833 push rsi
834 push rdx
835 push rcx
836 push rbx
837 push rax
838
839 mov rdi, gs:[{fpu_ptr}]
840 mov eax, dword ptr gs:[{xsave_lo}]
841 mov edx, dword ptr gs:[{xsave_hi}]
842 test eax, eax
843 jz .L_fxsave_\name
844 xsave64 [rdi]
845 jmp .L_fpu_saved_\name
846.L_fxsave_\name:
847 fxsave64 [rdi]
848.L_fpu_saved_\name:
849
850 cld
851 mov rdi, rsp
852 call \user_handler
853
854 mov rdi, gs:[{fpu_ptr}]
855 mov eax, dword ptr gs:[{xsave_lo}]
856 mov edx, dword ptr gs:[{xsave_hi}]
857 test eax, eax
858 jz .L_fxrstor_\name
859 xrstor64 [rdi]
860 jmp .L_fpu_restored_\name
861.L_fxrstor_\name:
862 fxrstor64 [rdi]
863.L_fpu_restored_\name:
864
865 pop rax
866 pop rbx
867 pop rcx
868 pop rdx
869 pop rsi
870 pop rdi
871 pop rbp
872 pop r8
873 pop r9
874 pop r10
875 pop r11
876 pop r12
877 pop r13
878 pop r14
879 pop r15
880
881 push rax
882 mov rax, [rsp + 16]
883 xchg rax, [rsp + 32]
884 mov [rsp + 16], rax
885 pop rax
886
887 swapgs
888 iretq
889
890.L_kernel_\name:
891 push rax
892 push rcx
893 push rdx
894 push rsi
895 push rdi
896 push r8
897 push r9
898 push r10
899 push r11
900 sub rsp, 256
901 movups [rsp], xmm0
902 movups [rsp + 16], xmm1
903 movups [rsp + 32], xmm2
904 movups [rsp + 48], xmm3
905 movups [rsp + 64], xmm4
906 movups [rsp + 80], xmm5
907 movups [rsp + 96], xmm6
908 movups [rsp + 112], xmm7
909 movups [rsp + 128], xmm8
910 movups [rsp + 144], xmm9
911 movups [rsp + 160], xmm10
912 movups [rsp + 176], xmm11
913 movups [rsp + 192], xmm12
914 movups [rsp + 208], xmm13
915 movups [rsp + 224], xmm14
916 movups [rsp + 240], xmm15
917 cld
918 call \kernel_handler
919 movups xmm0, [rsp]
920 movups xmm1, [rsp + 16]
921 movups xmm2, [rsp + 32]
922 movups xmm3, [rsp + 48]
923 movups xmm4, [rsp + 64]
924 movups xmm5, [rsp + 80]
925 movups xmm6, [rsp + 96]
926 movups xmm7, [rsp + 112]
927 movups xmm8, [rsp + 128]
928 movups xmm9, [rsp + 144]
929 movups xmm10, [rsp + 160]
930 movups xmm11, [rsp + 176]
931 movups xmm12, [rsp + 192]
932 movups xmm13, [rsp + 208]
933 movups xmm14, [rsp + 224]
934 movups xmm15, [rsp + 240]
935 add rsp, 256
936 pop r11
937 pop r10
938 pop r9
939 pop r8
940 pop rdi
941 pop rsi
942 pop rdx
943 pop rcx
944 pop rax
945 iretq
946
947.size \name, . - \name
948.endm
949
950DEV_IRQ_ENTRY dev_irq_entry, dev_irq_handler_inner, dev_irq_handler_kernel
951DEV_IRQ_ENTRY nvme_irq_entry, nvme_irq_handler_inner, nvme_irq_handler_kernel
952
953.macro SIMPLE_IRQ_ENTRY name, handler
954.global \name
955.type \name, @function
956\name:
957 test qword ptr [rsp + 8], 3
958 jz .L_k_\name
959
960 swapgs
961 push rax
962 push rcx
963 push rdx
964 push rsi
965 push rdi
966 push r8
967 push r9
968 push r10
969 push r11
970 sub rsp, 256
971 movups [rsp], xmm0
972 movups [rsp + 16], xmm1
973 movups [rsp + 32], xmm2
974 movups [rsp + 48], xmm3
975 movups [rsp + 64], xmm4
976 movups [rsp + 80], xmm5
977 movups [rsp + 96], xmm6
978 movups [rsp + 112], xmm7
979 movups [rsp + 128], xmm8
980 movups [rsp + 144], xmm9
981 movups [rsp + 160], xmm10
982 movups [rsp + 176], xmm11
983 movups [rsp + 192], xmm12
984 movups [rsp + 208], xmm13
985 movups [rsp + 224], xmm14
986 movups [rsp + 240], xmm15
987 cld
988 call \handler
989 movups xmm0, [rsp]
990 movups xmm1, [rsp + 16]
991 movups xmm2, [rsp + 32]
992 movups xmm3, [rsp + 48]
993 movups xmm4, [rsp + 64]
994 movups xmm5, [rsp + 80]
995 movups xmm6, [rsp + 96]
996 movups xmm7, [rsp + 112]
997 movups xmm8, [rsp + 128]
998 movups xmm9, [rsp + 144]
999 movups xmm10, [rsp + 160]
1000 movups xmm11, [rsp + 176]
1001 movups xmm12, [rsp + 192]
1002 movups xmm13, [rsp + 208]
1003 movups xmm14, [rsp + 224]
1004 movups xmm15, [rsp + 240]
1005 add rsp, 256
1006 pop r11
1007 pop r10
1008 pop r9
1009 pop r8
1010 pop rdi
1011 pop rsi
1012 pop rdx
1013 pop rcx
1014 pop rax
1015 swapgs
1016 iretq
1017
1018.L_k_\name:
1019 push rax
1020 push rcx
1021 push rdx
1022 push rsi
1023 push rdi
1024 push r8
1025 push r9
1026 push r10
1027 push r11
1028 sub rsp, 256
1029 movups [rsp], xmm0
1030 movups [rsp + 16], xmm1
1031 movups [rsp + 32], xmm2
1032 movups [rsp + 48], xmm3
1033 movups [rsp + 64], xmm4
1034 movups [rsp + 80], xmm5
1035 movups [rsp + 96], xmm6
1036 movups [rsp + 112], xmm7
1037 movups [rsp + 128], xmm8
1038 movups [rsp + 144], xmm9
1039 movups [rsp + 160], xmm10
1040 movups [rsp + 176], xmm11
1041 movups [rsp + 192], xmm12
1042 movups [rsp + 208], xmm13
1043 movups [rsp + 224], xmm14
1044 movups [rsp + 240], xmm15
1045 cld
1046 call \handler
1047 movups xmm0, [rsp]
1048 movups xmm1, [rsp + 16]
1049 movups xmm2, [rsp + 32]
1050 movups xmm3, [rsp + 48]
1051 movups xmm4, [rsp + 64]
1052 movups xmm5, [rsp + 80]
1053 movups xmm6, [rsp + 96]
1054 movups xmm7, [rsp + 112]
1055 movups xmm8, [rsp + 128]
1056 movups xmm9, [rsp + 144]
1057 movups xmm10, [rsp + 160]
1058 movups xmm11, [rsp + 176]
1059 movups xmm12, [rsp + 192]
1060 movups xmm13, [rsp + 208]
1061 movups xmm14, [rsp + 224]
1062 movups xmm15, [rsp + 240]
1063 add rsp, 256
1064 pop r11
1065 pop r10
1066 pop r9
1067 pop r8
1068 pop rdi
1069 pop rsi
1070 pop rdx
1071 pop rcx
1072 pop rax
1073 iretq
1074
1075.size \name, . - \name
1076.endm
1077
1078SIMPLE_IRQ_ENTRY com1_entry, com1_handler_inner
1079SIMPLE_IRQ_ENTRY kbd_entry, kbd_handler_inner
1080SIMPLE_IRQ_ENTRY iommu_fault_entry, iommu_fault_handler_inner
1081
1082.global pf_entry
1083.type pf_entry, @function
1084pf_entry:
1085 test qword ptr [rsp + 16], 3
1086 jz .pf_kernel
1087
1088 swapgs
1089
1090 push rax
1091 mov rax, [rsp + 24]
1092 xchg rax, [rsp + 40]
1093 mov [rsp + 24], rax
1094 pop rax
1095
1096 xchg [rsp], r15
1097 push r14
1098 push r13
1099 push r12
1100 push r11
1101 push r10
1102 push r9
1103 push r8
1104 push rbp
1105 push rdi
1106 push rsi
1107 push rdx
1108 push rcx
1109 push rbx
1110 push rax
1111
1112 mov rdi, rsp
1113 mov rsi, r15
1114 mov rdx, cr2
1115 call pf_user_handler
1116 ud2
1117
1118.pf_kernel:
1119 mov rdi, [rsp + 8]
1120 mov rsi, [rsp]
1121 mov rdx, cr2
1122 call pf_kernel_handler
1123 ud2
1124.size pf_entry, . - pf_entry
1125
1126.global gpf_entry
1127.type gpf_entry, @function
1128gpf_entry:
1129 test qword ptr [rsp + 16], 3
1130 jz .gpf_kernel
1131
1132 swapgs
1133
1134 push rax
1135 mov rax, [rsp + 24]
1136 xchg rax, [rsp + 40]
1137 mov [rsp + 24], rax
1138 pop rax
1139
1140 add rsp, 8
1141 push r15
1142 push r14
1143 push r13
1144 push r12
1145 push r11
1146 push r10
1147 push r9
1148 push r8
1149 push rbp
1150 push rdi
1151 push rsi
1152 push rdx
1153 push rcx
1154 push rbx
1155 push rax
1156
1157 mov rdi, rsp
1158 call gpf_user_handler
1159 ud2
1160
1161.gpf_kernel:
1162 mov rdi, [rsp + 8]
1163 mov rsi, [rsp]
1164 call gpf_kernel_handler
1165 ud2
1166.size gpf_entry, . - gpf_entry
1167
1168.macro ERRCODE_ENTRY name, id
1169.global \name
1170.type \name, @function
1171\name:
1172 test qword ptr [rsp + 16], 3
1173 jz .L_k_\name
1174 swapgs
1175 mov rdi, [rsp + 8]
1176 mov rsi, [rsp]
1177 mov rdx, \id
1178 call errcode_user_handler
1179 ud2
1180.L_k_\name:
1181 mov rdi, [rsp + 8]
1182 mov rsi, [rsp]
1183 mov rdx, \id
1184 call errcode_kernel_handler
1185 ud2
1186.size \name, . - \name
1187.endm
1188
1189ERRCODE_ENTRY errcode_entry_invalid_tss, 0
1190ERRCODE_ENTRY errcode_entry_seg_not_present, 1
1191ERRCODE_ENTRY errcode_entry_stack_segment, 2
1192ERRCODE_ENTRY errcode_entry_alignment_check, 3
1193ERRCODE_ENTRY errcode_entry_security_exception, 4
1194"#,
1195 fpu_ptr = const core::mem::offset_of!(super::syscall::PerCpu, fpu_state_ptr),
1196 xsave_lo = const core::mem::offset_of!(super::syscall::PerCpu, xsave_mask),
1197 xsave_hi = const core::mem::offset_of!(super::syscall::PerCpu, xsave_mask) + 4,
1198);