ref: c220d0acc67b3f5e0e8af2bbe41fa8e367e6aa60
dir: /sys/src/9/pc64/fpu.c/
#include "u.h" #include "../port/lib.h" #include "mem.h" #include "dat.h" #include "fns.h" #include "ureg.h" #include "io.h" enum { CR4Osfxsr = 1 << 9, CR4Oxmmex = 1 << 10, CR4Oxsave = 1 << 18, }; /* * SIMD Floating Point. * Assembler support to get at the individual instructions * is in l.s. */ extern void _clts(void); extern void _fldcw(u16int); extern void _fnclex(void); extern void _fninit(void); extern void _fxrstor(void*); extern void _fxsave(void*); extern void _xrstor(void*); extern void _xsave(void*); extern void _xsaveopt(void*); extern void _fwait(void); extern void _ldmxcsr(u32int); extern void _stts(void); static void mathemu(Ureg *ureg, void*); static void fpssesave(FPsave *s) { _fxsave(s); _stts(); } static void fpsserestore(FPsave *s) { _clts(); _fxrstor(s); } static void fpxsave(FPsave *s) { _xsave(s); _stts(); } static void fpxrestore(FPsave *s) { _clts(); _xrstor(s); } static void fpxsaves(FPsave *s) { _xsaveopt(s); _stts(); } static void fpxrestores(FPsave *s) { _clts(); _xrstor(s); } static void fpxsaveopt(FPsave *s) { _xsaveopt(s); _stts(); } /* * Turn the FPU on and initialise it for use. * Set the precision and mask the exceptions * we don't care about from the generic Mach value. */ void fpinit(void) { _clts(); _fninit(); _fwait(); _fldcw(0x0232); _ldmxcsr(0x1900); } static char* mathmsg[] = { nil, /* handled below */ "denormalized operand", "division by zero", "numeric overflow", "numeric underflow", "precision loss", }; static void mathnote(ulong status, uintptr pc, int kernel) { char *msg, note[ERRMAX]; int i; /* * Some attention should probably be paid here to the * exception masks and error summary. */ msg = "unknown exception"; for(i = 1; i <= 5; i++){ if(!((1<<i) & status)) continue; msg = mathmsg[i]; break; } if(status & 0x01){ if(status & 0x40){ if(status & 0x200) msg = "stack overflow"; else msg = "stack underflow"; }else msg = "invalid operation"; } snprint(note, sizeof note, "sys: fp: %s fppc=%#p status=0x%lux", msg, pc, status); if(kernel) panic("%s", note); postnote(up, 1, note, NDebug); } /* * math coprocessor error */ static void matherror(Ureg *ureg, void*) { if(!userureg(ureg)){ if(up == nil) mathnote(m->fpsave->fsw, m->fpsave->rip, 1); else mathnote(up->kfpsave->fsw, up->kfpsave->rip, 1); return; } if(up->fpstate != FPinactive){ _clts(); fpsave(up->fpsave); up->fpstate = FPinactive; } mathnote(up->fpsave->fsw, up->fpsave->rip, 0); } /* * SIMD error */ static void simderror(Ureg *ureg, void*) { if(!userureg(ureg)){ if(up == nil) mathnote(m->fpsave->mxcsr & 0x3f, ureg->pc, 1); else mathnote(up->kfpsave->mxcsr & 0x3f, ureg->pc, 1); return; } if(up->fpstate != FPinactive){ _clts(); fpsave(up->fpsave); up->fpstate = FPinactive; } mathnote(up->fpsave->mxcsr & 0x3f, ureg->pc, 0); } /* * math coprocessor segment overrun */ static void mathover(Ureg *ureg, void*) { if(!userureg(ureg)) panic("math overrun"); pexit("math overrun", 0); } void mathinit(void) { trapenable(VectorCERR, matherror, 0, "matherror"); if(m->cpuidfamily == 3) intrenable(IrqIRQ13, matherror, 0, BUSUNKNOWN, "matherror"); trapenable(VectorCNA, mathemu, 0, "mathemu"); trapenable(VectorCSO, mathover, 0, "mathover"); trapenable(VectorSIMD, simderror, 0, "simderror"); } /* * fpuinit(), called from cpuidentify() for each cpu. */ void fpuinit(void) { u64int cr4; ulong regs[4]; m->xcr0 = 0; cr4 = getcr4() | CR4Osfxsr|CR4Oxmmex; if((m->cpuidcx & (Xsave|Avx)) == (Xsave|Avx) && getconf("*noavx") == nil){ cr4 |= CR4Oxsave; putcr4(cr4); m->xcr0 = 7; /* x87, sse, avx */ putxcr0(m->xcr0); cpuid(0xd, 1, regs); if(regs[0] & Xsaves){ fpsave = fpxsaves; fprestore = fpxrestores; } else { if(regs[0] & Xsaveopt) fpsave = fpxsaveopt; else fpsave = fpxsave; fprestore = fpxrestore; } } else { cr4 &= ~CR4Oxsave; putcr4(cr4); fpsave = fpssesave; fprestore = fpsserestore; } m->fpsave = nil; m->fpstate = FPinit; _stts(); } static FPsave* fpalloc(void) { FPsave *save; while((save = mallocalign(sizeof(FPsave), FPalign, 0, 0)) == nil){ spllo(); resrcwait("no memory for FPsave"); splhi(); } return save; } static void fpfree(FPsave *save) { free(save); } void fpuprocsetup(Proc *p) { p->fpstate = FPinit; } void fpuprocfork(Proc *p) { int s; s = splhi(); switch(up->fpstate & ~FPillegal){ case FPprotected: _clts(); /* wet floor */ case FPactive: fpsave(up->fpsave); up->fpstate = FPinactive; /* wet floor */ case FPinactive: if(p->fpsave == nil) p->fpsave = fpalloc(); memmove(p->fpsave, up->fpsave, sizeof(FPsave)); p->fpstate = FPinactive; } splx(s); } void fpuprocsave(Proc *p) { if(p->state == Moribund){ if(p->fpstate == FPactive || p->kfpstate == FPactive){ _fnclex(); _stts(); } fpfree(p->fpsave); fpfree(p->kfpsave); p->fpsave = p->kfpsave = nil; p->fpstate = p->kfpstate = FPinit; return; } if(p->kfpstate == FPactive){ fpsave(p->kfpsave); p->kfpstate = FPinactive; return; } if(p->fpstate == FPprotected) _clts(); else if(p->fpstate != FPactive) return; fpsave(p->fpsave); p->fpstate = FPinactive; } void fpuprocrestore(Proc*) { /* * when the scheduler switches, * we can discard its fp state. */ switch(m->fpstate){ case FPactive: _fnclex(); _stts(); /* wet floor */ case FPinactive: fpfree(m->fpsave); m->fpsave = nil; m->fpstate = FPinit; } } /* * Protect or save FPU state and setup new state * (lazily in the case of user process) for the kernel. * All syscalls, traps and interrupts (except mathemu()!) * are handled between fpukenter() and fpukexit(), * so they can use floating point and vector instructions. */ FPsave* fpukenter(Ureg *) { if(up == nil){ switch(m->fpstate){ case FPactive: fpsave(m->fpsave); /* wet floor */ case FPinactive: m->fpstate = FPinit; return m->fpsave; } return nil; } switch(up->fpstate){ case FPactive: up->fpstate = FPprotected; _stts(); /* wet floor */ case FPprotected: return nil; } switch(up->kfpstate){ case FPactive: fpsave(up->kfpsave); /* wet floor */ case FPinactive: up->kfpstate = FPinit; return up->kfpsave; } return nil; } void fpukexit(Ureg *ureg, FPsave *save) { if(up == nil){ switch(m->fpstate){ case FPactive: _fnclex(); _stts(); /* wet floor */ case FPinactive: fpfree(m->fpsave); m->fpstate = FPinit; } m->fpsave = save; if(save != nil) m->fpstate = FPinactive; return; } if(up->fpstate == FPprotected){ if(userureg(ureg)){ up->fpstate = FPactive; _clts(); } return; } switch(up->kfpstate){ case FPactive: _fnclex(); _stts(); /* wet floor */ case FPinactive: fpfree(up->kfpsave); up->kfpstate = FPinit; } up->kfpsave = save; if(save != nil) up->kfpstate = FPinactive; } /* * Before restoring the state, check for any pending * exceptions, there's no way to restore the state without * generating an unmasked exception. * More attention should probably be paid here to the * exception masks and error summary. */ static int fpcheck(FPsave *save, int kernel) { ulong status, control; status = save->fsw; control = save->fcw; if((status & ~control) & 0x07F){ mathnote(status, save->rip, kernel); return 1; } return 0; } /* * math coprocessor emulation fault */ static void mathemu(Ureg *ureg, void*) { if(!userureg(ureg)){ if(up == nil){ switch(m->fpstate){ case FPinit: m->fpsave = fpalloc(); m->fpstate = FPactive; fpinit(); break; case FPinactive: fpcheck(m->fpsave, 1); fprestore(m->fpsave); m->fpstate = FPactive; break; default: panic("floating point error in irq"); } return; } if(up->fpstate == FPprotected){ _clts(); fpsave(up->fpsave); up->fpstate = FPinactive; } switch(up->kfpstate){ case FPinit: up->kfpsave = fpalloc(); up->kfpstate = FPactive; fpinit(); break; case FPinactive: fpcheck(up->kfpsave, 1); fprestore(up->kfpsave); up->kfpstate = FPactive; break; default: panic("floating point error in trap"); } return; } if(up->fpstate & FPillegal){ postnote(up, 1, "sys: floating point in note handler", NDebug); return; } switch(up->fpstate){ case FPinit: if(up->fpsave == nil) up->fpsave = fpalloc(); up->fpstate = FPactive; fpinit(); break; case FPinactive: if(fpcheck(up->fpsave, 0)) break; fprestore(up->fpsave); up->fpstate = FPactive; break; case FPprotected: up->fpstate = FPactive; _clts(); break; case FPactive: postnote(up, 1, "sys: floating point error", NDebug); break; } }