diff -Nru /sys/src/9/bcm/arch.c /sys/src/9/bcm/arch.c --- /sys/src/9/bcm/arch.c Tue Mar 15 23:01:37 2016 +++ /sys/src/9/bcm/arch.c Fri Mar 4 00:00:00 2016 @@ -156,6 +156,19 @@ // TODO: save and restore VFPv3 FP state once 5[cal] know the new registers. fpuprocsave(p); + /* + * Prevent the following scenario: + * pX sleeps on cpuA, leaving its page tables in mmul1 + * pX wakes up on cpuB, and exits, freeing its page tables + * pY on cpuB allocates a freed page table page and overwrites with data + * cpuA takes an interrupt, and is now running with bad page tables + * In theory this shouldn't hurt because only user address space tables + * are affected, and mmuswitch will clear mmul1 before a user process is + * dispatched. But empirically it correlates with weird problems, eg + * resetting of the core clock at 0x4000001C which confuses local timers. + */ + if(conf.nmach > 1) + mmuswitch(nil); } void diff -Nru /sys/src/9/bcm/archbcm.c /sys/src/9/bcm/archbcm.c --- /sys/src/9/bcm/archbcm.c Tue Mar 15 23:01:38 2016 +++ /sys/src/9/bcm/archbcm.c Fri Mar 4 00:00:00 2016 @@ -28,7 +28,7 @@ enum { Wdogfreq = 65536, - Wdogtime = 5, /* seconds, ≤ 15 */ + Wdogtime = 10, /* seconds, ≤ 15 */ }; /* @@ -62,7 +62,7 @@ ; } -static void +void wdogfeed(void) { u32int *r; diff -Nru /sys/src/9/bcm/archbcm2.c /sys/src/9/bcm/archbcm2.c --- /sys/src/9/bcm/archbcm2.c Tue Mar 15 23:01:38 2016 +++ /sys/src/9/bcm/archbcm2.c Fri Mar 4 00:00:00 2016 @@ -21,7 +21,7 @@ #define ARMLOCAL (VIRTIO+IOSIZE) Soc soc = { - .dramsize = 1024*MiB, + .dramsize = 0x3F000000, /* was 1024*MiB, but overlaps with physio */ .physio = 0x3F000000, .busdram = 0xC0000000, .busio = 0x7E000000, @@ -32,7 +32,7 @@ enum { Wdogfreq = 65536, - Wdogtime = 5, /* seconds, ≤ 15 */ + Wdogtime = 10, /* seconds, ≤ 15 */ }; /* @@ -86,7 +86,7 @@ ; } -static void +void wdogfeed(void) { u32int *r; @@ -109,11 +109,25 @@ char * cputype2name(char *buf, int size) { - ulong r; + u32int r; + uint part; + char *p; r = cpidget(); /* main id register */ assert((r >> 24) == 'A'); - seprint(buf, buf + size, "Cortex-A7 r%ldp%ld", + part = (r >> 4) & MASK(12); + switch(part){ + case 0xc07: + p = seprint(buf, buf + size, "Cortex-A7"); + break; + case 0xd03: + p = seprint(buf, buf + size, "Cortex-A53"); + break; + default: + p = seprint(buf, buf + size, "Unknown-%#x", part); + break; + } + seprint(p, buf + size, " r%ldp%ld", (r >> 20) & MASK(4), r & MASK(4)); return buf; } @@ -162,6 +176,24 @@ return 0; } +void +mboxclear(uint cpu) +{ + Mboxes *mb; + + mb = (Mboxes*)(ARMLOCAL + Mboxregs); + mb->clr[cpu].mbox1 = 1; +} + +void +wakecpu(uint cpu) +{ + Mboxes *mb; + + mb = (Mboxes*)(ARMLOCAL + Mboxregs); + mb->set[cpu].mbox1 = 1; +} + int startcpus(uint ncpu) { @@ -215,11 +247,11 @@ up = nil; machinit(); - mmuinit1(m->mmul1); mb = (Mboxes*)(ARMLOCAL + Mboxregs); mb->clr[cpu].doorbell = 1; trapinit(); clockinit(); + mmuinit1(); timersinit(); cpuidprint(); archreset(); diff -Nru /sys/src/9/bcm/arm.h /sys/src/9/bcm/arm.h --- /sys/src/9/bcm/arm.h Tue Mar 15 23:01:38 2016 +++ /sys/src/9/bcm/arm.h Thu Mar 3 00:00:00 2016 @@ -12,6 +12,7 @@ #define PsrMsvc 0x00000013 /* `protected mode for OS' */ #define PsrMmon 0x00000016 /* `secure monitor' (trustzone hyper) */ #define PsrMabt 0x00000017 +#define PsrMhyp 0x0000001A #define PsrMund 0x0000001B #define PsrMsys 0x0000001F /* `privileged user mode for OS' (trustzone) */ #define PsrMask 0x0000001F diff -Nru /sys/src/9/bcm/arm.s /sys/src/9/bcm/arm.s --- /sys/src/9/bcm/arm.s Tue Mar 15 23:01:38 2016 +++ /sys/src/9/bcm/arm.s Sat Mar 5 00:00:00 2016 @@ -23,12 +23,14 @@ MOVW $0, R0; \ MCR CpSC, 0, R0, C(CpCACHE), C(CpCACHEwb), CpCACHEwait -#define BARRIERS ISB; DSB +#define BARRIERS DSB; ISB #define MCRR(coproc, op, rd, rn, crm) \ WORD $(0xec400000|(rn)<<16|(rd)<<12|(coproc)<<8|(op)<<4|(crm)) #define MRRC(coproc, op, rd, rn, crm) \ WORD $(0xec500000|(rn)<<16|(rd)<<12|(coproc)<<8|(op)<<4|(crm)) +#define MSR(R, rn, m, m1) \ + WORD $(0xe120f200|(R)<<22|(m1)<<16|(m)<<8|(rn)) #define LDREX(fp,t) WORD $(0xe<<28|0x01900f9f | (fp)<<16 | (t)<<12) /* `The order of operands is from left to right in dataflow order' - asm man */ diff -Nru /sys/src/9/bcm/armv6.s /sys/src/9/bcm/armv6.s --- /sys/src/9/bcm/armv6.s Tue Mar 15 23:01:38 2016 +++ /sys/src/9/bcm/armv6.s Fri Mar 4 00:00:00 2016 @@ -10,6 +10,12 @@ TEXT armstart(SB), 1, $-4 /* + * SVC mode, interrupts disabled + */ + MOVW $(PsrDirq|PsrDfiq|PsrMsvc), R1 + MOVW R1, CPSR + + /* * disable the mmu and L1 caches * invalidate caches and tlb */ @@ -51,7 +57,7 @@ * enable caches, mmu, and high vectors */ MRC CpSC, 0, R0, C(CpCONTROL), C(0), CpMainctl - ORR $(CpChv|CpCdcache|CpCicache|CpCmmu), R0 + ORR $(CpChv|CpCdcache|CpCicache|CpCpredict|CpCmmu), R0 MCR CpSC, 0, R0, C(CpCONTROL), C(0), CpMainctl ISB @@ -98,12 +104,6 @@ MRC CpSC, 0, R0, C(CpSPM), C(CpSPMperf), CpSPMcyc RET -TEXT tmrget(SB), 1, $-4 /* local generic timer physical counter value */ - MOVW $0, R1 /* not in armv6 */ - MOVW R1, 0(R0) - MOVW R1, 4(R0) - RET - TEXT splhi(SB), 1, $-4 MOVW $(MACHADDR+4), R2 /* save caller pc in Mach */ MOVW R14, 0(R2) @@ -201,6 +201,7 @@ MOVW $0, R0 MCR CpSC, 0, R0, C(CpTLB), C(CpTLBinvu), CpTLBinv BARRIERS + MCR CpSC, 0, R0, C(CpCACHE), C(CpCACHEinvi), CpCACHEflushbtc RET /* @@ -253,7 +254,9 @@ * drain write buffer * writeback data cache range [va, va+n) */ +TEXT cachedwbtlb(SB), 1, $-4 TEXT cachedwbse(SB), 1, $-4 + MOVW R0, R1 /* DSB clears R0 */ DSB MOVW n+4(FP), R2 @@ -303,4 +306,19 @@ TEXT cacheiinv(SB), 1, $-4 MOVW $0, R0 MCR CpSC, 0, R0, C(CpCACHE), C(CpCACHEinvi), CpCACHEall + RET + +/* + * invalidate range of instruction cache + */ +TEXT cacheiinvse(SB), 1, $-4 + MOVW R0, R1 /* DSB clears R0 */ + DSB + MOVW n+4(FP), R2 + ADD R1, R2 + SUB $1, R2 + MCRR(CpSC, 0, 2, 1, CpCACHERANGEinvi) + MCR CpSC, 0, R0, C(CpCACHE), C(CpCACHEinvi), CpCACHEflushbtc + DSB + ISB RET diff -Nru /sys/src/9/bcm/armv7.s /sys/src/9/bcm/armv7.s --- /sys/src/9/bcm/armv7.s Tue Mar 15 23:01:39 2016 +++ /sys/src/9/bcm/armv7.s Sun Mar 6 00:00:00 2016 @@ -5,14 +5,21 @@ #include "arm.s" -#define CACHELINESZ 64 +#define CACHELINESZ 64 +#define ICACHELINESZ 32 +#undef DSB +#undef DMB +#undef ISB +#define DSB WORD $0xf57ff04f /* data synch. barrier; last f = SY */ #define DMB WORD $0xf57ff05f /* data mem. barrier; last f = SY */ +#define ISB WORD $0xf57ff06f /* instr. sync. barrier; last f = SY */ #define WFI WORD $0xe320f003 /* wait for interrupt */ #define WFI_EQ WORD $0x0320f003 /* wait for interrupt if eq */ +#define ERET WORD $0xe160006e /* exception return from HYP */ /* tas/cas strex debugging limits; started at 10000 */ -#define MAXSC 100000 +#define MAXSC 1000000 TEXT armstart(SB), 1, $-4 @@ -23,16 +30,19 @@ BNE reset /* + * go to SVC mode, interrupts disabled + */ + BL svcmode(SB) + + /* * disable the mmu and caches - * invalidate tlb */ MRC CpSC, 0, R1, C(CpCONTROL), C(0), CpMainctl BIC $(CpCdcache|CpCicache|CpCmmu), R1 ORR $(CpCsbo|CpCsw), R1 BIC $CpCsbz, R1 MCR CpSC, 0, R1, C(CpCONTROL), C(0), CpMainctl - MCR CpSC, 0, R0, C(CpTLB), C(CpTLBinvu), CpTLBinv - ISB + BARRIERS /* * clear mach and page tables @@ -46,12 +56,23 @@ BNE _ramZ /* + * turn SMP on + * invalidate tlb + */ + MRC CpSC, 0, R1, C(CpCONTROL), C(0), CpAuxctl + ORR $CpACsmp, R1 /* turn SMP on */ + MCR CpSC, 0, R1, C(CpCONTROL), C(0), CpAuxctl + BARRIERS + MCR CpSC, 0, R0, C(CpTLB), C(CpTLBinvu), CpTLBinv + BARRIERS + + /* * start stack at top of mach (physical addr) * set up page tables for kernel */ MOVW $PADDR(MACHADDR+MACHSIZE-4), R13 MOVW $PADDR(L1), R0 - BL ,mmuinit(SB) + BL mmuinit(SB) /* * set up domain access control and page table base @@ -59,7 +80,7 @@ MOVW $Client, R1 MCR CpSC, 0, R1, C(CpDAC), C(0) MOVW $PADDR(L1), R1 - ORR $(CpTTBs/*|CpTTBowba|CpTTBiwba*/), R1 + ORR $(CpTTBs|CpTTBowba|CpTTBiwba), R1 MCR CpSC, 0, R1, C(CpTTB), C(0) MCR CpSC, 0, R1, C(CpTTB), C(0), CpTTB1 /* cortex has two */ @@ -74,10 +95,6 @@ /* * enable caches, mmu, and high vectors */ - MRC CpSC, 0, R1, C(CpCONTROL), C(0), CpAuxctl - ORR $CpACsmp, R1 /* turn SMP on */ - MCR CpSC, 0, R1, C(CpCONTROL), C(0), CpAuxctl - BARRIERS MRC CpSC, 0, R0, C(CpCONTROL), C(0), CpMainctl ORR $(CpChv|CpCdcache|CpCicache|CpCmmu), R0 @@ -126,20 +143,28 @@ /* * SVC mode, interrupts disabled */ - MOVW $(PsrDirq|PsrDfiq|PsrMsvc), R1 - MOVW R1, CPSR + BL svcmode(SB) /* * disable the mmu and caches - * invalidate tlb */ MRC CpSC, 0, R1, C(CpCONTROL), C(0), CpMainctl BIC $(CpCdcache|CpCicache|CpCmmu), R1 ORR $(CpCsbo|CpCsw), R1 BIC $CpCsbz, R1 MCR CpSC, 0, R1, C(CpCONTROL), C(0), CpMainctl + BARRIERS + + /* + * turn SMP on + * invalidate tlb + */ + MRC CpSC, 0, R1, C(CpCONTROL), C(0), CpAuxctl + ORR $CpACsmp, R1 /* turn SMP on */ + MCR CpSC, 0, R1, C(CpCONTROL), C(0), CpAuxctl + BARRIERS MCR CpSC, 0, R0, C(CpTLB), C(CpTLBinvu), CpTLBinv - ISB + BARRIERS /* * find Mach for this cpu @@ -151,22 +176,13 @@ ADD R2, R0 /* R0 = &machaddr[cpuid] */ MOVW (R0), R0 /* R0 = machaddr[cpuid] */ CMP $0, R0 - MOVW.EQ $MACHADDR, R0 /* paranoia: use MACHADDR if 0 */ - SUB $KZERO, R0 /* phys addr */ - MOVW R0, R(MACH) /* m = PADDR(machaddr[cpuid]) */ + BEQ 0(PC) /* must not be zero */ + SUB $KZERO, R0, R(MACH) /* m = PADDR(machaddr[cpuid]) */ /* * start stack at top of local Mach */ - MOVW R(MACH), R13 - ADD $(MACHSIZE-4), R13 - - /* - * set up page tables for kernel - */ - MOVW 12(R(MACH)), R0 /* m->mmul1 */ - SUB $KZERO, R0 /* phys addr */ - BL ,mmuinit(SB) + ADD $(MACHSIZE-4), R(MACH), R13 /* * set up domain access control and page table base @@ -175,7 +191,7 @@ MCR CpSC, 0, R1, C(CpDAC), C(0) MOVW 12(R(MACH)), R1 /* m->mmul1 */ SUB $KZERO, R1 /* phys addr */ - ORR $(CpTTBs/*|CpTTBowba|CpTTBiwba*/), R1 + ORR $(CpTTBs|CpTTBowba|CpTTBiwba), R1 MCR CpSC, 0, R1, C(CpTTB), C(0) MCR CpSC, 0, R1, C(CpTTB), C(0), CpTTB1 /* cortex has two */ @@ -189,11 +205,6 @@ /* * enable caches, mmu, and high vectors */ - MRC CpSC, 0, R1, C(CpCONTROL), C(0), CpAuxctl - ORR $CpACsmp, R1 /* turn SMP on */ - MCR CpSC, 0, R1, C(CpCONTROL), C(0), CpAuxctl - BARRIERS - MRC CpSC, 0, R0, C(CpCONTROL), C(0), CpMainctl ORR $(CpChv|CpCdcache|CpCicache|CpCmmu), R0 MCR CpSC, 0, R0, C(CpCONTROL), C(0), CpMainctl @@ -221,10 +232,28 @@ * call cpustart and loop forever if it returns */ MRC CpSC, 0, R0, C(CpID), C(CpIDidct), CpIDmpid - AND $(MAXMACH-1), R0 /* mask out non-cpu-id bits */ + AND $(MAXMACH-1), R0 /* mask out non-cpu-id bits */ BL ,cpustart(SB) B ,0(PC) +/* + * get into SVC mode with interrupts disabled + * raspberry pi firmware since 29 Sept 2015 starts in HYP mode + */ +TEXT svcmode(SB), 1, $-4 + MOVW CPSR, R1 + AND $PsrMask, R1 + MOVW $PsrMhyp, R2 + CMP R2, R1 + MOVW $(PsrDirq|PsrDfiq|PsrMsvc), R1 + BNE nothyp + MSR(1, 1, 1, 0xe) /* MOVW R1, SPSR_HYP */ + MSR(0, 14, 1, 0xe) /* MOVW R14, ELR_HYP */ + ERET +nothyp: + MOVW R1, CPSR + RET + TEXT cpidget(SB), 1, $-4 /* main ID */ MRC CpSC, 0, R0, C(CpID), C(0), CpIDid RET @@ -249,14 +278,8 @@ MRC CpSC, 0, R0, C(CpCLD), C(CpCLDcyc), 0 RET -TEXT tmrget(SB), 1, $-4 /* local generic timer physical counter value */ - MRRC(CpSC, 0, 1, 2, CpTIMER) - MOVM.IA [R1-R2], (R0) - RET - TEXT splhi(SB), 1, $-4 - MOVW $(MACHADDR+4), R2 /* save caller pc in Mach */ - MOVW R14, 0(R2) + MOVW R14, 4(R(MACH)) /* save caller pc in m->splpc */ MOVW CPSR, R0 /* turn off irqs (but not fiqs) */ ORR $(PsrDirq), R0, R1 @@ -264,8 +287,7 @@ RET TEXT splfhi(SB), 1, $-4 - MOVW $(MACHADDR+4), R2 /* save caller pc in Mach */ - MOVW R14, 0(R2) + MOVW R14, 4(R(MACH)) /* save caller pc in m->splpc */ MOVW CPSR, R0 /* turn off irqs and fiqs */ ORR $(PsrDirq|PsrDfiq), R0, R1 @@ -280,13 +302,15 @@ TEXT spllo(SB), 1, $-4 MOVW CPSR, R0 /* turn on irqs and fiqs */ + MOVW $0, R1 + CMP.S R1, R(MACH) + MOVW.NE R1, 4(R(MACH)) /* clear m->splpc */ BIC $(PsrDirq|PsrDfiq), R0, R1 MOVW R1, CPSR RET TEXT splx(SB), 1, $-4 - MOVW $(MACHADDR+0x04), R2 /* save caller pc in Mach */ - MOVW R14, 0(R2) + MOVW R14, 4(R(MACH)) /* save caller pc in m->splpc */ MOVW R0, R1 /* reset interrupt level */ MOVW CPSR, R0 @@ -353,9 +377,7 @@ DSB MOVW nrdy(SB), R0 CMP $0, R0 -/*** with WFI, local timer interrupts can be lost and dispatching stops WFI_EQ -***/ DSB MOVW R3, CPSR /* splx */ @@ -370,6 +392,7 @@ * invalidate tlb */ TEXT mmuinvalidate(SB), 1, $-4 + DSB MOVW $0, R0 MCR CpSC, 0, R0, C(CpTLB), C(CpTLBinvu), CpTLBinv BARRIERS @@ -380,13 +403,14 @@ * invalidate tlb entry for virtual page address va, ASID 0 */ TEXT mmuinvalidateaddr(SB), 1, $-4 + DSB MCR CpSC, 0, R0, C(CpTLB), C(CpTLBinvu), CpTLBinvse BARRIERS RET /* * `single-element' cache operations. - * in arm arch v7, they operate on all cache levels, so separate + * in arm arch v7, if effective to PoC, they operate on all cache levels, so separate * l2 functions are unnecessary. */ @@ -408,6 +432,14 @@ CMP.S R0, R1 BGT _dwbse B _wait + +/* + * TLB on armv7 loads from cache, so no need for writeback + */ +TEXT cachedwbtlb(SB), $-4 + DSB + ISB + RET TEXT cachedwbinvse(SB), $-4 /* D writeback+invalidate SE */ MOVW R0, R2 diff -Nru /sys/src/9/bcm/cache.v7.s /sys/src/9/bcm/cache.v7.s --- /sys/src/9/bcm/cache.v7.s Tue Mar 15 23:01:39 2016 +++ /sys/src/9/bcm/cache.v7.s Fri Mar 4 00:00:00 2016 @@ -3,9 +3,29 @@ * shared by l.s and rebootcode.s */ +#define BPIALL MCR CpSC, 0, R0, C(CpCACHE), C(5), 6 /* branch predictor invalidate all */ + TEXT cacheiinv(SB), $-4 /* I invalidate */ + DSB MOVW $0, R0 MCR CpSC, 0, R0, C(CpCACHE), C(CpCACHEinvi), CpCACHEall /* ok on cortex */ + BPIALL /* redundant? */ + DSB + ISB + RET + +TEXT cacheiinvse(SB), $0 /* I invalidate SE */ + MOVW 4(FP), R1 + ADD R0, R1 + BIC $(ICACHELINESZ - 1), R0 + DSB +_iinvse: + MCR CpSC, 0, R0, C(CpCACHE), C(CpCACHEinvi), CpCACHEse + ADD $ICACHELINESZ, R0 + CMP.S R0, R1 + BGT _iinvse + BPIALL + DSB ISB RET diff -Nru /sys/src/9/bcm/clock.c /sys/src/9/bcm/clock.c --- /sys/src/9/bcm/clock.c Tue Mar 15 23:01:39 2016 +++ /sys/src/9/bcm/clock.c Fri Mar 4 00:00:00 2016 @@ -33,7 +33,7 @@ SystimerFreq = 1*Mhz, MaxPeriod = SystimerFreq / HZ, - MinPeriod = SystimerFreq / (100*HZ), + MinPeriod = 10, }; @@ -86,6 +86,8 @@ { Systimers *tn; + if(m->machno != 0) + panic("cpu%d: unexpected system timer interrupt", m->machno); tn = (Systimers*)SYSTIMERS; /* dismiss interrupt */ tn->cs = 1<<3; @@ -108,7 +110,10 @@ tm = (Armtimer*)ARMTIMER; tm->ctl = 0; - //wdogoff(); + if(cpuserver) + wdogfeed(); + else + wdogoff(); } void @@ -121,8 +126,8 @@ if(((cprdsc(0, CpID, CpIDfeat, 1) >> 16) & 0xF) != 0) { /* generic timer supported */ if(m->machno == 0){ - *(ulong*)(ARMLOCAL + Localctl) = 0; /* magic */ - *(ulong*)(ARMLOCAL + Prescaler) = 0x06aaaaab; /* magic for 1 Mhz */ + *(ulong*)(ARMLOCAL + Localctl) = 0; /* input clock is 19.2Mhz crystal */ + *(ulong*)(ARMLOCAL + Prescaler) = 0x06aaaaab; /* divide by (2^31/Prescaler) for 1Mhz */ } cpwrsc(0, CpTIMER, CpTIMERphys, CpTIMERphysctl, Imask); } @@ -168,7 +173,7 @@ cpwrsc(0, CpTIMER, CpTIMERphys, CpTIMERphysctl, Enable); }else{ tn = (Systimers*)SYSTIMERS; - tn->c3 = (ulong)(now + period); + tn->c3 = tn->clo + period; } } @@ -178,20 +183,16 @@ Systimers *tn; ulong lo, hi; uvlong now; - int s; if(hz) *hz = SystimerFreq; tn = (Systimers*)SYSTIMERS; - s = splhi(); do{ hi = tn->chi; lo = tn->clo; }while(tn->chi != hi); now = (uvlong)hi<<32 | lo; - m->fastclock = now; - splx(s); - return m->fastclock; + return now; } ulong @@ -224,7 +225,7 @@ { if(SystimerFreq != 1*Mhz) return fastticks2us(fastticks(nil)); - return fastticks(nil); + return ((Systimers*)SYSTIMERS)->clo; } void diff -Nru /sys/src/9/bcm/dma.c /sys/src/9/bcm/dma.c --- /sys/src/9/bcm/dma.c Tue Mar 15 23:01:41 2016 +++ /sys/src/9/bcm/dma.c Thu Jan 14 00:00:00 2016 @@ -25,7 +25,7 @@ enum { Nchan = 7, /* number of dma channels */ Regsize = 0x100, /* size of regs for each chan */ - Cbalign = 32, /* control block byte alignment */ + Cbalign = 64, /* control block byte alignment (allow for 64-byte cache on bcm2836) */ Dbg = 0, /* registers for each dma controller */ diff -Nru /sys/src/9/bcm/etherusb.c /sys/src/9/bcm/etherusb.c --- /sys/src/9/bcm/etherusb.c Tue Mar 15 23:01:42 2016 +++ /sys/src/9/bcm/etherusb.c Fri Mar 4 00:00:00 2016 @@ -409,6 +409,11 @@ } static void +etherusbshutdown(Ether*) +{ +} + +static void etherusbattach(Ether* edev) { Ctlr *ctlr; @@ -439,7 +444,7 @@ edev->arg = edev; /* TODO: promiscuous, multicast (for ipv6), shutdown (for reboot) */ // edev->promiscuous = etherusbpromiscuous; -// edev->shutdown = etherusbshutdown; + edev->shutdown = etherusbshutdown; edev->multicast = etherusbmulticast; return 0; diff -Nru /sys/src/9/bcm/fns.h /sys/src/9/bcm/fns.h --- /sys/src/9/bcm/fns.h Tue Mar 15 23:01:42 2016 +++ /sys/src/9/bcm/fns.h Fri Mar 4 00:00:00 2016 @@ -10,7 +10,9 @@ extern void cachedinvse(void*, int); extern void cachedwbse(void*, int); extern void cachedwbinvse(void*, int); +extern void cachedwbtlb(void*, int); extern void cacheiinv(void); +extern void cacheiinvse(void*, int); extern void cacheuwbinv(void); extern uintptr cankaddr(uintptr pa); extern int cas32(void*, u32int, u32int); @@ -67,7 +69,7 @@ extern void l2cacheuwbinv(void); extern void links(void); extern void mmuinit(void*); -extern void mmuinit1(void*); +extern void mmuinit1(void); extern void mmuinvalidate(void); extern void mmuinvalidateaddr(u32int); extern uintptr mmukmap(uintptr, uintptr, usize); @@ -97,6 +99,7 @@ extern void vectors(void); extern void vtable(void); extern void wdogoff(void); +extern void wdogfeed(void); /* * floating point emulation diff -Nru /sys/src/9/bcm/io.h /sys/src/9/bcm/io.h --- /sys/src/9/bcm/io.h Tue Mar 15 23:01:44 2016 +++ /sys/src/9/bcm/io.h Mon Feb 15 00:00:00 2016 @@ -18,6 +18,11 @@ IRQlocal = 96, IRQcntps = IRQlocal + 0, IRQcntpns = IRQlocal + 1, + IRQmbox0 = IRQlocal + 4, + IRQmbox1 = IRQlocal + 5, + IRQmbox2 = IRQlocal + 6, + IRQmbox3 = IRQlocal + 7, + IRQlocaltmr = IRQlocal + 11, IRQfiq = IRQusb, /* only one source can be FIQ */ diff -Nru /sys/src/9/bcm/l.s /sys/src/9/bcm/l.s --- /sys/src/9/bcm/l.s Tue Mar 15 23:01:44 2016 +++ /sys/src/9/bcm/l.s Thu Mar 3 00:00:00 2016 @@ -20,12 +20,6 @@ MOVW $0, R0 /* - * SVC mode, interrupts disabled - */ - MOVW $(PsrDirq|PsrDfiq|PsrMsvc), R1 - MOVW R1, CPSR - - /* * start stack at top of mach (physical addr) */ MOVW $PADDR(MACHADDR+MACHSIZE-4), R13 diff -Nru /sys/src/9/bcm/main.c /sys/src/9/bcm/main.c --- /sys/src/9/bcm/main.c Tue Mar 15 23:01:45 2016 +++ /sys/src/9/bcm/main.c Sat Mar 5 00:00:00 2016 @@ -235,11 +235,7 @@ if (m->machno != 0) { /* synchronise with cpu 0 */ m->ticks = m0->ticks; - m->fastclock = m0->fastclock; - m->delayloop = m0->delayloop; } - - //machon(m->machno); } void @@ -303,7 +299,7 @@ m = (Mach*)MACHADDR; memset(edata, 0, end - edata); /* clear bss */ mach0init(); - mmuinit1((void*)L1); + m->mmul1 = (PTE*)L1; machon(0); optionsinit("/boot/boot boot"); @@ -344,6 +340,7 @@ swapinit(); userinit(); launchinit(getncpus()); + mmuinit1(); schedinit(); assert(0); /* shouldn't have returned */ @@ -590,15 +587,29 @@ active.exiting = 1; unlock(&active); - if(once) + if(once) { + delay(m->machno*100); /* stagger them */ iprint("cpu%d: exiting\n", m->machno); + } spllo(); - for(ms = 5*1000; ms > 0; ms -= TK2MS(2)){ + if (m->machno == 0) + ms = 5*1000; + else + ms = 2*1000; + for(; ms > 0; ms -= TK2MS(2)){ delay(TK2MS(2)); if(active.machs == 0 && consactive() == 0) break; } - delay(100*m->machno); + if(active.ispanic){ + if(!cpuserver) + for(;;) + ; + if(getconf("*debug")) + delay(5*60*1000); + else + delay(10000); + } } /* @@ -617,7 +628,6 @@ f = (void*)REBOOTADDR; intrcpushutdown(); cacheuwbinv(); - l2cacheuwbinv(); (*f)(0, 0, 0); for(;;){} } @@ -681,8 +691,7 @@ * should be the only processor running now */ - delay(5000); - print("active.machs = %x\n", active.machs); + delay(500); print("reboot entry %#lux code %#lux size %ld\n", PADDR(entry), PADDR(code), size); delay(100); diff -Nru /sys/src/9/bcm/mem.h /sys/src/9/bcm/mem.h --- /sys/src/9/bcm/mem.h Tue Mar 15 23:01:45 2016 +++ /sys/src/9/bcm/mem.h Thu Mar 3 00:00:00 2016 @@ -52,7 +52,7 @@ #define UZERO 0 /* user segment */ #define UTZERO (UZERO+BY2PG) /* user text start */ #define UTROUND(t) ROUNDUP((t), BY2PG) -#define USTKTOP 0x20000000 /* user segment end +1 */ +#define USTKTOP 0x40000000 /* user segment end +1 */ #define USTKSIZE (8*1024*1024) /* user stack size */ #define TSTKTOP (USTKTOP-USTKSIZE) /* sysexec temporary stack */ #define TSTKSIZ 256 diff -Nru /sys/src/9/bcm/mkfile /sys/src/9/bcm/mkfile --- /sys/src/9/bcm/mkfile Tue Mar 15 23:01:45 2016 +++ /sys/src/9/bcm/mkfile Sat Mar 5 00:00:00 2016 @@ -1,4 +1,4 @@ -CONF=pi +CONF=pi2 CONFLIST=pi picpu pifat pi2 pi2cpu EXTRACOPIES= @@ -59,7 +59,8 @@ $DEVS\ $PORT\ -# HFILES= +HFILES=\ + arm.h LIB=\ /$objtype/lib/libmemlayer.a\ @@ -104,14 +105,14 @@ arch.$O clock.$O fpiarm.$O main.$O mmu.$O screen.$O syscall.$O trap.$O: \ /$objtype/include/ureg.h -archbcm.$O devether.$0: etherif.h ../port/netif.h -archbcm.$O: ../port/flashif.h +archbcm.$O archbcm2.$O devether.$0 etherusb.$O: etherif.h ../port/netif.h fpi.$O fpiarm.$O fpimem.$O: ../port/fpi.h -l.$O lexception.$O lproc.$O mmu.$O: arm.s mem.h +l.$O lexception.$O lproc.$O armv6.$O armv7.$O: arm.s +armv7.$O: cache.v7.s main.$O: errstr.h init.h reboot.h -devmouse.$O mouse.$O screen.$O: screen.h -devusb.$O: ../port/usb.h -usbehci.$O usbohci.$O usbuhci.$O: ../port/usb.h usbehci.h uncached.h +mouse.$O screen.$O: screen.h +devusb.$O usbdwc.$O: ../port/usb.h +usbdwc.$O: dwcotg.h init.h:D: ../port/initcode.c init9.s $CC ../port/initcode.c @@ -125,7 +126,7 @@ reboot.h:D: rebootcode.s arm.s arm.h mem.h $AS rebootcode.s # -lc is only for memmove. -T arg is PADDR(REBOOTADDR) - $LD -l -s -T0x3400 -R4 -o reboot.out rebootcode.$O -lc + $LD -l -s -T0x1800 -R4 -o reboot.out rebootcode.$O -lc {echo 'uchar rebootcode[]={' xd -1x reboot.out | sed -e '1,2d' -e 's/^[0-9a-f]+ //' -e 's/ ([0-9a-f][0-9a-f])/0x\1,/g' diff -Nru /sys/src/9/bcm/mmu.c /sys/src/9/bcm/mmu.c --- /sys/src/9/bcm/mmu.c Tue Mar 15 23:01:45 2016 +++ /sys/src/9/bcm/mmu.c Sun Mar 6 00:00:00 2016 @@ -15,10 +15,11 @@ enum { L1lo = UZERO/MiB, /* L1X(UZERO)? */ L1hi = (USTKTOP+MiB-1)/MiB, /* L1X(USTKTOP+MiB-1)? */ + L2size = 256*sizeof(PTE), }; /* - * Set up initial PTEs for this cpu (called with mmu off) + * Set up initial PTEs for cpu0 (called with mmu off) */ void mmuinit(void *a) @@ -56,7 +57,7 @@ l1[L1X(va)] = pa|Dom0|L1AP(Krw)|Section; /* - * double map exception vectors at top of virtual memory + * double map exception vectors near top of virtual memory */ va = HVECTORS; l1[L1X(va)] = (uintptr)l2|Dom0|Coarse; @@ -64,21 +65,17 @@ } void -mmuinit1(void *a) +mmuinit1() { PTE *l1; - l1 = (PTE*)a; - m->mmul1 = l1; + l1 = m->mmul1; /* * undo identity map of first MB of ram */ l1[L1X(PHYSDRAM)] = 0; - cachedwbse(&l1[L1X(PHYSDRAM)], sizeof(PTE)); - - //cacheuwbinv(); - //l2cacheuwbinv(); + cachedwbtlb(&l1[L1X(PHYSDRAM)], sizeof(PTE)); mmuinvalidateaddr(PHYSDRAM); } @@ -92,10 +89,11 @@ l2 = &proc->mmul2; for(page = *l2; page != nil; page = page->next){ if(clear) - memset(UINT2PTR(page->va), 0, BY2PG); + memset(UINT2PTR(page->va), 0, L2size); l1[page->daddr] = Fault; l2 = &page->next; } + coherence(); *l2 = proc->mmul2cache; proc->mmul2cache = proc->mmul2; proc->mmul2 = nil; @@ -104,29 +102,24 @@ static void mmul1empty(void) { -#ifdef notdef -/* there's a bug in here */ PTE *l1; /* clean out any user mappings still in l1 */ - if(m->mmul1lo > L1lo){ + if(m->mmul1lo > 0){ if(m->mmul1lo == 1) m->mmul1[L1lo] = Fault; else memset(&m->mmul1[L1lo], 0, m->mmul1lo*sizeof(PTE)); - m->mmul1lo = L1lo; + m->mmul1lo = 0; } - if(m->mmul1hi < L1hi){ - l1 = &m->mmul1[m->mmul1hi]; - if((L1hi - m->mmul1hi) == 1) + if(m->mmul1hi > 0){ + l1 = &m->mmul1[L1hi - m->mmul1hi]; + if(m->mmul1hi == 1) *l1 = Fault; else - memset(l1, 0, (L1hi - m->mmul1hi)*sizeof(PTE)); - m->mmul1hi = L1hi; + memset(l1, 0, m->mmul1hi*sizeof(PTE)); + m->mmul1hi = 0; } -#else - memset(&m->mmul1[L1lo], 0, (L1hi - L1lo)*sizeof(PTE)); -#endif /* notdef */ } void @@ -136,17 +129,7 @@ PTE *l1; Page *page; - /* do kprocs get here and if so, do they need to? */ -/*** "This is plausible, but wrong" - Charles Forsyth 1 Mar 2015 - if(m->mmupid == proc->pid && !proc->newtlb) - return; -***/ - m->mmupid = proc->pid; - - /* write back dirty and invalidate l1 caches */ - cacheuwbinv(); - - if(proc->newtlb){ + if(proc != nil && proc->newtlb){ mmul2empty(proc, 1); proc->newtlb = 0; } @@ -155,19 +138,21 @@ /* move in new map */ l1 = m->mmul1; + if(proc != nil) for(page = proc->mmul2; page != nil; page = page->next){ x = page->daddr; l1[x] = PPN(page->pa)|Dom0|Coarse; - /* know here that L1lo < x < L1hi */ - if(x+1 - m->mmul1lo < m->mmul1hi - x) - m->mmul1lo = x+1; - else - m->mmul1hi = x; + if(x >= L1lo + m->mmul1lo && x < L1hi - m->mmul1hi){ + if(x+1 - L1lo < L1hi - x) + m->mmul1lo = x+1 - L1lo; + else + m->mmul1hi = L1hi - x; + } } /* make sure map is in memory */ /* could be smarter about how much? */ - cachedwbse(&l1[L1X(UZERO)], (L1hi - L1lo)*sizeof(PTE)); + cachedwbtlb(&l1[L1X(UZERO)], (L1hi - L1lo)*sizeof(PTE)); /* lose any possible stale tlb entries */ mmuinvalidate(); @@ -189,9 +174,6 @@ { Page *page, *next; - /* write back dirty and invalidate l1 caches */ - cacheuwbinv(); - mmul2empty(proc, 0); for(page = proc->mmul2cache; page != nil; page = next){ next = page->next; @@ -207,7 +189,7 @@ /* make sure map is in memory */ /* could be smarter about how much? */ - cachedwbse(&m->mmul1[L1X(UZERO)], (L1hi - L1lo)*sizeof(PTE)); + cachedwbtlb(&m->mmul1[L1X(UZERO)], (L1hi - L1lo)*sizeof(PTE)); /* lose any possible stale tlb entries */ mmuinvalidate(); @@ -216,39 +198,45 @@ void putmmu(uintptr va, uintptr pa, Page* page) { - int x; + int x, s; Page *pg; PTE *l1, *pte; + /* + * disable interrupts to prevent flushmmu (called from hzclock) + * from clearing page tables while we are setting them + */ + s = splhi(); x = L1X(va); l1 = &m->mmul1[x]; if(*l1 == Fault){ - /* wasteful - l2 pages only have 256 entries - fix */ + /* l2 pages only have 256 entries - wastes 3K per 1M of address space */ if(up->mmul2cache == nil){ - /* auxpg since we don't need much? memset if so */ + spllo(); pg = newpage(1, 0, 0); + splhi(); + /* if newpage slept, we might be on a different cpu */ + l1 = &m->mmul1[x]; pg->va = VA(kmap(pg)); - } - else{ + }else{ pg = up->mmul2cache; up->mmul2cache = pg->next; - memset(UINT2PTR(pg->va), 0, BY2PG); } pg->daddr = x; pg->next = up->mmul2; up->mmul2 = pg; - /* force l2 page to memory */ - cachedwbse((void *)pg->va, BY2PG); + /* force l2 page to memory (armv6) */ + cachedwbtlb((void *)pg->va, L2size); *l1 = PPN(pg->pa)|Dom0|Coarse; - cachedwbse(l1, sizeof *l1); + cachedwbtlb(l1, sizeof *l1); - if(x >= m->mmul1lo && x < m->mmul1hi){ - if(x+1 - m->mmul1lo < m->mmul1hi - x) - m->mmul1lo = x+1; + if(x >= L1lo + m->mmul1lo && x < L1hi - m->mmul1hi){ + if(x+1 - L1lo < L1hi - x) + m->mmul1lo = x+1 - L1lo; else - m->mmul1hi = x; + m->mmul1hi = L1hi - x; } } pte = UINT2PTR(KADDR(PPN(*l1))); @@ -266,23 +254,19 @@ else x |= L2AP(Uro); pte[L2X(va)] = PPN(pa)|x; - cachedwbse(&pte[L2X(va)], sizeof pte[0]); + cachedwbtlb(&pte[L2X(va)], sizeof(PTE)); /* clear out the current entry */ mmuinvalidateaddr(PPN(va)); - /* write back dirty entries - we need this because the pio() in - * fault.c is writing via a different virt addr and won't clean - * its changes out of the dcache. Page coloring doesn't work - * on this mmu because the virtual cache is set associative - * rather than direct mapped. - */ if(page->cachectl[m->machno] == PG_TXTFLUSH){ /* pio() sets PG_TXTFLUSH whenever a text pg has been written */ - cacheiinv(); + cachedwbse((void*)(page->pa|KZERO), BY2PG); + cacheiinvse((void*)page->va, BY2PG); page->cachectl[m->machno] = PG_NOFLUSH; } - checkmmu(va, PPN(pa)); + //checkmmu(va, PPN(pa)); + splx(s); } void* @@ -305,7 +289,6 @@ if((*pte & (Fine|Section|Coarse)) != Section) return nil; *pte &= ~L1ptedramattrs; - *pte |= L1sharable; mmuinvalidateaddr(va); cachedwbinvse(pte, 4); @@ -344,20 +327,31 @@ *pte++ = (pa+n)|Dom0|L1AP(Krw)|Section; mmuinvalidateaddr(va+n); } - cachedwbse(pte0, (uintptr)pte - (uintptr)pte0); + cachedwbtlb(pte0, (uintptr)pte - (uintptr)pte0); return va + o; } - void checkmmu(uintptr va, uintptr pa) { - USED(va); - USED(pa); + int x; + PTE *l1, *pte; + + x = L1X(va); + l1 = &m->mmul1[x]; + if(*l1 == Fault){ + iprint("checkmmu cpu%d va=%lux l1 %p=%ux\n", m->machno, va, l1, *l1); + return; + } + pte = KADDR(PPN(*l1)); + pte += L2X(va); + if(pa == ~0 || (pa != 0 && PPN(*pte) != pa)) + iprint("checkmmu va=%lux pa=%lux l1 %p=%ux pte %p=%ux\n", va, pa, l1, *l1, pte, *pte); } void kunmap(KMap *k) { - cachedwbinvse(k, BY2PG); + USED(k); + coherence(); } diff -Nru /sys/src/9/bcm/pi /sys/src/9/bcm/pi --- /sys/src/9/bcm/pi Tue Mar 15 23:01:45 2016 +++ /sys/src/9/bcm/pi Sat Mar 5 00:00:00 2016 @@ -19,10 +19,10 @@ kbin kbd latin1 uart -# fakertc + fakertc + ether netif sd usb - ether netif link archbcm diff -Nru /sys/src/9/bcm/pi2 /sys/src/9/bcm/pi2 --- /sys/src/9/bcm/pi2 Tue Mar 15 23:01:45 2016 +++ /sys/src/9/bcm/pi2 Sat Mar 5 00:00:00 2016 @@ -22,9 +22,9 @@ fakertc # rtc3231 i2c + ether netif sd usb - ether netif link archbcm2 diff -Nru /sys/src/9/bcm/pi2cpu /sys/src/9/bcm/pi2cpu --- /sys/src/9/bcm/pi2cpu Tue Mar 15 23:01:46 2016 +++ /sys/src/9/bcm/pi2cpu Sat Mar 5 00:00:00 2016 @@ -20,9 +20,9 @@ uart fakertc + ether netif sd usb - ether netif link archbcm2 diff -Nru /sys/src/9/bcm/picpu /sys/src/9/bcm/picpu --- /sys/src/9/bcm/picpu Tue Mar 15 23:01:46 2016 +++ /sys/src/9/bcm/picpu Sat Mar 5 00:00:00 2016 @@ -20,9 +20,9 @@ uart fakertc + ether netif sd usb - ether netif link archbcm diff -Nru /sys/src/9/bcm/rebootcode.s /sys/src/9/bcm/rebootcode.s --- /sys/src/9/bcm/rebootcode.s Tue Mar 15 23:01:46 2016 +++ /sys/src/9/bcm/rebootcode.s Fri Mar 4 00:00:00 2016 @@ -98,6 +98,8 @@ MOVW 12(R(MACH)), R2 /* m->mmul1 (virtual addr) */ MOVW $PTEDRAM, R1 /* PTE bits */ MOVW R1, (R2) + DSB + MCR CpSC, 0, R2, C(CpCACHE), C(CpCACHEwb), CpCACHEse /* invalidate stale TLBs again */ BARRIERS diff -Nru /sys/src/9/bcm/screen.c /sys/src/9/bcm/screen.c --- /sys/src/9/bcm/screen.c Tue Mar 15 23:01:46 2016 +++ /sys/src/9/bcm/screen.c Mon Jan 11 00:00:00 2016 @@ -293,12 +293,15 @@ static int screensize(void) { - char *p; + char *p, buf[32]; char *f[3]; int width, height, depth; p = getconf("vgasize"); - if(p == nil || getfields(p, f, nelem(f), 0, "x") != nelem(f) || + if(p == nil || memccpy(buf, p, '\0', sizeof buf) == nil) + return -1; + + if(getfields(buf, f, nelem(f), 0, "x") != nelem(f) || (width = atoi(f[0])) < 16 || (height = atoi(f[1])) <= 0 || (depth = atoi(f[2])) <= 0) diff -Nru /sys/src/9/bcm/taslock.c /sys/src/9/bcm/taslock.c --- /sys/src/9/bcm/taslock.c Tue Mar 15 23:01:47 2016 +++ /sys/src/9/bcm/taslock.c Fri Mar 4 00:00:00 2016 @@ -247,6 +247,7 @@ sr = l->sr; l->m = nil; + coherence(); l->key = 0; coherence(); m->ilockdepth--; diff -Nru /sys/src/9/bcm/trap.c /sys/src/9/bcm/trap.c --- /sys/src/9/bcm/trap.c Tue Mar 15 23:01:48 2016 +++ /sys/src/9/bcm/trap.c Sun Feb 28 00:00:00 2016 @@ -150,7 +150,7 @@ x = perfticks(); diff = x - m->perf.intrts; - m->perf.intrts = x; + m->perf.intrts = 0; m->perf.inintr += diff; if(up == nil && m->perf.inidle > diff) @@ -194,8 +194,14 @@ fiq(Ureg *ureg) { Vctl *v; + int inintr; - m->perf.intrts = perfticks(); + if(m->perf.intrts) + inintr = 1; + else{ + inintr = 0; + m->perf.intrts = perfticks(); + } v = vfiq; if(v == nil) panic("cpu%d: unexpected item in bagging area", m->machno); @@ -204,7 +210,8 @@ coherence(); v->f(ureg, v->a); coherence(); - intrtime(); + if(!inintr) + intrtime(); } void @@ -221,8 +228,11 @@ v->irq = irq; v->cpu = 0; if(irq >= IRQlocal){ - enable = (u32int*)(LOCALREGS + Localtimerint) + m->machno; v->reg = (u32int*)(LOCALREGS + Localintpending) + m->machno; + if(irq >= IRQmbox0) + enable = (u32int*)(LOCALREGS + Localmboxint) + m->machno; + else + enable = (u32int*)(LOCALREGS + Localtimerint) + m->machno; v->mask = 1 << (irq - IRQlocal); v->cpu = m->machno; }else if(irq >= IRQbasic){ @@ -245,7 +255,10 @@ }else{ v->next = vctl; vctl = v; - if(irq >= IRQlocal) + if(irq >= IRQmbox0){ + if(irq <= IRQmbox3) + *enable |= 1 << (irq - IRQmbox0); + }else if(irq >= IRQlocal) *enable |= 1 << (irq - IRQlocal); else *enable = v->mask; diff -Nru /sys/src/9/bcm/uartmini.c /sys/src/9/bcm/uartmini.c --- /sys/src/9/bcm/uartmini.c Tue Mar 15 23:01:48 2016 +++ /sys/src/9/bcm/uartmini.c Sun Mar 6 00:00:00 2016 @@ -445,12 +445,13 @@ if(p != nil) okled = strtol(p, 0, 0); else - okled = OkLed; + return; p = getconf("bcm2709.disk_led_active_low"); if(p == nil) p = getconf("bcm2708.disk_led_active_low"); polarity = (p == nil || *p == '1'); gpiosel(okled, Output); } - gpioout(okled, on^polarity); + if(okled != 0) + gpioout(okled, on^polarity); } diff -Nru /sys/src/9/bcm/usbdwc.c /sys/src/9/bcm/usbdwc.c --- /sys/src/9/bcm/usbdwc.c Tue Mar 15 23:01:48 2016 +++ /sys/src/9/bcm/usbdwc.c Wed Mar 2 00:00:00 2016 @@ -435,7 +435,7 @@ else continue; } - if(dir == Epin && ep->ttype == Tbulk && n == nleft){ + if(dir == Epin && ep->ttype == Tbulk){ nt = (hctsiz & Xfersize) - (hc->hctsiz & Xfersize); if(nt != n){ if(n == ROUND(nt, 4))