Use long descriptor format for mmu page tables to support more than 4GB physical memory. Reference: /n/sources/patch/9-bcm-lpae Date: Thu Apr 15 18:49:31 BST 2021 Signed-off-by: miller@hamnavoe.com --- /sys/src/9/bcm/arm.h Thu Apr 15 18:47:13 2021 +++ /sys/src/9/bcm/arm.h Thu Apr 15 18:47:13 2021 @@ -227,6 +227,7 @@ */ #define CpTLDlock 0 /* TLB lockdown registers */ #define CpTLDpreload 1 /* TLB preload */ +#define CpTLDmair 2 #define CpTLDi 0 /* TLB instr. lockdown reg. */ #define CpTLDd 1 /* " data " " */ @@ -269,6 +270,14 @@ #define CpTTBowb (3<<3) /* outer write-back no write-allocate */ #define CpTTBs (1<<1) /* page table in shareable memory */ #define CpTTBbase ~0x7F /* mask off control bits */ + +/* + * CpTTBctl bits + */ +#define EAElpae (1<<31) /* large phyiscal address extension */ +#define SH0inner (3<<12) /* TTB0 inner shareable */ +#define ORGN0wb (1<<10) /* TTB0 outer region writeback cacheable */ +#define IRGN0wb (1<<8) /* TTB0 inner region writeback cacheable */ /* * MMU page table entries. --- /sys/src/9/bcm/armv7.s Thu Apr 15 18:47:13 2021 +++ /sys/src/9/bcm/armv7.s Thu Apr 15 18:47:13 2021 @@ -76,14 +76,38 @@ BL mmuinit(SB) /* - * set up domain access control and page table base + * set up domain access control, translation mode and page table base */ MOVW $Client, R1 MCR CpSC, 0, R1, C(CpDAC), C(0) + + MOVW $PADDR(L1), R1 + + MOVW ttbcr(SB), R2 + MCR CpSC, 0, R2, C(CpTTB), C(0), CpTTBctl + BARRIERS + AND.S $EAElpae, R2 + BNE lpae0 + MOVW $PADDR(L1), R1 ORR $(CpTTBs|CpTTBowba|CpTTBiwba), R1 MCR CpSC, 0, R1, C(CpTTB), C(0) MCR CpSC, 0, R1, C(CpTTB), C(0), CpTTB1 /* cortex has two */ + B lpaex0 + +lpae0: + ADD $(L1SIZE-64), R1 + + BARRIERS + MOVW $0, R2 + MCRR(CpSC, 0, 1, 2, CpTTB) /* TTBR0 */ + + MOVW mair0(SB), R2 + MCR CpSC, 0, R2, C(CpTLD), C(CpTLDmair) + ISB + MCR CpSC, 0, R0, C(CpTLB), C(CpTLBinvu), CpTLBinv + BARRIERS +lpaex0: /* * invalidate my caches before enabling @@ -187,15 +211,39 @@ ADD $(MACHSIZE-4), R(MACH), R13 /* - * set up domain access control and page table base + * set up domain access control, translation mode and page table base */ MOVW $Client, R1 MCR CpSC, 0, R1, C(CpDAC), C(0) + MOVW 12(R(MACH)), R1 /* m->mmul1 */ SUB $KZERO, R1 /* phys addr */ + + MOVW ttbcr(SB), R2 + MCR CpSC, 0, R2, C(CpTTB), C(0), CpTTBctl + AND.S $EAElpae, R2 + BNE lpae + ORR $(CpTTBs|CpTTBowba|CpTTBiwba), R1 MCR CpSC, 0, R1, C(CpTTB), C(0) MCR CpSC, 0, R1, C(CpTTB), C(0), CpTTB1 /* cortex has two */ + B lpaex + +lpae: + /* + * L0 page table (4 entries) + */ + ADD $(L1SIZE-64), R1 + + BARRIERS + MOVW $0, R2 + MCRR(CpSC, 0, 1, 2, CpTTB) + MOVW mair0(SB), R2 + MCR CpSC, 0, R2, C(CpTLD), C(CpTLDmair) + ISB + MCR CpSC, 0, R0, C(CpTLB), C(CpTLBinvu), CpTLBinv + BARRIERS +lpaex: /* * invalidate my caches before enabling @@ -274,6 +322,10 @@ TEXT cpctget(SB), 1, $-4 /* cache type */ MRC CpSC, 0, R0, C(CpID), C(CpIDidct), CpIDct + RET + +TEXT ttbrget(SB), 1, $-4 + MRC CpSC, 0, R0, C(CpTTB), C(0) RET TEXT lcycles(SB), 1, $-4 --- /sys/src/9/bcm/dat.h Thu Apr 15 18:47:14 2021 +++ /sys/src/9/bcm/dat.h Thu Apr 15 18:47:14 2021 @@ -28,6 +28,7 @@ typedef struct PMMU PMMU; typedef struct Proc Proc; typedef u32int PTE; +typedef u64int LPTE; typedef struct Soc Soc; typedef struct Uart Uart; typedef struct Ureg Ureg; @@ -96,11 +97,11 @@ struct Confmem { - uintptr base; + uvlong base; usize npage; - uintptr limit; - uintptr kbase; - uintptr klimit; + uvlong limit; + uvlong kbase; + uvlong klimit; }; struct Conf @@ -108,6 +109,7 @@ ulong nmach; /* processors */ ulong nproc; /* processes */ Confmem mem[2]; /* physical memory */ + Confmem himem; /* physical memory above 4GiB */ ulong npage; /* total physical pages of memory */ usize upages; /* user page pool */ ulong copymode; /* 0 is copy on write, 1 is copy on reference */ @@ -155,11 +157,17 @@ */ struct MMMU { - PTE* mmul1; /* l1 for this processor */ + union { + PTE* mmul1; /* l1 for this processor */ + LPTE* mmull1; /* l1 for this processor (long descriptors) */ + }; int mmul1lo; int mmul1hi; int mmupid; - PTE* kmapl2; /* l2 for section containing kmap area and vectors */ + union { + PTE* kmapl2; /* l2 for section containing kmap area and vectors */ + LPTE* kmapll2;/* as above (long descriptors) */ + }; }; /* @@ -172,7 +180,10 @@ Page* mmul2; Page* mmul2cache; /* free mmu pages */ int nkmap; - PTE kmaptab[NKMAPS]; + union { + PTE kmaptab[NKMAPS]; + LPTE kmapltab[NKMAPS]; + }; }; #include "../port/portdat.h" @@ -260,7 +271,7 @@ extern register Proc* up; /* R9 */ extern uintptr kseg0; extern Mach* machaddr[MAXMACH]; -extern ulong memsize; +extern uvlong memsize; extern int normalprint; /* --- /sys/src/9/bcm/fns.h Thu Apr 15 18:47:15 2021 +++ /sys/src/9/bcm/fns.h Thu Apr 15 18:47:14 2021 @@ -77,6 +77,7 @@ extern int l2ap(int); extern void l2cacheuwbinv(void); extern void links(void); +extern void lpapageinit(void); extern void mmuinit(void*); extern void mmuinit1(void); extern void mmuinvalidate(void); --- /sys/src/9/bcm/gisb.c Thu Apr 15 18:47:15 2021 +++ /sys/src/9/bcm/gisb.c Thu Apr 15 18:47:15 2021 @@ -13,7 +13,7 @@ /* * GISB arbiter registers */ -static u32int *regs = (u32int*)(VIRTIO + 0x400000); +static u32int *regs = (u32int*)(VIRTIO + 0x600000); enum { ArbMasterMask = 0x004/4, --- /sys/src/9/bcm/main.c Thu Apr 15 18:47:15 2021 +++ /sys/src/9/bcm/main.c Thu Apr 15 18:47:15 2021 @@ -31,7 +31,7 @@ uintptr kseg0 = KZERO; Mach* machaddr[MAXMACH]; Conf conf; -ulong memsize = 128*1024*1024; +uvlong memsize = 128*1024*1024; /* * Option arguments from the command line. @@ -269,7 +269,7 @@ memset(mm, 0, MACHSIZE); mm->machno = mach; - memmove(l1, m->mmul1, L1SIZE); /* clone cpu0's l1 table */ + mmuinit(l1); /* clone cpu0's l1 table */ cachedwbse(l1, L1SIZE); mm->mmul1 = l1; cachedwbse(mm, MACHSIZE); @@ -308,6 +308,8 @@ ataginit((Atag*)BOOTARGS); confinit(); /* figures out amount of memory */ xinit(); + /* set clock rate to arm_freq from config.txt (default pi1:700Mhz pi2:900MHz) */ + setclkrate(ClkArm, 0); uartconsinit(); screeninit(); @@ -321,8 +323,7 @@ for(;;) ; } - /* set clock rate to arm_freq from config.txt (default pi1:700Mhz pi2:900MHz) */ - setclkrate(ClkArm, 0); + print("vcore reports memory = 0x%llux\n", conf.mem[0].limit); trapinit(); clockinit(); printinit(); @@ -340,6 +341,7 @@ links(); chandevreset(); /* most devices are discovered here */ pageinit(); + lpapageinit(); swapinit(); userinit(); launchinit(getncpus()); @@ -506,6 +508,7 @@ void confinit(void) { + extern ulong ttbcr; int i, userpcnt; ulong kpages; uintptr pa; @@ -518,7 +521,7 @@ cpuserver = 0; } if((p = getconf("*maxmem")) != nil){ - memsize = strtoul(p, 0, 0) - PHYSDRAM; + memsize = strtoull(p, 0, 0) - PHYSDRAM; if (memsize < 16*MB) /* sanity */ memsize = 16*MB; } @@ -540,21 +543,31 @@ break; case 0xC00000: conf.mem[1].base = 1*GiB; - conf.mem[1].limit = 0xFFF00000; + conf.mem[1].limit = 0xFF000000; break; case 0xD00000: conf.mem[1].base = 1*GiB; - conf.mem[1].limit = 0xFFF00000; + conf.mem[1].limit = 0xFF000000; + if(ttbcr){ + /* lpae supported */ + conf.himem.base = 4LL*GiB; + conf.himem.limit = 8LL*GiB; + } break; } if(conf.mem[1].limit > soc.dramsize) conf.mem[1].limit = soc.dramsize; if(p != nil){ - if(memsize < conf.mem[0].limit){ - conf.mem[0].limit = memsize; - conf.mem[1].limit = 0; - }else if(memsize >= conf.mem[1].base && memsize < conf.mem[1].limit) - conf.mem[1].limit = memsize; + for(i = 0; i < nelem(conf.mem); i++){ + if(memsize < conf.mem[i].base) + conf.mem[i].limit = conf.mem[i].base; + else if(memsize < conf.mem[i].limit) + conf.mem[i].limit = memsize; + } + if(memsize < conf.himem.base) + conf.himem.limit = conf.himem.base; + else if(memsize < conf.himem.limit) + conf.himem.limit = memsize; } if(p = getconf("*kernelpercent")) @@ -569,6 +572,7 @@ conf.npage = 0; pa = PADDR(PGROUND(PTR2UINT(end))); + pa += BY2PG; /* * we assume that the kernel is at the beginning of one of the @@ -582,6 +591,7 @@ conf.mem[i].npage = (conf.mem[i].limit - conf.mem[i].base)/BY2PG; conf.npage += conf.mem[i].npage; } + conf.himem.npage = (conf.himem.limit - conf.himem.base)/BY2PG; if(userpcnt < 10 || userpcnt > 99) userpcnt = 90; @@ -591,7 +601,7 @@ conf.ialloc = ((conf.npage-conf.upages)/2)*BY2PG; /* set up other configuration parameters */ - conf.nproc = 100 + ((conf.npage*BY2PG)/MB)*5; + conf.nproc = 100 + (conf.npage/(MB/BY2PG))*5; if(cpuserver) conf.nproc *= 3; if(conf.nproc > 2000) --- /sys/src/9/bcm/mem.h Thu Apr 15 18:47:16 2021 +++ /sys/src/9/bcm/mem.h Thu Apr 15 18:47:16 2021 @@ -41,7 +41,7 @@ #define KZERO KSEG0 /* kernel address space */ #define CONFADDR (KZERO+0x100) /* unparsed plan9.ini */ #define MACHADDR (KZERO+0x2000) /* Mach structure */ -#define L2 (KZERO+0x3000) /* L2 ptes for vectors etc */ +#define L2 (KZERO+0x3000) /* L2 ptes for vectors etc (non-lpae) */ #define VCBUFFER (KZERO+0x3400) /* videocore mailbox buffer */ #define FIQSTKTOP (KZERO+0x4000) /* FIQ stack */ #define L1 (KZERO+0x4000) /* tt ptes: 16KiB aligned */ @@ -46,12 +46,12 @@ #define FIQSTKTOP (KZERO+0x4000) /* FIQ stack */ #define L1 (KZERO+0x4000) /* tt ptes: 16KiB aligned */ #define KTZERO (KZERO+0x8000) /* kernel text start */ -#define VIRTPCI 0xFD000000 /* pcie address space (pi4 only) */ +#define VIRTPCI 0xFC000000 /* pcie address space (pi4 only) */ +#define FRAMEBUFFER 0xFD000000 /* video framebuffer */ #define VIRTIO 0xFE000000 /* i/o registers */ #define IOSIZE (10*MiB) #define ARMLOCAL (VIRTIO+IOSIZE) /* armv7 only */ -#define VGPIO (ARMLOCAL+MiB) /* virtual gpio for pi3 ACT LED */ -#define FRAMEBUFFER (VGPIO+MiB) /* video framebuffer */ +#define VGPIO (ARMLOCAL+2*MiB) /* virtual gpio for pi3 ACT LED */ #define UZERO 0 /* user segment */ #define UTZERO (UZERO+BY2PG) /* user text start */ @@ -82,7 +82,7 @@ #define PTEPERTAB (PTEMAPMEM/BY2PG) #define SEGMAPSIZE 1984 #define SSEGMAPSIZE 16 -#define PPN(x) ((x)&~(BY2PG-1)) +#define PPN(x) ((x) & (~(BY2PG-1) | PTEHIMEM)) /* * With a little work these move to port. @@ -92,6 +92,7 @@ #define PTEWRITE (1<<1) #define PTEUNCACHED (1<<2) #define PTEKERNEL (1<<3) +#define PTEHIMEM (0xF<<8) /* * Physical machine information from here on. --- /sys/src/9/bcm/mkfile Thu Apr 15 18:47:16 2021 +++ /sys/src/9/bcm/mkfile Thu Apr 15 18:47:16 2021 @@ -51,7 +51,7 @@ fpiarm.$O\ fpimem.$O\ main.$O\ - mmu.$O\ +# mmu.$O\ syscall.$O\ # trap.$O\ $DEVS\ --- /sys/src/9/bcm/mmu.c Thu Apr 15 18:47:17 2021 +++ /sys/src/9/bcm/mmu.c Thu Apr 15 18:47:17 2021 @@ -21,8 +21,14 @@ KMAPADDR = 0xFFF00000, }; +ulong ttbcr = 0; +ulong mair0; +void lpapageinit(void) {} + /* - * Set up initial PTEs for cpu0 (called with mmu off) + * Set up initial PTEs + * Called before main, with mmu off, to initialise cpu0's tables + * Called from launchinit to clone cpu0's tables for other cpus */ void mmuinit(void *a) @@ -31,6 +37,10 @@ uintptr pa, pe, va; l1 = (PTE*)a; + if((uintptr)l1 != PADDR(L1)){ + memmove(l1, m->mmul1, L1SIZE); + return; + } l2 = (PTE*)PADDR(L2); /* @@ -63,12 +73,12 @@ l1[L1X(va)] = pa|Dom0|L1AP(Krw)|Section|L1noexec; /* * pi4 hack: ether and pcie are in segment 0xFD5xxxxx not 0xFE5xxxxx - * gisb is in segment 0xFC4xxxxx not FE4xxxxx + * gisb is in segment 0xFC4xxxxx not FE4xxxxx (and we map it at va FE6xxxxx) */ va = VIRTIO + 0x500000; pa = soc.physio - 0x1000000 + 0x500000; l1[L1X(va)] = pa|Dom0|L1AP(Krw)|Section|L1noexec; - va = VIRTIO + 0x400000; + va = VIRTIO + 0x600000; pa = soc.physio - 0x2000000 + 0x400000; l1[L1X(va)] = pa|Dom0|L1AP(Krw)|Section|L1noexec; --- /sys/src/9/bcm/mmu64.c Thu Jan 1 00:00:00 1970 +++ /sys/src/9/bcm/mmu64.c Thu Apr 15 18:47:17 2021 @@ -0,0 +1,560 @@ +/* + * armv7/armv8-a long-descriptor memory management + * + * Three levels of page tables: + * L0 table: 4 entries, each mapping 1GiB virtual space, as either + * a contiguous block, or + * an L1 page: 512 entries, each mapping 2MiB, as either + * a contiguous block, or + * an L2 page: 512 entries, each mapping 4KiB + * + */ + +#include "u.h" +#include "../port/lib.h" +#include "mem.h" +#include "dat.h" +#include "fns.h" + +#include "arm.h" + +typedef uvlong LPTE; + +#define L0X(va) FEXT((va), 30, 2) +#define L1X(va) FEXT((va), 21, 11) +#define L2X(va) FEXT((va), 12, 9) + +enum { + L1lo = L1X(UZERO), + L1hi = L1X(USTKTOP+2*MiB-1), + L2size = BY2PG, + KMAPADDR = 0xFFE00000, + + LXN = 1LL<<54, + LPXN = 1LL<<53, + LAF = 1<<10, + LShareOuter = 2<<8, + LShareInner = 3<<8, + LKrw = 0<<6, + LUrw = 1<<6, + LUro = 3<<6, + LMem = 0<<2, + LUncached = 1<<2, + LDevice = 2<<2, + LBlock = 1<<0, + LTable = 3<<0, + LPage = 3<<0, + + MMem = 0xFF, /* MEM_WB */ + MUncached = 0x44, /* MEM_UC */ + MDevice = 0x04, /* DEV_nGnRE*/ + + EAE_LPAE = 1<<31, + SH0_INNER = 3<<12, + ORGN0_WB = 1<<10, + IRGN0_WB = 1<<8, + + LAttrDRAM = LKrw | LAF | LShareInner | LMem, + LAttrPhys = LPXN | LAF | LShareOuter | LUncached, + LAttrIO = LKrw | LAF | LXN | LPXN | LShareOuter | LDevice, + LAttrUser = LPXN | LAF | LShareInner | LMem, +}; + +#define XPPN(pa) ((pa)&~(BY2PG-1) | (uvlong)((pa)&PTEHIMEM) << (32-8)) +#define LPPN(pte) ((pte)&0xFFFFFF000ULL) + +/* + * TTBCR and MAIR0 register settings for LPAE (used in armv7.s) + */ +ulong mair0 = MMem<<0 | MUncached<<8 | MDevice<<16; +ulong ttbcr = EAElpae | SH0inner | ORGN0wb | IRGN0wb; + +/* + * Set up initial PTEs + * Called before main, with mmu off, to initialise cpu0's tables + * Called from launchinit to clone cpu0's tables for other cpus + */ +void +mmuinit(void *a) +{ + LPTE *l0, *l1, *l2; + uintptr pa, pe, va; + int i; + + l1 = (LPTE*)a; + if((uintptr)l1 != PADDR(L1)) + memmove(l1, m->mmul1, L1SIZE); + + /* + * embed L0 table near end of L1 table + */ + l0 = (LPTE*)((uintptr)a + L1SIZE - 64); + pa = PADDR(l1); + for(i = 0; i < 4; i++){ + l0[i] = pa|LAttrDRAM|LTable; + pa += BY2PG; + } + + if((uintptr)l1 != PADDR(L1)) + return; + + /* + * map ram at KZERO, bounded above by VIRTPCI + */ + //l0[KZERO/GiB] = PHYSDRAM|LAttrDRAM|LBlock; + //va = KZERO + GiB; + va = KZERO; + pe = VIRTPCI - KZERO; + if(pe > soc.dramsize) + pe = soc.dramsize; + for(pa = PHYSDRAM; pa < PHYSDRAM+pe; pa += 2*MiB){ + l1[L1X(va)] = pa|LAttrDRAM|LBlock; + va += 2*MiB; + } + + /* + * identity map first 2MB of ram so mmu can be enabled + */ + l1[L1X(PHYSDRAM)] = PHYSDRAM|LAttrDRAM|LBlock; + /* + * map i/o registers + */ + va = VIRTIO; + for(pa = soc.physio; pa < soc.physio+IOSIZE; pa += 2*MiB){ + l1[L1X(va)] = pa|LAttrIO|LBlock; + va += 2*MiB; + } + pa = soc.armlocal; + if(pa) + l1[L1X(va)] = pa|LAttrIO|LBlock; + /* + * pi4 hack: ether and pcie are in segment 0xFD5xxxxx not 0xFE5xxxxx + * gisb is in segment 0xFC4xxxxx not FE4xxxxx (and we map it at va FE6xxxxx) + */ + va = VIRTIO + 0x400000; + pa = soc.physio - 0x1000000 + 0x400000; + l1[L1X(va)] = pa|LAttrIO|LBlock; + va = VIRTIO + 0x600000; + pa = soc.physio - 0x2000000 + 0x400000; + l1[L1X(va)] = pa|LAttrIO|LBlock; + + /* + * double map exception vectors near top of virtual memory + * not ready for malloc, so alias first L1 page (which has only one entry) + * as temporary L2 page + */ + l2 = (LPTE*)PADDR(L1); + //memset(l2, 0, BY2PG); + va = HVECTORS; + l2[L2X(va)] = PHYSDRAM|LAttrDRAM|LPage; + l1[L1X(va)] = (uintptr)l2|LAttrDRAM|LTable; +} + +void +mmuinit1() +{ + LPTE *l1, *l2; + uintptr va; + + l1 = m->mmull1; + + /* + * first L1 page: undo identity map and L2 alias + */ + memset(l1, 0, BY2PG); + cachedwbtlb(l1, BY2PG); + + /* + * make a local mapping for highest 2MB of virtual space + * containing kmap area and exception vectors + */ + va = HVECTORS; + m->kmapll2 = l2 = mallocalign(L2size, L2size, 0, 0); + l2[L2X(va)] = PHYSDRAM|LAttrDRAM|LPage; + l1[L1X(va)] = PADDR(l2)|LAttrDRAM|LTable; + cachedwbtlb(&l1[L1X(va)], sizeof(LPTE)); + + mmuinvalidate(); +} + +static int +nonzero(uint *p) +{ + int i; + for(i = 0; i < BY2PG/sizeof(uint); i++) + if(*p++ != 0) + return 1; + return 0; +} + +static void +mmul2empty(Proc* proc, int clear) +{ + LPTE *l1; + Page **l2, *page; + + l1 = m->mmull1; + l2 = &proc->mmul2; + for(page = *l2; page != nil; page = page->next){ + if(clear){ + if(l1[page->daddr] == Fault){ + l1[page->daddr] = XPPN(page->pa)|LAttrDRAM|LTable; + coherence(); + } + memset((void*)(0xFF000000 + L2size*page->daddr), 0, L2size); + } + l1[page->daddr] = Fault; + l2 = &page->next; + } + coherence(); + *l2 = proc->mmul2cache; + proc->mmul2cache = proc->mmul2; + proc->mmul2 = nil; +} + +static void +mmul1empty(void) +{ + LPTE *l1; + + /* clean out any user mappings still in l1 */ + if(m->mmul1lo > 0){ + if(m->mmul1lo == 1) + m->mmull1[L1lo] = Fault; + else + memset(&m->mmull1[L1lo], 0, m->mmul1lo*sizeof(LPTE)); + m->mmul1lo = 0; + } + if(m->mmul1hi > 0){ + l1 = &m->mmull1[L1hi - m->mmul1hi]; + if(m->mmul1hi == 1) + *l1 = Fault; + else + memset(l1, 0, m->mmul1hi*sizeof(LPTE)); + m->mmul1hi = 0; + } + if(m->kmapll2 != nil) + memset(m->kmapll2, 0, NKMAPS*sizeof(LPTE)); +} + +void +mmuswitch(Proc* proc) +{ + int x; + LPTE *l1; + Page *page; + + if(proc != nil && proc->newtlb){ + mmul2empty(proc, 1); + proc->newtlb = 0; + } + + mmul1empty(); + + /* move in new map */ + l1 = m->mmull1; + if(proc != nil){ + for(page = proc->mmul2; page != nil; page = page->next){ + x = page->daddr; + l1[x] = XPPN(page->pa)|LAttrDRAM|LTable; + if(x >= L1lo + m->mmul1lo && x < L1hi - m->mmul1hi){ + if(x+1 - L1lo < L1hi - x) + m->mmul1lo = x+1 - L1lo; + else + m->mmul1hi = L1hi - x; + } + } + if(proc->nkmap) + memmove(m->kmapll2, proc->kmapltab, sizeof(proc->kmapltab)); + } + + /* make sure map is in memory */ + /* could be smarter about how much? */ + cachedwbtlb(&l1[L1X(UZERO)], (L1hi - L1lo)*sizeof(LPTE)); + if(proc != nil && proc->nkmap) + cachedwbtlb(m->kmapll2, sizeof(proc->kmapltab)); + + /* lose any possible stale tlb entries */ + mmuinvalidate(); +} + +void +flushmmu(void) +{ + int s; + + s = splhi(); + up->newtlb = 1; + mmuswitch(up); + splx(s); +} + +void +mmurelease(Proc* proc) +{ + Page *page, *next; + + mmul2empty(proc, 0); + for(page = proc->mmul2cache; page != nil; page = next){ + next = page->next; + if(--page->ref) + panic("mmurelease: page->ref %d", page->ref); + pagechainhead(page); + } + if(proc->mmul2cache && palloc.r.p) + wakeup(&palloc.r); + proc->mmul2cache = nil; + + mmul1empty(); + + /* make sure map is in memory */ + /* could be smarter about how much? */ + cachedwbtlb(&m->mmull1[L1X(UZERO)], (L1hi - L1lo)*sizeof(LPTE)); + + /* lose any possible stale tlb entries */ + mmuinvalidate(); +} + +void +putmmu(uintptr va, uintptr pa, Page* page) +{ + int x, s; + Page *pg; + LPTE *l1, *pte; + LPTE nx; + static int chat = 0; + + /* + * disable interrupts to prevent flushmmu (called from hzclock) + * from clearing page tables while we are setting them + */ + s = splhi(); + x = L1X(va); + l1 = &m->mmull1[x]; + pte = (LPTE*)(0xFF000000 + L2size*x); + if(*l1 == Fault){ + /* l2 pages have 512 entries */ + if(up->mmul2cache == nil){ + spllo(); + pg = newpage(1, 0, 0); + splhi(); + /* if newpage slept, we might be on a different cpu */ + l1 = &m->mmull1[x]; + }else{ + pg = up->mmul2cache; + up->mmul2cache = pg->next; + } + pg->daddr = x; + pg->next = up->mmul2; + up->mmul2 = pg; + + *l1 = XPPN(pg->pa)|LAttrDRAM|LTable; + cachedwbtlb(l1, sizeof *l1); + + /* force l2 page to memory */ + if(nonzero((uint*)pte)) + if(chat++ < 32){ + iprint("nonzero L2 page from cache pa %lux va %lux daddr %lux\n", + pg->pa, pg->va, pg->daddr); + memset(pte, 0, L2size); + } + cachedwbtlb(pte, L2size); + + if(x >= L1lo + m->mmul1lo && x < L1hi - m->mmul1hi){ + if(x+1 - L1lo < L1hi - x) + m->mmul1lo = x+1 - L1lo; + else + m->mmul1hi = L1hi - x; + } + } + + /* protection bits are + * PTERONLY|PTEVALID; + * PTEWRITE|PTEVALID; + * PTEWRITE|PTEUNCACHED|PTEVALID; + */ + nx = LAttrUser|LPage; + if(pa & PTEUNCACHED) + nx = LAttrPhys|LPage; + if(pa & PTEWRITE) + nx |= LUrw; + else + nx |= LUro; + pte[L2X(va)] = XPPN(pa)|nx; + if(0)if(chat++ < 32) + iprint("putmmu %#p %lux => %llux\n", va, pa, pte[L2X(va)]); + cachedwbtlb(&pte[L2X(va)], sizeof(LPTE)); + + /* clear out the current entry */ + mmuinvalidateaddr(va); + + if(page->cachectl[m->machno] == PG_TXTFLUSH){ + /* pio() sets PG_TXTFLUSH whenever a text pg has been written */ + if(cankaddr(page->pa)) + cachedwbse((void*)(page->pa|KZERO), BY2PG); + cacheiinvse((void*)page->va, BY2PG); + page->cachectl[m->machno] = PG_NOFLUSH; + } + checkmmu(va, pa); + splx(s); +} + +/* + * Return the number of bytes that can be accessed via KADDR(pa). + * If pa is not a valid argument to KADDR, return 0. + */ +uintptr +cankaddr(uintptr pa) +{ + if(pa == 0) + return 0; + if((pa - PHYSDRAM) < VIRTPCI-KZERO) + return PHYSDRAM + VIRTPCI-KZERO - pa; + return 0; +} + +uintptr +mmukmapx(uintptr va, uvlong pa, usize size) +{ + int o; + usize n; + LPTE *pte, *pte0; + + print("mmukmapx %#p => %llux (%lux)\n", va, pa, size); + assert((va & (2*MiB-1)) == 0); + o = pa & (2*MiB-1); + pa -= o; + size += o; + pte = pte0 = &m->mmull1[L1X(va)]; + for(n = 0; n < size; n += 2*MiB) + if(*pte++ != Fault) + return 0; + pte = pte0; + for(n = 0; n < size; n += 2*MiB){ + *pte++ = (pa+n)|LAttrIO|LBlock; + mmuinvalidateaddr(va+n); + } + cachedwbtlb(pte0, (uintptr)pte - (uintptr)pte0); + return va + o; +} + +uintptr +mmukmap(uintptr va, uintptr pa, usize size) +{ + return mmukmapx(va, pa, size); +} + +void +checkmmu(uintptr va, uintptr pa) +{ + int x; + LPTE *l1, *pte; + + x = L1X(va); + l1 = &m->mmull1[x]; + if((*l1<able) != LTable){ + iprint("checkmmu cpu%d va=%lux l1 %p=%llux\n", m->machno, va, l1, *l1); + return; + } + pte = (LPTE*)(0xFF000000 + L2size*x); + pte += L2X(va); + if(pa == ~0 || (pa != 0 && LPPN(*pte) != XPPN(pa))){ + iprint("checkmmu va=%lux pa=%lux l1 %p=%llux pte %p", va, pa, l1, *l1, pte); + iprint("=%llux\n", *pte); + } +} + +KMap* +kmap(Page *p) +{ + int s, i; + uintptr va; + uvlong pa; + + pa = XPPN(p->pa); + if(pa < VIRTPCI-KZERO) + return KADDR(pa); + if(up == nil) + panic("kmap without up %#p", getcallerpc(&pa)); + s = splhi(); + if(up->nkmap == NKMAPS) + panic("kmap overflow %#p", getcallerpc(&pa)); + for(i = 0; i < NKMAPS; i++) + if(up->kmapltab[i] == 0) + break; + if(i == NKMAPS) + panic("can't happen"); + up->nkmap++; + va = KMAPADDR + i*BY2PG; + up->kmapltab[i] = pa|LAttrDRAM|LPage; + m->kmapll2[i] = up->kmapltab[i]; + cachedwbtlb(&m->kmapll2[i], sizeof(LPTE)); + mmuinvalidateaddr(va); + splx(s); + return (KMap*)va; +} + +void +kunmap(KMap *k) +{ + int i; + uintptr va; + + coherence(); + va = (uintptr)k; + if(L1X(va) != L1X(KMAPADDR)) + return; + /* wasteful: only needed for text pages aliased within data cache */ + cachedwbse((void*)va, BY2PG); + i = L2X(va); + up->kmapltab[i] = 0; + m->kmapll2[i] = 0; + up->nkmap--; + cachedwbtlb(&m->kmapll2[i], sizeof(LPTE)); + mmuinvalidateaddr(va); +} + +/* + * Append pages with physical addresses above 4GiB to the freelist + */ +void +lpapageinit(void) +{ + int j; + ulong npage; + Page *pages, *p; + uvlong pa; + + npage = conf.himem.npage; + if(npage == 0) + return; + + pages = xalloc(npage*sizeof(Page)); + if(pages == 0) + panic("lpapageinit"); + + p = pages; + pa = conf.himem.base; + for(j=0; jprev = p-1; + p->next = p+1; + p->pa = pa; + p->pa |= (pa >> (32-8)) & PTEHIMEM; + p++; + pa += BY2PG; + } + palloc.tail->next = pages; + pages[0].prev = palloc.tail; + palloc.tail = p - 1; + palloc.tail->next = 0; + + palloc.freecount += npage; + palloc.user += npage; + + /* Paging numbers */ + swapalloc.highwater = (palloc.user*5)/100; + swapalloc.headroom = swapalloc.highwater + (swapalloc.highwater/4); + + print("%ldM extended memory: ", npage*(BY2PG/1024)/1024); + print("%ldM user total\n", palloc.user*(BY2PG/1024)/1024); +} --- /sys/src/9/bcm/pi Thu Apr 15 18:47:18 2021 +++ /sys/src/9/bcm/pi Thu Apr 15 18:47:17 2021 @@ -49,6 +49,7 @@ misc armv6 + mmu trap uartmini gpio sdmmc emmc --- /sys/src/9/bcm/pi2 Thu Apr 15 18:47:18 2021 +++ /sys/src/9/bcm/pi2 Thu Apr 15 18:47:18 2021 @@ -10,6 +10,7 @@ arch ssl tls + bridge log cap fs ip arp chandial ip ipv6 ipaux iproute netlog nullmedium pktmedium ptclbsum inferno @@ -33,6 +34,7 @@ archbcm2 loopbackmedium ethermedium + netdevmedium sdhost usbdwc etherusb @@ -49,6 +51,7 @@ misc armv7 + mmu trap uartmini gpio sdmmc emmc --- /sys/src/9/bcm/pi2cpu Thu Apr 15 18:47:18 2021 +++ /sys/src/9/bcm/pi2cpu Thu Apr 15 18:47:18 2021 @@ -10,6 +10,7 @@ arch ssl tls + bridge log cap fs ip arp chandial ip ipv6 ipaux iproute netlog nullmedium pktmedium ptclbsum inferno @@ -33,6 +34,7 @@ archbcm2 loopbackmedium ethermedium + netdevmedium sdhost usbdwc etherusb @@ -49,6 +51,7 @@ misc armv7 + mmu trap uartmini gpio sdmmc emmc --- /sys/src/9/bcm/pi2wifi Thu Apr 15 18:47:19 2021 +++ /sys/src/9/bcm/pi2wifi Thu Apr 15 18:47:19 2021 @@ -48,6 +48,7 @@ misc armv7 + mmu trap uartmini gpio sdmmc emmc --- /sys/src/9/bcm/pi4 Thu Apr 15 18:47:19 2021 +++ /sys/src/9/bcm/pi4 Thu Apr 15 18:47:19 2021 @@ -10,6 +10,7 @@ arch ssl tls + bridge log cap fs ip arp chandial ip ipv6 ipaux iproute netlog nullmedium pktmedium ptclbsum inferno @@ -53,6 +54,7 @@ misc armv7 + mmu64 trap4 uartmini gpio sdmmc --- /sys/src/9/bcm/pi4cpu Thu Apr 15 18:47:20 2021 +++ /sys/src/9/bcm/pi4cpu Thu Apr 15 18:47:19 2021 @@ -10,6 +10,7 @@ arch ssl tls + bridge log cap fs ip arp chandial ip ipv6 ipaux iproute netlog nullmedium pktmedium ptclbsum inferno @@ -53,6 +54,7 @@ misc armv7 + mmu64 trap4 uartmini gpio sdmmc --- /sys/src/9/bcm/pi4wifi Thu Apr 15 18:47:20 2021 +++ /sys/src/9/bcm/pi4wifi Thu Apr 15 18:47:20 2021 @@ -53,6 +53,7 @@ misc armv7 + mmu64 trap4 uartmini gpio sdmmc --- /sys/src/9/bcm/picpu Thu Apr 15 18:47:20 2021 +++ /sys/src/9/bcm/picpu Thu Apr 15 18:47:20 2021 @@ -49,6 +49,7 @@ misc armv6 + mmu trap uartmini gpio sdmmc emmc --- /sys/src/9/bcm/pifat Thu Apr 15 18:47:21 2021 +++ /sys/src/9/bcm/pifat Thu Apr 15 18:47:21 2021 @@ -34,6 +34,7 @@ misc armv6 + mmu trap uartmini gpio sdmmc emmc --- /sys/src/9/bcm/piwifi Thu Apr 15 18:47:21 2021 +++ /sys/src/9/bcm/piwifi Thu Apr 15 18:47:21 2021 @@ -48,6 +48,7 @@ misc armv6 + mmu trap uartmini gpio sdmmc emmc --- /sys/src/9/bcm/rebootcode.s Thu Apr 15 18:47:22 2021 +++ /sys/src/9/bcm/rebootcode.s Thu Apr 15 18:47:22 2021 @@ -4,6 +4,7 @@ #include "arm.s" #define PTEDRAM (Dom0|L1AP(Krw)|Section) +#define PTELPAE (1<<10|3<<8|1<<0) /* AF | ShareInner | Block */ #define WFI WORD $0xe320f003 /* wait for interrupt */ #define WFE WORD $0xe320f002 /* wait for event */ @@ -30,8 +31,13 @@ /* redo double map of first MiB PHYSDRAM = KZERO */ MOVW 12(R(MACH)), R2 /* m->mmul1 (virtual addr) */ - MOVW $PTEDRAM, R1 /* PTE bits */ + MRC CpSC, 0, R3, C(CpTTB), C(0), CpTTBctl + AND.S $EAElpae, R3 + MOVW.EQ $PTEDRAM, R1 /* PTE bits (non-lpae) */ + MOVW.NE $PTELPAE, R1 /* PTE bits (lpae) */ MOVW R1, (R2) + MOVW $0, R1 + MOVW R1, 4(R2) DSB MCR CpSC, 0, R2, C(CpCACHE), C(CpCACHEwb), CpCACHEse --- /sys/src/9/bcm/trap.c Thu Apr 15 18:47:23 2021 +++ /sys/src/9/bcm/trap.c Thu Apr 15 18:47:23 2021 @@ -441,7 +441,6 @@ panic("terminal exception at %#lux", ureg->pc); break; case 0x4: /* icache maint fault */ - case 0x6: /* access flag fault (page) */ case 0x8: /* precise external abort, non-xlat'n */ case 0x28: case 0xc: /* l1 translation, precise ext. abort */ @@ -474,6 +473,7 @@ panic("kernel access violation: pc %#lux va %#p", ureg->pc, va); break; + case 0x6: /* access flag fault (page) */ case 0xd: case 0xf: /* permission error, copy on write or real permission error */ --- /sys/src/9/bcm/trap4.c Thu Apr 15 18:47:23 2021 +++ /sys/src/9/bcm/trap4.c Thu Apr 15 18:47:23 2021 @@ -711,7 +711,6 @@ panic("terminal exception at %#lux", ureg->pc); break; case 0x4: /* icache maint fault */ - case 0x6: /* access flag fault (page) */ case 0x8: /* precise external abort, non-xlat'n */ case 0x28: case 0x16: /* imprecise ext. abort, non-xlt'n */ @@ -721,7 +720,6 @@ break; case 0xc: /* l1 translation, precise ext. abort */ case 0x2c: - case 0xe: /* l2 translation, precise ext. abort */ case 0x2e: panic("external translation abort %#lux pc %#lux addr %#p", fsr, ureg->pc, va); @@ -734,6 +732,7 @@ break; case 0x5: /* translation fault, no section entry */ case 0x7: /* translation fault, no page entry */ + case 0xe: /* l2 translation, precise ext. abort */ faultarm(ureg, va, user, !writetomem(inst)); break; case 0x9: @@ -750,6 +749,7 @@ panic("kernel access violation: pc %#lux va %#p", ureg->pc, va); break; + case 0x6: /* access flag fault (page) */ case 0xd: case 0xf: /* permission error, copy on write or real permission error */