git: 9front

Download patch

ref: 1017c54c78841a955fe87aa1cb6d0256d5ac8fce
parent: 4f4a7afe98b8bd09f20db27200728fa5f0109323
author: cinap_lenrek <cinap_lenrek@felloff.net>
date: Tue Dec 22 11:29:55 EST 2020 

kernel: avoid palloc lock during mmurelease()

Previously, mmurelease() was always called with
palloc spinlock held.

This is unneccesary for some mmurelease()
implementations as they wont release pages
to the palloc pool.

This change removes pagechainhead() and
pagechaindone() and replaces them with just
freepages() call, which aquires the palloc
lock internally as needed.

freepages() avoids holding the palloc lock
while walking the linked list of pages,
avoding some lock contention.


--- a/sys/src/9/bcm/mmu.c
+++ b/sys/src/9/bcm/mmu.c
@@ -173,17 +173,9 @@
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %lud", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
--- a/sys/src/9/bcm64/mmu.c
+++ b/sys/src/9/bcm64/mmu.c
@@ -518,20 +518,10 @@
 void
 mmurelease(Proc *p)
 {
-	Page *t;
-
 	mmuswitch(nil);
 	mmufree(p);
-
-	if((t = p->mmufree) != nil){
-		do {
-			p->mmufree = t->next;
-			if(--t->ref != 0)
-				panic("mmurelease: bad page ref");
-			pagechainhead(t);
-		} while((t = p->mmufree) != nil);
-		pagechaindone();
-	}
+	freepages(p->mmufree, nil, 0);
+	p->mmufree = nil;
 }
 
 void
--- a/sys/src/9/cycv/mmu.c
+++ b/sys/src/9/cycv/mmu.c
@@ -197,23 +197,21 @@
 void
 mmurelease(Proc *proc)
 {
-	Page *p, *n;
+	Page *p;
 
-	if(islo())
-		panic("mmurelease: islo");
-	
 	l1switch(&m->l1, 0);
-	if(proc->kmaptable != nil){
+	if((p = proc->kmaptable) != nil){
+		if(p->ref != 1)
+			panic("mmurelease: kmap ref %ld", p->ref);
 		if(proc->l1 == nil)
 			panic("mmurelease: no l1");
-		if(decref(proc->kmaptable) != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
-		if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa)
-			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa);
+		if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa)
+			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa);
 		proc->l1->va[L1X(KMAP)] = 0;
-		pagechainhead(proc->kmaptable);
+		p->next = proc->mmufree;
+		proc->mmufree = p;
 		proc->kmaptable = nil;
 	}
 	if(proc->l1 != nil){
@@ -221,14 +219,7 @@
 		l1free(proc->l1);
 		proc->l1 = nil;
 	}
-	for(p = proc->mmufree; p != nil; p = n){
-		n = p->next;
-		if(decref(p) != 0)
-			panic("mmurelease: p->ref %ld", p->ref);
-		pagechainhead(p);
-	}
-	if(proc->mmufree != nil)
-		pagechaindone();
+	freepages(proc->mmufree, nil, 0);
 	proc->mmufree = nil;
 }
 
--- a/sys/src/9/kw/mmu.c
+++ b/sys/src/9/kw/mmu.c
@@ -252,20 +252,12 @@
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate l1 caches */
 	cacheuwbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %lud", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
--- a/sys/src/9/omap/mmu.c
+++ b/sys/src/9/omap/mmu.c
@@ -234,20 +234,12 @@
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate l1 caches */
 	cacheuwbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
--- a/sys/src/9/pc/mmu.c
+++ b/sys/src/9/pc/mmu.c
@@ -320,23 +320,20 @@
  *   cleaning any user entries in the pdb (proc->mmupdb);
  *   if there's a pdb put it in the cache of pre-initialised pdb's
  *   for this processor (m->pdbpool) or on the process' free list;
- *   finally, place any pages freed back into the free pool (palloc).
- * This routine is only called from schedinit() with palloc locked.
+ *   finally, place any pages freed back into the free pool (freepages).
  */
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
 	ulong *pdb;
+	Page *page;
 
-	if(islo())
-		panic("mmurelease: islo");
 	taskswitch(PADDR(m->pdb), (ulong)m + BY2PG);
-	if(proc->kmaptable != nil){
+	if((page = proc->kmaptable) != nil){
+		if(page->ref != 1)
+			panic("mmurelease: kmap ref %ld", page->ref);
 		if(proc->mmupdb == nil)
 			panic("mmurelease: no mmupdb");
-		if(--proc->kmaptable->ref != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
 		/*
@@ -343,31 +340,28 @@
 		 * remove kmaptable from pdb before putting pdb up for reuse.
 		 */
 		pdb = tmpmap(proc->mmupdb);
-		if(PPN(pdb[PDX(KMAP)]) != proc->kmaptable->pa)
+		if(PPN(pdb[PDX(KMAP)]) != page->pa)
 			panic("mmurelease: bad kmap pde %#.8lux kmap %#.8lux",
-				pdb[PDX(KMAP)], proc->kmaptable->pa);
+				pdb[PDX(KMAP)], page->pa);
 		pdb[PDX(KMAP)] = 0;
 		tmpunmap(pdb);
+
 		/*
 		 * move kmaptable to free list.
 		 */
-		pagechainhead(proc->kmaptable);
+		page->next = proc->mmufree;
+		proc->mmufree = page;
 		proc->kmaptable = nil;
 	}
-	if(proc->mmupdb != nil){
+	if((page = proc->mmupdb) != nil){
 		mmuptefree(proc);
-		mmupdbfree(proc, proc->mmupdb);
+		mmupdbfree(proc, page);
 		proc->mmupdb = nil;
 	}
-	for(page = proc->mmufree; page != nil; page = next){
-		next = page->next;
-		if(--page->ref != 0)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
+	if((page = proc->mmufree) != nil){
+		freepages(page, nil, 0);
+		proc->mmufree = nil;
 	}
-	if(proc->mmufree != nil)
-		pagechaindone();
-	proc->mmufree = nil;
 	if(proc->ldt != nil){
 		free(proc->ldt);
 		proc->ldt = nil;
--- a/sys/src/9/port/devsegment.c
+++ b/sys/src/9/port/devsegment.c
@@ -469,7 +469,7 @@
 {
 	KMap *k;
 	Segment *s;
-	Page **f, *p, *l, *h;
+	Page **f, *p, *l, *h, *t;
 	ulong n, i;
 	int color;
 
@@ -492,12 +492,13 @@
 			continue;
 
 		i = 0;
-		h = nil;
+		h = t = nil;
 		f = &palloc.head;
 		while((p = *f) != nil){
 			if(p > &l[-len] && p <= l){
 				*f = p->next;
-				p->next = h;
+				if((p->next = h) == nil)
+					t = p;
 				h = p;
 				if(++i < len)
 					continue;
@@ -505,15 +506,15 @@
 			}
 			f = &p->next;
 		}
-		palloc.freecount -= i;
 
 		if(i != len){
-			while((p = h) != nil){
-				h = h->next;
-				pagechainhead(p);
+			if(h != nil){
+				t->next = palloc.head;
+				palloc.head = h;
 			}
 			goto Retry;
 		}
+		palloc.freecount -= i;
 		unlock(&palloc);
 
 		p = &l[-len];
--- a/sys/src/9/port/page.c
+++ b/sys/src/9/port/page.c
@@ -11,7 +11,7 @@
 pageinit(void)
 {
 	int color, i, j;
-	Page *p;
+	Page *p, **t;
 	Pallocmem *pm;
 	vlong m, v, u;
 
@@ -29,8 +29,12 @@
 	}
 
 	color = 0;
+	palloc.freecount = 0;
 	palloc.head = nil;
+
+	t = &palloc.head;
 	p = palloc.pages;
+
 	for(i=0; i<nelem(palloc.mem); i++){
 		pm = &palloc.mem[i];
 		for(j=0; j<pm->npage; j++){
@@ -40,7 +44,8 @@
 				continue;
 			p->color = color;
 			color = (color+1)%NCOLOR;
-			pagechainhead(p);
+			*t = p, t = &p->next;
+			palloc.freecount++;
 			p++;
 		}
 	}
@@ -65,15 +70,7 @@
 	print("%lldM swap\n", v/(1024*1024));
 }
 
-void
-pagechainhead(Page *p)
-{
-	p->next = palloc.head;
-	palloc.head = p;
-	palloc.freecount++;
-}
-
-void
+static void
 pagechaindone(void)
 {
 	if(palloc.pwait[0].p != nil && wakeup(&palloc.pwait[0]) != nil)
@@ -85,11 +82,23 @@
 void
 freepages(Page *head, Page *tail, ulong np)
 {
-	assert(palloc.Lock.p == up);
+	if(head == nil)
+		return;
+	if(tail == nil){
+		tail = head;
+		for(np = 1;; np++){
+			tail->ref = 0;
+			if(tail->next == nil)
+				break;
+			tail = tail->next;
+		}
+	}
+	lock(&palloc);
 	tail->next = palloc.head;
 	palloc.head = head;
 	palloc.freecount += np;
 	pagechaindone();
+	unlock(&palloc);
 }
 
 ulong
@@ -138,11 +147,8 @@
 	}
 	putimage(i);
 
-	if(np > 0){
-		lock(&palloc);
+	if(np > 0)
 		freepages(fh, ft, np);
-		unlock(&palloc);
-	}
 
 	return np;
 }
@@ -237,11 +243,8 @@
 		decref(p);
 		return;
 	}
-	if(decref(p) == 0){
-		lock(&palloc);
+	if(decref(p) == 0)
 		freepages(p, p, 1);
-		unlock(&palloc);
-	}
 }
 
 void
--- a/sys/src/9/port/portfns.h
+++ b/sys/src/9/port/portfns.h
@@ -218,8 +218,6 @@
 int		openmode(ulong);
 Block*		packblock(Block*);
 Block*		padblock(Block*, int);
-void		pagechaindone(void);
-void		pagechainhead(Page*);
 void		pageinit(void);
 ulong		pagereclaim(Image*);
 void		panic(char*, ...);
--- a/sys/src/9/port/proc.c
+++ b/sys/src/9/port/proc.c
@@ -81,27 +81,21 @@
 		case Moribund:
 			up->state = Dead;
 			edfstop(up);
-			if(up->edf != nil)
+			if(up->edf != nil){
 				free(up->edf);
-			up->edf = nil;
+				up->edf = nil;
+			}
 
-			/*
-			 * Holding locks from pexit:
-			 * 	procalloc
-			 *	palloc
-			 */
 			mmurelease(up);
-			unlock(&palloc);
 
-			updatecpu(up);
+			lock(&procalloc);
 			up->mach = nil;
-
 			up->qnext = procalloc.free;
 			procalloc.free = up;
-
 			/* proc is free now, make sure unlock() wont touch it */
 			up = procalloc.Lock.p = nil;
 			unlock(&procalloc);
+
 			sched();
 		}
 		coherence();
@@ -1222,10 +1216,6 @@
 		}
 	}
 	qunlock(&up->seglock);
-
-	/* Sched must not loop for these locks */
-	lock(&procalloc);
-	lock(&palloc);
 
 	edfstop(up);
 	up->state = Moribund;
--- a/sys/src/9/teg2/mmu.c
+++ b/sys/src/9/teg2/mmu.c
@@ -475,20 +475,12 @@
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate caches */
 	l1cache->wbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
--- a/sys/src/9/xen/mmu.c
+++ b/sys/src/9/xen/mmu.c
@@ -282,15 +282,8 @@
 		}
 	}
 
-	for(page = proc->mmufree; page; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld\n", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmufree)
-		pagechaindone();
-	proc->mmufree = 0;
+	freepages(proc->mmufree, nil, 0);
+	proc->mmufree = nil;
 }
 
 static Page*
--- a/sys/src/9/zynq/mmu.c
+++ b/sys/src/9/zynq/mmu.c
@@ -205,23 +205,22 @@
 void
 mmurelease(Proc *proc)
 {
-	Page *p, *n;
+	Page *p;
 
-	if(islo())
-		panic("mmurelease: islo");
-	
 	l1switch(&m->l1, 0);
-	if(proc->kmaptable != nil){
+	if((p = proc->kmaptable) != nil){
+		if(p->ref != 1)
+			panic("mmurelease: kmap ref %ld", p->ref);
 		if(proc->l1 == nil)
 			panic("mmurelease: no l1");
-		if(decref(proc->kmaptable) != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
-		if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa)
-			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa);
+		if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa)
+			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa);
 		proc->l1->va[L1X(KMAP)] = 0;
-		pagechainhead(proc->kmaptable);
+
+		p->next = proc->mmufree;
+		proc->mmufree = p;
 		proc->kmaptable = nil;
 	}
 	if(proc->l1 != nil){
@@ -229,14 +228,7 @@
 		l1free(proc->l1);
 		proc->l1 = nil;
 	}
-	for(p = proc->mmufree; p != nil; p = n){
-		n = p->next;
-		if(decref(p) != 0)
-			panic("mmurelease: p->ref %ld", p->ref);
-		pagechainhead(p);
-	}
-	if(proc->mmufree != nil)
-		pagechaindone();
+	freepages(proc->mmufree, nil, 0);
 	proc->mmufree = nil;
 }
 
-- 
⑨