ref: c8500961fbf7f65a5fd5e0e522c702b17a79ca27
parent: 0b55bc315e748a6f3b855a6611eab059729a748f
author: 9ferno <gophone2015@gmail.com>
date: Fri Oct 14 16:20:54 EDT 2022
streamline the use of Ref and Iobuf.io
--- a/all.h
+++ b/all.h
@@ -38,16 +38,32 @@
/*
using Ref to avoid locking up the hash bucket when a process
is waiting for a lock on an Iobuf in that hash bucket.
+
+ The Ref ensures that an Iobuf is not stolen before another process
+ can get to wlock()'ing it after letting go of the lock on the hash bucket.
+ We cannot hold the lock on the hash bucket until we wlock() the iobuf
+ as that blocks other processes from using the hash bucket. This could
+ also result in a deadlock. For example, the directory entry is block 18,
+ which hashes to a hash index of 7. A writer() locked the directory entry
+ iobuf and wants to add a data block 84 to the directory entry. Block 84
+ hashes to the same hash index of 7. Another process wanting to access the
+ directory entry is waiting for a lock on that io buffer. While doing so,
+ it has locked the hash bucket. Now, this has caused a deadlock between
+ both these processes. The first process cannot proceed until it can
+ lock the hash bucket holding block 84 and is still holding the lock
+ on the directory entry in block 18. The second process cannot lock
+ block 18 and is holding the lock on the hash bucket.
+
for locking a buffer:
- qlock(hash bucket); incref(buffer); qunlock(hash bucket); wlock(buffer)
+ qlock(hash bucket); incref(buffer); qunlock(hash bucket);
+ wlock(buffer); decref(buffer);
+
for stealing an unused buffer:
qlock(hash bucket);
- find a buffer with ref == 0 and wlock'able.
- incref(buffer);
+ find a buffer with ref == 0 and wlock()'able.
qunlock(hash bucket);
- wlock(buffer);
+
for unlocking a buffer:
- decref(buffer);
wunlock(buffer);
*/
struct Iobuf
@@ -58,8 +74,10 @@
u16 len; /* number of blocks of data xiobuf points to */
Iobuf *fore; /* for lru */
Iobuf *back; /* for lru */
- u8 *xiobuf; /* "real" buffer pointer */
- Content *io; /* buffer pointer that is only active while locked. */
+ union{+ u8 *xiobuf; /* "real" buffer pointer */
+ Content *io; /* cast'able to contents */
+ };
int flags;
};
--- a/docs/mafs.ms
+++ b/docs/mafs.ms
@@ -640,8 +640,10 @@
u16 len; /* number of blocks of data xiobuf points to */
Iobuf *fore; /* for lru */
Iobuf *back; /* for lru */
- u8 *xiobuf; /* "real" buffer pointer */
- Content *io; /* buffer pointer that is only active while locked. */
+ union{+ u8 *xiobuf; /* "real" buffer pointer */
+ Content *io; /* cast'able to contents */
+ }
int flags;
};
.fi
binary files a/docs/mafs.pdf b/docs/mafs.pdf differ
--- a/iobuf.c
+++ b/iobuf.c
@@ -35,7 +35,6 @@
p->blkno = 0;
p->len = len;
p->xiobuf = emalloc9p(len*Rawblocksize);
- p->io = nil;
return p;
}
@@ -95,7 +94,7 @@
incref(p);
qunlock(hp);
if(chatty9p > 4)
- dprint(" after qunlock(hp) hp 0x%p blkno %llud\n",+ dprint(" in cache, after qunlock(hp) hp 0x%p blkno %llud\n",hp, blkno);
if(p->len != len){if(chatty9p > 4)
@@ -112,13 +111,11 @@
p->xiobuf = emalloc9p(len*Rawblocksize);
p->len = len;
devread(blkno, p->xiobuf, len);
- chkwunlock(p);
- }
- if(flags & Bmod){- if(chatty9p > 4)
- dprint(" in cache iobuf 0x%p has len %llud blkno %llud len %llud .."- " wlock() p->readers %d\n",
- p, p->len, blkno, len, p->readers);
+ if((flags & Bmod) == 0){+ chkwunlock(p);
+ rlock(p);
+ }
+ }else if(flags & Bmod){wlock(p);
if(chatty9p > 4)
dprint(" after wlock() blkno %llud\n", blkno);@@ -128,8 +125,8 @@
" rlock()\n", p, p->len, blkno, len);
rlock(p);
}
+ decref(p);
p->flags |= flags;
- p->io = (Content*)p->xiobuf;
return p;
}
p = p->fore;
@@ -155,14 +152,12 @@
p->xiobuf = emalloc9p(len*Rawblocksize);
p->len = len;
}
- incref(p);
- chkwunlock(p); /* lock'ing done in found */
hp->link = p;
if(chatty9p > 4)
dprint(" stealing iobuf 0x%p of %llud blocks"" for blkno %llud len %llud\n",
p, p->len, blkno, len);
- goto found;
+ goto found; /* p is wlock() */
}
s = p;
}while(p != hp->link);
@@ -173,7 +168,9 @@
if(chatty9p > 4)
dprint(" adding new Iobuf for blkno %llud\n", blkno);p = newbuf(hp, len);
- incref(p);
+ if(chatty9p > 4)
+ dprint(" .. wlock() blkno %llud\n", blkno);+ wlock(p);
found:
p->blkno = blkno;
@@ -181,18 +178,16 @@
if(chatty9p > 4)
dprint(" after qunlock(hp) hp 0x%p blkno %llud\n",hp, blkno);
- if(flags & Bmod){+ devread(blkno, p->xiobuf, len);
+ if((flags & Bmod) == 0){if(chatty9p > 4)
- dprint(" .. wlock() blkno %llud\n", blkno);- wlock(p);
- }else{- if(chatty9p > 4)
- dprint(" .. rlock() blkno %llud\n", blkno);+ dprint("new buffer: switching from wlock() to rlock() blkno %llud\n", blkno);+ incref(p);
+ wunlock(p);
rlock(p);
+ decref(p);
}
- devread(blkno, p->xiobuf, len);
p->flags = flags;
- p->io = (Content*)p->xiobuf;
return p;
}
@@ -230,9 +225,9 @@
if(chatty9p > 4)
dprint("putbuf p->blkno 0x%d t->c->type %d devtab[t->c->type]->dc %c\n"" p 0x%p p->flags 0x%ux %d\n"
- " p->xiobuf 0x%p p->iobuf 0x%p",
+ " p->xiobuf 0x%p",
p->blkno, t->c->type, devtab[t->c->type]->dc,
- p, p->flags, p->flags, p->xiobuf, p->io);
+ p, p->flags, p->flags, p->xiobuf);
*/
void
@@ -312,8 +307,6 @@
}
}
- p->io = nil;
- decref(p);
if(p->flags & Bmod){p->flags = 0;
chkwunlock(p);
@@ -419,11 +412,11 @@
}
dprint("showbuf p 0x%p blkno %llud len %d\n"" fore 0x%p back 0x%p\n"
- " xiobuf 0x%p io 0x%p flags 0x%x\n"
+ " xiobuf 0x%p flags 0x%x\n"
" caller %#p\n",
p, p->blkno, p->len,
p->fore, p->back,
- p->xiobuf, p->io, p->flags,
+ p->xiobuf, p->flags,
getcallerpc(&p));
if(p->io != nil)
showblock((u8*)p->io);