add flushpart; avoid O_DIRECT on linux

10 files changed, 39 insertions, 21 deletions

diff --git a/src/cmd/venti/srv/arena.c b/src/cmd/venti/srv/arena.c
index 4d8ab578..a8aaaa79 100644
--- a/src/cmd/venti/srv/arena.c
+++ b/src/cmd/venti/srv/arena.c
@@ -613,7 +613,8 @@ wbarenahead(Arena *arena)
 	 * during initialization.
 	 */
 	bad = packarenahead(&head, b->data)<0 ||
-	      writepart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize)<0;
+	      writepart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize)<0 ||
+	      flushpart(arena->part)<0;
 	freezblock(b);
 	if(bad)
 		return -1;
diff --git a/src/cmd/venti/srv/arenas.c b/src/cmd/venti/srv/arenas.c
index 05cb396b..cb402e20 100644
--- a/src/cmd/venti/srv/arenas.c
+++ b/src/cmd/venti/srv/arenas.c
@@ -222,7 +222,8 @@ wbarenapart(ArenaPart *ap)
 		freezblock(b);
 		return -1;
 	}
-	if(writepart(ap->part, PartBlank, b->data, HeadSize) < 0){
+	if(writepart(ap->part, PartBlank, b->data, HeadSize) < 0 ||
+	   flushpart(ap->part) < 0){
 		seterr(EAdmin, "can't write arena partition header: %r");
 		freezblock(b);
 		return -1;
@@ -325,7 +326,7 @@ wbarenamap(AMap *am, int n, Part *part, u64int base, u64int size)
 		freezblock(b);
 		return -1;
 	}
-	if(writepart(part, base, b->data, size) < 0){
+	if(writepart(part, base, b->data, size) < 0 || flushpart(part) < 0){
 		seterr(EAdmin, "can't write arena set: %r");
 		freezblock(b);
 		return -1;
diff --git a/src/cmd/venti/srv/bloom.c b/src/cmd/venti/srv/bloom.c
index bd398815..0710d362 100644
--- a/src/cmd/venti/srv/bloom.c
+++ b/src/cmd/venti/srv/bloom.c
@@ -121,7 +121,11 @@ int
 writebloom(Bloom *b)
 {
 	wbbloomhead(b);
-	return writepart(b->part, 0, b->data, b->size);
+	if(writepart(b->part, 0, b->data, b->size) < 0)
+		return -1;
+	if(flushpart(b->part) < 0)
+		return -1;
+	return 0;
 }
 
 /*
diff --git a/src/cmd/venti/srv/dcache.c b/src/cmd/venti/srv/dcache.c
index 52a5364d..9066296f 100644
--- a/src/cmd/venti/srv/dcache.c
+++ b/src/cmd/venti/srv/dcache.c
@@ -734,7 +734,7 @@ parallelwrites(DBlock **b, DBlock **eb, int dirty)
 	for(p=b; p<q; p++){
 		if(part != (*p)->part){
 			part = (*p)->part;
-			flushpart(part);
+			flushpart(part);	/* what if it fails? */
 		}
 	}
 
diff --git a/src/cmd/venti/srv/fns.h b/src/cmd/venti/srv/fns.h
index c631bfd1..263ccd30 100644
--- a/src/cmd/venti/srv/fns.h
+++ b/src/cmd/venti/srv/fns.h
@@ -38,6 +38,7 @@ Arena		*findarena(char *name);
 int		flushciblocks(Arena *arena);
 void		flushdcache(void);
 void		flushicache(void);
+int		flushpart(Part*);
 void		flushqueue(void);
 void		fmtzbinit(Fmt *f, ZBlock *b);
 void		freearena(Arena *arena);
diff --git a/src/cmd/venti/srv/icachewrite.c b/src/cmd/venti/srv/icachewrite.c
index 8f8fed4e..49344c9f 100644
--- a/src/cmd/venti/srv/icachewrite.c
+++ b/src/cmd/venti/srv/icachewrite.c
@@ -175,13 +175,12 @@ icachewritesect(Index *ix, ISect *is, u8int *buf)
 		diskaccess(1);
 
 		trace(TraceProc, "icachewritesect writepart", addr, nbuf);
-		if(writepart(is->part, addr, buf, nbuf) < 0){
+		if(writepart(is->part, addr, buf, nbuf) < 0 || flushpart(is->part) < 0){
 			/* XXX more details here */
 			fprint(2, "icachewriteproc writepart: %r\n");
 			err = -1;
 			continue;
 		}
-		flushpart(is->part);
 		addstat(StatIsectWriteBytes, nbuf);
 		addstat(StatIsectWrite, 1);
 		icacheclean(chunk);
diff --git a/src/cmd/venti/srv/index.c b/src/cmd/venti/srv/index.c
index 7d477e76..a5ffeef9 100644
--- a/src/cmd/venti/srv/index.c
+++ b/src/cmd/venti/srv/index.c
@@ -144,7 +144,8 @@ wbindex(Index *ix)
 		return -1;
 	}
 	for(i = 0; i < ix->nsects; i++){
-		if(writepart(ix->sects[i]->part, ix->sects[i]->tabbase, b->data, ix->tabsize) < 0){
+		if(writepart(ix->sects[i]->part, ix->sects[i]->tabbase, b->data, ix->tabsize) < 0
+		|| flushpart(ix->sects[i]->part) < 0){
 			seterr(EOk, "can't write index: %r");
 			freezblock(b);
 			return -1;
@@ -498,7 +499,7 @@ wbisect(ISect *is)
 		freezblock(b);
 		return -1;
 	}
-	if(writepart(is->part, PartBlank, b->data, HeadSize) < 0){
+	if(writepart(is->part, PartBlank, b->data, HeadSize) < 0 || flushpart(is->part) < 0){
 		seterr(EAdmin, "can't write index section header: %r");
 		freezblock(b);
 		return -1;
diff --git a/src/cmd/venti/srv/mirrorarenas.c b/src/cmd/venti/srv/mirrorarenas.c
index 0555336a..77634e84 100644
--- a/src/cmd/venti/srv/mirrorarenas.c
+++ b/src/cmd/venti/srv/mirrorarenas.c
@@ -80,7 +80,7 @@ ereadpart(Part *p, u64int offset, u8int *buf, u32int count)
 int
 ewritepart(Part *p, u64int offset, u8int *buf, u32int count)
 {
-	if(writepart(p, offset, buf, count) != count){
+	if(writepart(p, offset, buf, count) != count || flushpart(p) < 0){
 		chat("%T writepart %s at %#llux+%ud: %r\n", p->name, offset, count);
 		return -1;
 	}
diff --git a/src/cmd/venti/srv/part.c b/src/cmd/venti/srv/part.c
index e33c7b76..2be6a19f 100644
--- a/src/cmd/venti/srv/part.c
+++ b/src/cmd/venti/srv/part.c
@@ -16,16 +16,6 @@
 #include "dat.h"
 #include "fns.h"
 
-/* TODO for linux:
-don't use O_DIRECT.
-use
-	posix_fadvise(fd, 0, 0, POSIX_FADV_NOREUSE);
-after block is read and also use
-	posix_fadvise(fd, 0, 0, POSIX_FADV_RANDOM);
-to disable readahead on the index partition.
-bump block size of bloom filter higher.
-*/
-
 u32int	maxblocksize;
 int	readonly;
 
@@ -116,6 +106,9 @@ initpart(char *name, int mode)
 		mode &= (OREAD|OWRITE|ORDWR);
 		mode |= OREAD;
 	}
+#ifdef __linux__	/* sorry, but linus made O_DIRECT unusable! */
+	mode &= ~ODIRECT;
+#endif
 	part->fd = open(file, mode);
 	if(part->fd < 0){
 		if((mode&(OREAD|OWRITE|ORDWR)) == ORDWR)
@@ -130,6 +123,9 @@ initpart(char *name, int mode)
 		}
 		fprint(2, "warning: %s opened for reading only\n", name);
 	}
+#ifdef __linux__	/* sorry again!  still linus's fault! */
+	posix_fadvise(part->fd, 0, 0, POSIX_FADV_RANDOM);	/* disable readahead */
+#endif
 	part->offset = lo;
 	dir = dirfstat(part->fd);
 	if(dir == nil){
@@ -166,9 +162,18 @@ initpart(char *name, int mode)
 	return part;
 }
 
-void
+int
 flushpart(Part *part)
 {
+	USED(part);
+#ifdef __linux__	/* grrr! */
+	if(fsync(part->fd) < 0){
+		logerr(EAdmin, "flushpart %s: %r", part->name);
+		return -1;
+	}
+	posix_fadvise(part->fd, 0, 0, POSIX_FADV_DONTNEED);
+#endif
+	return 0;
 }
 
 void
@@ -420,6 +425,9 @@ rwpart(Part *part, int isread, u64int offset, u8int *buf, u32int count)
 #endif
 		break;
 	}
+#ifdef __linux__	/* sigh */
+	posix_fadvise(part->fd, part->offset+offset, n, POSIX_FADV_DONTNEED);
+#endif
 	return n;
 }
 int
diff --git a/src/cmd/venti/srv/zeropart.c b/src/cmd/venti/srv/zeropart.c
index b94b7284..97d6038e 100644
--- a/src/cmd/venti/srv/zeropart.c
+++ b/src/cmd/venti/srv/zeropart.c
@@ -23,5 +23,8 @@ zeropart(Part *part, int blocksize)
 		if(writepart(part, addr, b->data, blocksize) < 0)
 			sysfatal("can't initialize %s: %r", part->name);
 
+	if(flushpart(part) < 0)
+		sysfatal("can't flush writes to %s: %r", part->name);
+
 	freezblock(b);
 }