venti: import changes from plan 9

R=rsc
CC=plan9port.codebot
http://codereview.appspot.com/4523057

20 files changed, 94 insertions, 38 deletions

diff --git a/src/cmd/venti/dump.c b/src/cmd/venti/dump.c
index c148b866..44f13e06 100644
--- a/src/cmd/venti/dump.c
+++ b/src/cmd/venti/dump.c
@@ -103,7 +103,6 @@ threadmain(int argc, char *argv[])
 	fmtinstall('F', vtfcallfmt);
 	fmtinstall('V', vtscorefmt);
 
-	type = -1;
 	ARGBEGIN{
 	case 'h':
 		host = EARGF(usage());
diff --git a/src/cmd/venti/root.c b/src/cmd/venti/root.c
index d41d44bc..3ade4be6 100644
--- a/src/cmd/venti/root.c
+++ b/src/cmd/venti/root.c
@@ -4,6 +4,12 @@
 #include <libsec.h>
 #include <thread.h>
 
+enum
+{
+	// XXX What to do here?
+	VtMaxLumpSize = 65536,
+};
+
 void
 usage(void)
 {
@@ -38,7 +44,7 @@ threadmain(int argc, char *argv[])
 	if(argc == 0)
 		usage();
 
-	buf = vtmallocz(8192);
+	buf = vtmallocz(VtMaxLumpSize);
 
 	z = vtdial(host);
 	if(z == nil)
@@ -52,7 +58,7 @@ threadmain(int argc, char *argv[])
 			fprint(2, "cannot parse score '%s': %r\n", argv[i]);
 			continue;
 		}
-		n = vtread(z, score, VtRootType, buf, 8192);
+		n = vtread(z, score, VtRootType, buf, VtMaxLumpSize);
 		if(n < 0){
 			fprint(2, "could not read block %V: %r\n", score);
 			continue;
diff --git a/src/cmd/venti/srv/arena.c b/src/cmd/venti/srv/arena.c
index 08952aa1..ddeb3b4c 100644
--- a/src/cmd/venti/srv/arena.c
+++ b/src/cmd/venti/srv/arena.c
@@ -728,9 +728,12 @@ okarena(Arena *arena)
 		logerr(ECorrupt, "arena %s uncompressed size inconsistent with used space %lld %d %lld", arena->name, arena->diskstats.uncsize, arena->diskstats.clumps, arena->diskstats.used);
 	 */
 
+	/*
+	 * this happens; it's harmless.
+	 *
 	if(arena->ctime > arena->wtime)
 		logerr(ECorrupt, "arena %s creation time after last write time", arena->name);
-
+	 */
 	return ok;
 }
 
diff --git a/src/cmd/venti/srv/arenas.c b/src/cmd/venti/srv/arenas.c
index cb402e20..0316c4c8 100644
--- a/src/cmd/venti/srv/arenas.c
+++ b/src/cmd/venti/srv/arenas.c
@@ -148,6 +148,7 @@ initarenapart(Part *part)
 
 	ap->arenas = MKNZ(Arena*, ap->narenas);
 	for(i = 0; i < ap->narenas; i++){
+		debugarena = i;
 		ap->arenas[i] = initarena(part, ap->map[i].start, ap->map[i].stop - ap->map[i].start, ap->blocksize);
 		if(ap->arenas[i] == nil){
 			seterr(ECorrupt, "%s: %r", ap->map[i].name);
@@ -168,8 +169,11 @@ initarenapart(Part *part)
 		}
 	}
 
-	for(i = 0; i < ap->narenas; i++)
+	for(i = 0; i < ap->narenas; i++) {
+		debugarena = i;
 		addarena(ap->arenas[i]);
+	}
+	debugarena = -1;
 
 	return ap;
 }
@@ -359,7 +363,8 @@ parseamap(IFile *f, AMapN *amn)
 	}
 	n = v;
 	if(n > MaxAMap){
-		seterr(ECorrupt, "illegal number of elements in %s", f->name);
+		seterr(ECorrupt, "illegal number of elements %d in %s",
+			n, f->name);
 		return -1;
 	}
 	am = MKNZ(AMap, n);
diff --git a/src/cmd/venti/srv/buildindex.c b/src/cmd/venti/srv/buildindex.c
index ce5103a7..08f0901a 100644
--- a/src/cmd/venti/srv/buildindex.c
+++ b/src/cmd/venti/srv/buildindex.c
@@ -50,18 +50,19 @@ static void	arenapartproc(void*);
 void
 usage(void)
 {
-	fprint(2, "usage: buildindex [-bd] [-i isect]... [-M imem] venti.conf\n");
+	fprint(2, "usage: buildindex [-b] [-i isect]... [-M imem] venti.conf\n");
 	threadexitsall("usage");
 }
 
 void
 threadmain(int argc, char *argv[])
 {
-	int fd, i, napart;
+	int fd, i, napart, nfinish, maxdisks;
 	u32int bcmem, imem;
 	Config conf;
 	Part *p;
 	
+	maxdisks = 100000;
 	ventifmtinstall();
 	imem = 256*1024*1024;
 	ARGBEGIN{
@@ -78,6 +79,9 @@ threadmain(int argc, char *argv[])
 	case 'M':
 		imem = unittoull(EARGF(usage()));
 		break;
+	case 'm':	/* temporary - might go away */
+		maxdisks = atoi(EARGF(usage()));
+		break;
 	default:
 		usage();
 		break;
@@ -146,17 +150,21 @@ threadmain(int argc, char *argv[])
 	/* start arena procs */
 	p = nil;
 	napart = 0;
+	nfinish = 0;
 	arenadonechan = chancreate(sizeof(void*), 0);
 	for(i=0; i<ix->narenas; i++){
 		if(ix->arenas[i]->part != p){
 			p = ix->arenas[i]->part;
 			vtproc(arenapartproc, p);
-			napart++;
+			if(++napart >= maxdisks){
+				recvp(arenadonechan);
+				nfinish++;
+			}
 		}
 	}
 
 	/* wait for arena procs to finish */
-	for(i=0; i<napart; i++)
+	for(nfinish=0; nfinish<napart; nfinish++)
 		recvp(arenadonechan);
 
 	/* tell index procs to finish */
@@ -844,6 +852,11 @@ isectproc(void *v)
 			sysfatal("not enough memory");
 		nminibuf = nbuf;
 	}
+	if (nbuf == 0) {
+		fprint(2, "%s: brand-new index, no work to do\n", argv0);
+		threadexitsall(nil);
+	}
+
 	/* size buffer to use extra memory */
 	bufsize = MinBufSize;
 	while(bufsize*2*nbuf <= isectmem && bufsize < MaxBufSize)
diff --git a/src/cmd/venti/srv/checkarenas.c b/src/cmd/venti/srv/checkarenas.c
index 54eb2e44..4ad03a29 100644
--- a/src/cmd/venti/srv/checkarenas.c
+++ b/src/cmd/venti/srv/checkarenas.c
@@ -128,8 +128,10 @@ threadmain(int argc, char *argv[])
 	initdcache(8 * MaxDiskBlock);
 
 	for(i = 0; i < ap->narenas; i++)
-		if(should(ap->arenas[i]->name, argc, argv))
+		if(should(ap->arenas[i]->name, argc, argv)) {
+			debugarena = i;
 			checkarena(ap->arenas[i], scan, fix);
+		}
 
 	if(verbose > 1)
 		printstats();
diff --git a/src/cmd/venti/srv/config.c b/src/cmd/venti/srv/config.c
index 1664147f..8c29fe95 100644
--- a/src/cmd/venti/srv/config.c
+++ b/src/cmd/venti/srv/config.c
@@ -75,7 +75,7 @@ runconfig(char *file, Config *config)
 	if(readifile(&f, file) < 0)
 		return -1;
 	memset(config, 0, sizeof *config);
-	config->mem = 0xFFFFFFFFUL;
+	config->mem = Unspecified;
 	ok = -1;
 	line = nil;
 	for(;;){
@@ -142,7 +142,7 @@ runconfig(char *file, Config *config)
 					flds[1], file);
 				break;
 			}
-			if(config->mem != 0xFFFFFFFFUL){
+			if(config->mem != Unspecified){
 				seterr(EAdmin, "duplicate mem lines in config file %s", file);
 				break;
 			}
diff --git a/src/cmd/venti/srv/conv.c b/src/cmd/venti/srv/conv.c
index 29b8c5f0..e6a6cbfe 100644
--- a/src/cmd/venti/srv/conv.c
+++ b/src/cmd/venti/srv/conv.c
@@ -15,6 +15,8 @@
 #define	U32PUT(p,v)	(p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF
 #define	U64PUT(p,v,t32)	t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32)
 
+int debugarena = -1;		/* hack to improve error reporting */
+
 static struct {
 	u32int m;
 	char *s;
@@ -112,7 +114,9 @@ unpackarena(Arena *arena, u8int *buf)
 
 	m = U32GET(p);
 	if(m != ArenaMagic){
-		seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaMagic (%#lux)", fmtmagic(fbuf, m), ArenaMagic);
+		seterr(ECorrupt, "arena %d has wrong magic number: %s "
+			"expected ArenaMagic (%#lux)", debugarena,
+			fmtmagic(fbuf, m), ArenaMagic);
 		return -1;
 	}
 	p += U32Size;
@@ -308,7 +312,9 @@ unpackarenahead(ArenaHead *head, u8int *buf)
 
 	m = U32GET(p);
 	if(m != ArenaHeadMagic){
-		seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaHeadMagic (%#lux)", fmtmagic(fbuf, m), ArenaHeadMagic);
+		seterr(ECorrupt, "arena %d head has wrong magic number: %s "
+			"expected ArenaHeadMagic (%#lux)", debugarena,
+			fmtmagic(fbuf, m), ArenaHeadMagic);
 		return -1;
 	}
 
diff --git a/src/cmd/venti/srv/dat.h b/src/cmd/venti/srv/dat.h
index 7b2bf49d..3d8e7841 100644
--- a/src/cmd/venti/srv/dat.h
+++ b/src/cmd/venti/srv/dat.h
@@ -54,6 +54,7 @@ enum
 	MaxIo			= 64*1024,	/* max size of a single read or write operation */
 	ICacheBits		= 16,		/* default bits for indexing icache */
 	MaxAMap			= 31*1024,	/* max. allowed arenas in an address mapping; must be < 32*1024 */
+	Unspecified		= TWID32,
 
 	/*
 	 * return codes from syncarena
@@ -750,6 +751,7 @@ extern	int		l1quantum;
 extern	int		ignorebloom;
 extern	int		icacheprefetch;
 extern	int		syncwrites;
+extern	int		debugarena; /* print in arena error msgs; -1==unknown */
 
 extern	Stats	*stathist;
 extern	int	nstathist;
diff --git a/src/cmd/venti/srv/findscore.c b/src/cmd/venti/srv/findscore.c
index b68e0f8b..412b07d4 100644
--- a/src/cmd/venti/srv/findscore.c
+++ b/src/cmd/venti/srv/findscore.c
@@ -93,7 +93,7 @@ threadmain(int argc, char *argv[])
 
 	file = argv[0];
 	if(strscore(argv[1], score) < 0)
-		sysfatal("bad score %s\n", argv[1]);
+		sysfatal("bad score %s", argv[1]);
 
 	part = initpart(file, OREAD|ODIRECT);
 	if(part == nil)
diff --git a/src/cmd/venti/srv/fmtarenas.c b/src/cmd/venti/srv/fmtarenas.c
index 29e581c4..f196f22d 100644
--- a/src/cmd/venti/srv/fmtarenas.c
+++ b/src/cmd/venti/srv/fmtarenas.c
@@ -105,7 +105,7 @@ threadmain(int argc, char *argv[])
 		if(limit >= ap->size || ap->size - limit < MinArenaSize){
 			limit = ap->size;
 			if(limit - addr < MinArenaSize)
-				sysfatal("bad arena set math: runt arena at %lld,%lld %lld\n", addr, limit, ap->size);
+				sysfatal("bad arena set math: runt arena at %lld,%lld %lld", addr, limit, ap->size);
 		}
 
 		snprint(aname, ANameSize, "%s%d", name, i);
diff --git a/src/cmd/venti/srv/fmtindex.c b/src/cmd/venti/srv/fmtindex.c
index d24e0619..2a5148ea 100644
--- a/src/cmd/venti/srv/fmtindex.c
+++ b/src/cmd/venti/srv/fmtindex.c
@@ -82,11 +82,11 @@ threadmain(int argc, char *argv[])
 			arenas[n] = ap->arenas[j];
 			if(n < ix->narenas){
 				if(arenas[n] != ix->arenas[n])
-					sysfatal("mismatched arenas %s and %s at slot %d\n",
+					sysfatal("mismatched arenas %s and %s at slot %d",
 						arenas[n]->name, ix->arenas[n]->name, n);
 				amap[n] = ix->amap[n];
 				if(amap[n].start != addr)
-					sysfatal("mis-located arena %s in index %s\n", arenas[n]->name, ix->name);
+					sysfatal("mis-located arena %s in index %s", arenas[n]->name, ix->name);
 				addr = amap[n].stop;
 			}else{
 				amap[n].start = addr;
diff --git a/src/cmd/venti/srv/icachewrite.c b/src/cmd/venti/srv/icachewrite.c
index 5d8d437d..e1406ef1 100644
--- a/src/cmd/venti/srv/icachewrite.c
+++ b/src/cmd/venti/srv/icachewrite.c
@@ -251,7 +251,6 @@ icachewritecoord(void *v)
 		as = icachestate();
 		if(as.arena==iwrite.as.arena && as.aa==iwrite.as.aa){
 			/* will not be able to do anything more than last flush - kick disk */
-			fprint(2, "icache: nothing to do - kick dcache\n");
 			trace(TraceProc, "icachewritecoord kick dcache");
 			kickdcache();
 			trace(TraceProc, "icachewritecoord kicked dcache");
diff --git a/src/cmd/venti/srv/index.c b/src/cmd/venti/srv/index.c
index 0b68928a..07bf81c8 100644
--- a/src/cmd/venti/srv/index.c
+++ b/src/cmd/venti/srv/index.c
@@ -328,16 +328,20 @@ newindex(char *name, ISect **sects, int n)
 	}
 
 	if(nb >= ((u64int)1 << 32)){
-		seterr(EBug, "index too large");
-		return nil;
+		fprint(2, "%s: index is 2^32 blocks or more; ignoring some of it\n",
+			argv0);
+		nb = ((u64int)1 << 32) - 1;
 	}
 
 	div = (((u64int)1 << 32) + nb - 1) / nb;
-	ub = (((u64int)1 << 32) - 1) / div + 1;
 	if(div < 100){
-		seterr(EBug, "index divisor too coarse [%lld buckets]", nb);
-		return nil;
+		fprint(2, "%s: index divisor %d too coarse; "
+			"index larger than needed, ignoring some of it\n",
+			argv0, div);
+		div = 100;
+		nb = (((u64int)1 << 32) - 1) / (100 - 1);
 	}
+	ub = (((u64int)1 << 32) - 1) / div + 1;
 	if(ub > nb){
 		seterr(EBug, "index initialization math wrong");
 		return nil;
diff --git a/src/cmd/venti/srv/printarena.c b/src/cmd/venti/srv/printarena.c
index da70a259..399385ca 100644
--- a/src/cmd/venti/srv/printarena.c
+++ b/src/cmd/venti/srv/printarena.c
@@ -24,7 +24,7 @@ rdarena(Arena *arena, u64int offset)
 	e = arena->base + arena->size;
 	if(offset != ~(u64int)0) {
 		if(offset >= e-a)
-			sysfatal("bad offset %llud >= %llud\n",
+			sysfatal("bad offset %llud >= %llud",
 				offset, e-a);
 		aa = offset;
 	} else
@@ -111,7 +111,7 @@ threadmain(int argc, char *argv[])
 		head.size, head.clumpmagic);
 
 	if(aoffset+head.size > part->size)
-		sysfatal("arena is truncated: want %llud bytes have %llud\n",
+		sysfatal("arena is truncated: want %llud bytes have %llud",
 			head.size, part->size);
 
 	partblocksize(part, head.blocksize);
diff --git a/src/cmd/venti/srv/printarenapart.c b/src/cmd/venti/srv/printarenapart.c
index ed642c95..5367d966 100644
--- a/src/cmd/venti/srv/printarenapart.c
+++ b/src/cmd/venti/srv/printarenapart.c
@@ -26,7 +26,7 @@ rdarena(Arena *arena, u64int offset)
 	e = arena->base + arena->size;
 	if(offset != ~(u64int)0) {
 		if(offset >= e-a)
-			sysfatal("bad offset %llud >= %llud\n",
+			sysfatal("bad offset %llud >= %llud",
 				offset, e-a);
 		aa = offset;
 	} else
diff --git a/src/cmd/venti/srv/rdarena.c b/src/cmd/venti/srv/rdarena.c
index 909cc206..0ccc1d96 100644
--- a/src/cmd/venti/srv/rdarena.c
+++ b/src/cmd/venti/srv/rdarena.c
@@ -2,7 +2,7 @@
 #include "dat.h"
 #include "fns.h"
 
-static int	verbose;
+static int verbose, quiet;
 
 void
 usage(void)
@@ -18,8 +18,10 @@ rdarena(Arena *arena)
 	u64int a, e;
 	u32int bs;
 
-	fprint(2, "copying %s to standard output\n", arena->name);
-	printarena(2, arena);
+	if (!quiet) {
+		fprint(2, "copying %s to standard output\n", arena->name);
+		printarena(2, arena);
+	}
 
 	bs = MaxIoSize;
 	if(bs < arena->blocksize)
@@ -51,6 +53,9 @@ threadmain(int argc, char *argv[])
 	statsinit();
 
 	ARGBEGIN{
+	case 'q':
+		quiet++;
+		break;
 	case 'v':
 		verbose++;
 		break;
@@ -87,5 +92,5 @@ threadmain(int argc, char *argv[])
 		}
 	}
 
-	sysfatal("couldn't find arena %s\n", aname);
+	sysfatal("couldn't find arena %s", aname);
 }
diff --git a/src/cmd/venti/srv/syncindex.c b/src/cmd/venti/srv/syncindex.c
index 8f521558..6bf996ae 100644
--- a/src/cmd/venti/srv/syncindex.c
+++ b/src/cmd/venti/srv/syncindex.c
@@ -56,7 +56,7 @@ threadmain(int argc, char *argv[])
 	if(verbose)
 		printindex(2, mainindex);
 	if(syncindex(mainindex) < 0)
-		sysfatal("failed to sync index=%s: %r\n", mainindex->name);
+		sysfatal("failed to sync index=%s: %r", mainindex->name);
 	flushicache();
 	flushdcache();
 
diff --git a/src/cmd/venti/srv/venti.c b/src/cmd/venti/srv/venti.c
index 81495376..ce95c1fb 100644
--- a/src/cmd/venti/srv/venti.c
+++ b/src/cmd/venti/srv/venti.c
@@ -109,6 +109,9 @@ threadmain(int argc, char *argv[])
 	fprint(2, "conf...");
 	if(initventi(configfile, &config) < 0)
 		sysfatal("can't init server: %r");
+	/*
+	 * load bloom filter
+	 */
 	if(mainindex->bloom && loadbloom(mainindex->bloom) < 0)
 		sysfatal("can't load bloom filter: %r");
 
@@ -139,15 +142,22 @@ threadmain(int argc, char *argv[])
 
 	if(mem == 0xffffffffUL)
 		mem = 1 * 1024 * 1024;
+
+	/*
+	 * lump cache
+	 */
 	if(0) fprint(2, "initialize %d bytes of lump cache for %d lumps\n",
 		mem, mem / (8 * 1024));
 	initlumpcache(mem, mem / (8 * 1024));
 
+	/*
+	 * index cache
+	 */
 	initicache(icmem);
 	initicachewrite();
 
 	/*
-	 * need a block for every arena and every process
+	 * block cache: need a block for every arena and every process
 	 */
 	minbcmem = maxblocksize * 
 		(mainindex->narenas + mainindex->nsects*4 + 16);
@@ -186,6 +196,8 @@ threadmain(int argc, char *argv[])
 		ventiserver(nil);
 	else
 		vtproc(ventiserver, nil);
+
+	threadexits(nil);
 }
 
 static void
diff --git a/src/cmd/venti/srv/wrarena.c b/src/cmd/venti/srv/wrarena.c
index 9308ca88..fee8e427 100644
--- a/src/cmd/venti/srv/wrarena.c
+++ b/src/cmd/venti/srv/wrarena.c
@@ -75,7 +75,7 @@ rdarena(Arena *arena, u64int offset)
 	e = arena->base + arena->size;
 	if(offset != ~(u64int)0) {
 		if(offset >= e - a)
-			sysfatal("bad offset %#llx >= %#llx\n", offset, e - a);
+			sysfatal("bad offset %#llx >= %#llx", offset, e - a);
 		aa = offset;
 	} else
 		aa = 0;
@@ -87,8 +87,8 @@ rdarena(Arena *arena, u64int offset)
 			break;
 		if(a < aa || ci.type == VtCorruptType){
 			if(ci.type == VtCorruptType)
-				fprint(2, "corrupt at %#llx: +%d\n",
-					a, ClumpSize+ci.size);
+				fprint(2, "%s: corrupt clump read at %#llx: +%d\n",
+					argv0, a, ClumpSize+ci.size);
 			continue;
 		}
 		lump = loadclump(arena, a, 0, &cl, score, 0);
@@ -187,7 +187,7 @@ threadmain(int argc, char *argv[])
 		sysfatal("corrupted arena header: %r");
 
 	if(aoffset+head.size > part->size)
-		sysfatal("arena is truncated: want %llud bytes have %llud\n",
+		sysfatal("arena is truncated: want %llud bytes have %llud",
 			head.size, part->size);
 
 	partblocksize(part, head.blocksize);