From f5a8ea6fd8908c6f42670b8546239fdbc7fdbf03 Mon Sep 17 00:00:00 2001 From: David du Colombier <0intro@gmail.com> Date: Thu, 2 Jun 2011 09:33:56 -0400 Subject: venti: import changes from plan 9 R=rsc CC=plan9port.codebot http://codereview.appspot.com/4523057 --- src/cmd/venti/dump.c | 1 - src/cmd/venti/root.c | 10 ++++++++-- src/cmd/venti/srv/arena.c | 5 ++++- src/cmd/venti/srv/arenas.c | 9 +++++++-- src/cmd/venti/srv/buildindex.c | 21 +++++++++++++++++---- src/cmd/venti/srv/checkarenas.c | 4 +++- src/cmd/venti/srv/config.c | 4 ++-- src/cmd/venti/srv/conv.c | 10 ++++++++-- src/cmd/venti/srv/dat.h | 2 ++ src/cmd/venti/srv/findscore.c | 2 +- src/cmd/venti/srv/fmtarenas.c | 2 +- src/cmd/venti/srv/fmtindex.c | 4 ++-- src/cmd/venti/srv/icachewrite.c | 1 - src/cmd/venti/srv/index.c | 14 +++++++++----- src/cmd/venti/srv/printarena.c | 4 ++-- src/cmd/venti/srv/printarenapart.c | 2 +- src/cmd/venti/srv/rdarena.c | 13 +++++++++---- src/cmd/venti/srv/syncindex.c | 2 +- src/cmd/venti/srv/venti.c | 14 +++++++++++++- src/cmd/venti/srv/wrarena.c | 8 ++++---- 20 files changed, 94 insertions(+), 38 deletions(-) (limited to 'src/cmd/venti') diff --git a/src/cmd/venti/dump.c b/src/cmd/venti/dump.c index c148b866..44f13e06 100644 --- a/src/cmd/venti/dump.c +++ b/src/cmd/venti/dump.c @@ -103,7 +103,6 @@ threadmain(int argc, char *argv[]) fmtinstall('F', vtfcallfmt); fmtinstall('V', vtscorefmt); - type = -1; ARGBEGIN{ case 'h': host = EARGF(usage()); diff --git a/src/cmd/venti/root.c b/src/cmd/venti/root.c index d41d44bc..3ade4be6 100644 --- a/src/cmd/venti/root.c +++ b/src/cmd/venti/root.c @@ -4,6 +4,12 @@ #include #include +enum +{ + // XXX What to do here? + VtMaxLumpSize = 65536, +}; + void usage(void) { @@ -38,7 +44,7 @@ threadmain(int argc, char *argv[]) if(argc == 0) usage(); - buf = vtmallocz(8192); + buf = vtmallocz(VtMaxLumpSize); z = vtdial(host); if(z == nil) @@ -52,7 +58,7 @@ threadmain(int argc, char *argv[]) fprint(2, "cannot parse score '%s': %r\n", argv[i]); continue; } - n = vtread(z, score, VtRootType, buf, 8192); + n = vtread(z, score, VtRootType, buf, VtMaxLumpSize); if(n < 0){ fprint(2, "could not read block %V: %r\n", score); continue; diff --git a/src/cmd/venti/srv/arena.c b/src/cmd/venti/srv/arena.c index 08952aa1..ddeb3b4c 100644 --- a/src/cmd/venti/srv/arena.c +++ b/src/cmd/venti/srv/arena.c @@ -728,9 +728,12 @@ okarena(Arena *arena) logerr(ECorrupt, "arena %s uncompressed size inconsistent with used space %lld %d %lld", arena->name, arena->diskstats.uncsize, arena->diskstats.clumps, arena->diskstats.used); */ + /* + * this happens; it's harmless. + * if(arena->ctime > arena->wtime) logerr(ECorrupt, "arena %s creation time after last write time", arena->name); - + */ return ok; } diff --git a/src/cmd/venti/srv/arenas.c b/src/cmd/venti/srv/arenas.c index cb402e20..0316c4c8 100644 --- a/src/cmd/venti/srv/arenas.c +++ b/src/cmd/venti/srv/arenas.c @@ -148,6 +148,7 @@ initarenapart(Part *part) ap->arenas = MKNZ(Arena*, ap->narenas); for(i = 0; i < ap->narenas; i++){ + debugarena = i; ap->arenas[i] = initarena(part, ap->map[i].start, ap->map[i].stop - ap->map[i].start, ap->blocksize); if(ap->arenas[i] == nil){ seterr(ECorrupt, "%s: %r", ap->map[i].name); @@ -168,8 +169,11 @@ initarenapart(Part *part) } } - for(i = 0; i < ap->narenas; i++) + for(i = 0; i < ap->narenas; i++) { + debugarena = i; addarena(ap->arenas[i]); + } + debugarena = -1; return ap; } @@ -359,7 +363,8 @@ parseamap(IFile *f, AMapN *amn) } n = v; if(n > MaxAMap){ - seterr(ECorrupt, "illegal number of elements in %s", f->name); + seterr(ECorrupt, "illegal number of elements %d in %s", + n, f->name); return -1; } am = MKNZ(AMap, n); diff --git a/src/cmd/venti/srv/buildindex.c b/src/cmd/venti/srv/buildindex.c index ce5103a7..08f0901a 100644 --- a/src/cmd/venti/srv/buildindex.c +++ b/src/cmd/venti/srv/buildindex.c @@ -50,18 +50,19 @@ static void arenapartproc(void*); void usage(void) { - fprint(2, "usage: buildindex [-bd] [-i isect]... [-M imem] venti.conf\n"); + fprint(2, "usage: buildindex [-b] [-i isect]... [-M imem] venti.conf\n"); threadexitsall("usage"); } void threadmain(int argc, char *argv[]) { - int fd, i, napart; + int fd, i, napart, nfinish, maxdisks; u32int bcmem, imem; Config conf; Part *p; + maxdisks = 100000; ventifmtinstall(); imem = 256*1024*1024; ARGBEGIN{ @@ -78,6 +79,9 @@ threadmain(int argc, char *argv[]) case 'M': imem = unittoull(EARGF(usage())); break; + case 'm': /* temporary - might go away */ + maxdisks = atoi(EARGF(usage())); + break; default: usage(); break; @@ -146,17 +150,21 @@ threadmain(int argc, char *argv[]) /* start arena procs */ p = nil; napart = 0; + nfinish = 0; arenadonechan = chancreate(sizeof(void*), 0); for(i=0; inarenas; i++){ if(ix->arenas[i]->part != p){ p = ix->arenas[i]->part; vtproc(arenapartproc, p); - napart++; + if(++napart >= maxdisks){ + recvp(arenadonechan); + nfinish++; + } } } /* wait for arena procs to finish */ - for(i=0; inarenas; i++) - if(should(ap->arenas[i]->name, argc, argv)) + if(should(ap->arenas[i]->name, argc, argv)) { + debugarena = i; checkarena(ap->arenas[i], scan, fix); + } if(verbose > 1) printstats(); diff --git a/src/cmd/venti/srv/config.c b/src/cmd/venti/srv/config.c index 1664147f..8c29fe95 100644 --- a/src/cmd/venti/srv/config.c +++ b/src/cmd/venti/srv/config.c @@ -75,7 +75,7 @@ runconfig(char *file, Config *config) if(readifile(&f, file) < 0) return -1; memset(config, 0, sizeof *config); - config->mem = 0xFFFFFFFFUL; + config->mem = Unspecified; ok = -1; line = nil; for(;;){ @@ -142,7 +142,7 @@ runconfig(char *file, Config *config) flds[1], file); break; } - if(config->mem != 0xFFFFFFFFUL){ + if(config->mem != Unspecified){ seterr(EAdmin, "duplicate mem lines in config file %s", file); break; } diff --git a/src/cmd/venti/srv/conv.c b/src/cmd/venti/srv/conv.c index 29b8c5f0..e6a6cbfe 100644 --- a/src/cmd/venti/srv/conv.c +++ b/src/cmd/venti/srv/conv.c @@ -15,6 +15,8 @@ #define U32PUT(p,v) (p)[0]=((v)>>24)&0xFF;(p)[1]=((v)>>16)&0xFF;(p)[2]=((v)>>8)&0xFF;(p)[3]=(v)&0xFF #define U64PUT(p,v,t32) t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32) +int debugarena = -1; /* hack to improve error reporting */ + static struct { u32int m; char *s; @@ -112,7 +114,9 @@ unpackarena(Arena *arena, u8int *buf) m = U32GET(p); if(m != ArenaMagic){ - seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaMagic (%#lux)", fmtmagic(fbuf, m), ArenaMagic); + seterr(ECorrupt, "arena %d has wrong magic number: %s " + "expected ArenaMagic (%#lux)", debugarena, + fmtmagic(fbuf, m), ArenaMagic); return -1; } p += U32Size; @@ -308,7 +312,9 @@ unpackarenahead(ArenaHead *head, u8int *buf) m = U32GET(p); if(m != ArenaHeadMagic){ - seterr(ECorrupt, "arena has wrong magic number: %s expected ArenaHeadMagic (%#lux)", fmtmagic(fbuf, m), ArenaHeadMagic); + seterr(ECorrupt, "arena %d head has wrong magic number: %s " + "expected ArenaHeadMagic (%#lux)", debugarena, + fmtmagic(fbuf, m), ArenaHeadMagic); return -1; } diff --git a/src/cmd/venti/srv/dat.h b/src/cmd/venti/srv/dat.h index 7b2bf49d..3d8e7841 100644 --- a/src/cmd/venti/srv/dat.h +++ b/src/cmd/venti/srv/dat.h @@ -54,6 +54,7 @@ enum MaxIo = 64*1024, /* max size of a single read or write operation */ ICacheBits = 16, /* default bits for indexing icache */ MaxAMap = 31*1024, /* max. allowed arenas in an address mapping; must be < 32*1024 */ + Unspecified = TWID32, /* * return codes from syncarena @@ -750,6 +751,7 @@ extern int l1quantum; extern int ignorebloom; extern int icacheprefetch; extern int syncwrites; +extern int debugarena; /* print in arena error msgs; -1==unknown */ extern Stats *stathist; extern int nstathist; diff --git a/src/cmd/venti/srv/findscore.c b/src/cmd/venti/srv/findscore.c index b68e0f8b..412b07d4 100644 --- a/src/cmd/venti/srv/findscore.c +++ b/src/cmd/venti/srv/findscore.c @@ -93,7 +93,7 @@ threadmain(int argc, char *argv[]) file = argv[0]; if(strscore(argv[1], score) < 0) - sysfatal("bad score %s\n", argv[1]); + sysfatal("bad score %s", argv[1]); part = initpart(file, OREAD|ODIRECT); if(part == nil) diff --git a/src/cmd/venti/srv/fmtarenas.c b/src/cmd/venti/srv/fmtarenas.c index 29e581c4..f196f22d 100644 --- a/src/cmd/venti/srv/fmtarenas.c +++ b/src/cmd/venti/srv/fmtarenas.c @@ -105,7 +105,7 @@ threadmain(int argc, char *argv[]) if(limit >= ap->size || ap->size - limit < MinArenaSize){ limit = ap->size; if(limit - addr < MinArenaSize) - sysfatal("bad arena set math: runt arena at %lld,%lld %lld\n", addr, limit, ap->size); + sysfatal("bad arena set math: runt arena at %lld,%lld %lld", addr, limit, ap->size); } snprint(aname, ANameSize, "%s%d", name, i); diff --git a/src/cmd/venti/srv/fmtindex.c b/src/cmd/venti/srv/fmtindex.c index d24e0619..2a5148ea 100644 --- a/src/cmd/venti/srv/fmtindex.c +++ b/src/cmd/venti/srv/fmtindex.c @@ -82,11 +82,11 @@ threadmain(int argc, char *argv[]) arenas[n] = ap->arenas[j]; if(n < ix->narenas){ if(arenas[n] != ix->arenas[n]) - sysfatal("mismatched arenas %s and %s at slot %d\n", + sysfatal("mismatched arenas %s and %s at slot %d", arenas[n]->name, ix->arenas[n]->name, n); amap[n] = ix->amap[n]; if(amap[n].start != addr) - sysfatal("mis-located arena %s in index %s\n", arenas[n]->name, ix->name); + sysfatal("mis-located arena %s in index %s", arenas[n]->name, ix->name); addr = amap[n].stop; }else{ amap[n].start = addr; diff --git a/src/cmd/venti/srv/icachewrite.c b/src/cmd/venti/srv/icachewrite.c index 5d8d437d..e1406ef1 100644 --- a/src/cmd/venti/srv/icachewrite.c +++ b/src/cmd/venti/srv/icachewrite.c @@ -251,7 +251,6 @@ icachewritecoord(void *v) as = icachestate(); if(as.arena==iwrite.as.arena && as.aa==iwrite.as.aa){ /* will not be able to do anything more than last flush - kick disk */ - fprint(2, "icache: nothing to do - kick dcache\n"); trace(TraceProc, "icachewritecoord kick dcache"); kickdcache(); trace(TraceProc, "icachewritecoord kicked dcache"); diff --git a/src/cmd/venti/srv/index.c b/src/cmd/venti/srv/index.c index 0b68928a..07bf81c8 100644 --- a/src/cmd/venti/srv/index.c +++ b/src/cmd/venti/srv/index.c @@ -328,16 +328,20 @@ newindex(char *name, ISect **sects, int n) } if(nb >= ((u64int)1 << 32)){ - seterr(EBug, "index too large"); - return nil; + fprint(2, "%s: index is 2^32 blocks or more; ignoring some of it\n", + argv0); + nb = ((u64int)1 << 32) - 1; } div = (((u64int)1 << 32) + nb - 1) / nb; - ub = (((u64int)1 << 32) - 1) / div + 1; if(div < 100){ - seterr(EBug, "index divisor too coarse [%lld buckets]", nb); - return nil; + fprint(2, "%s: index divisor %d too coarse; " + "index larger than needed, ignoring some of it\n", + argv0, div); + div = 100; + nb = (((u64int)1 << 32) - 1) / (100 - 1); } + ub = (((u64int)1 << 32) - 1) / div + 1; if(ub > nb){ seterr(EBug, "index initialization math wrong"); return nil; diff --git a/src/cmd/venti/srv/printarena.c b/src/cmd/venti/srv/printarena.c index da70a259..399385ca 100644 --- a/src/cmd/venti/srv/printarena.c +++ b/src/cmd/venti/srv/printarena.c @@ -24,7 +24,7 @@ rdarena(Arena *arena, u64int offset) e = arena->base + arena->size; if(offset != ~(u64int)0) { if(offset >= e-a) - sysfatal("bad offset %llud >= %llud\n", + sysfatal("bad offset %llud >= %llud", offset, e-a); aa = offset; } else @@ -111,7 +111,7 @@ threadmain(int argc, char *argv[]) head.size, head.clumpmagic); if(aoffset+head.size > part->size) - sysfatal("arena is truncated: want %llud bytes have %llud\n", + sysfatal("arena is truncated: want %llud bytes have %llud", head.size, part->size); partblocksize(part, head.blocksize); diff --git a/src/cmd/venti/srv/printarenapart.c b/src/cmd/venti/srv/printarenapart.c index ed642c95..5367d966 100644 --- a/src/cmd/venti/srv/printarenapart.c +++ b/src/cmd/venti/srv/printarenapart.c @@ -26,7 +26,7 @@ rdarena(Arena *arena, u64int offset) e = arena->base + arena->size; if(offset != ~(u64int)0) { if(offset >= e-a) - sysfatal("bad offset %llud >= %llud\n", + sysfatal("bad offset %llud >= %llud", offset, e-a); aa = offset; } else diff --git a/src/cmd/venti/srv/rdarena.c b/src/cmd/venti/srv/rdarena.c index 909cc206..0ccc1d96 100644 --- a/src/cmd/venti/srv/rdarena.c +++ b/src/cmd/venti/srv/rdarena.c @@ -2,7 +2,7 @@ #include "dat.h" #include "fns.h" -static int verbose; +static int verbose, quiet; void usage(void) @@ -18,8 +18,10 @@ rdarena(Arena *arena) u64int a, e; u32int bs; - fprint(2, "copying %s to standard output\n", arena->name); - printarena(2, arena); + if (!quiet) { + fprint(2, "copying %s to standard output\n", arena->name); + printarena(2, arena); + } bs = MaxIoSize; if(bs < arena->blocksize) @@ -51,6 +53,9 @@ threadmain(int argc, char *argv[]) statsinit(); ARGBEGIN{ + case 'q': + quiet++; + break; case 'v': verbose++; break; @@ -87,5 +92,5 @@ threadmain(int argc, char *argv[]) } } - sysfatal("couldn't find arena %s\n", aname); + sysfatal("couldn't find arena %s", aname); } diff --git a/src/cmd/venti/srv/syncindex.c b/src/cmd/venti/srv/syncindex.c index 8f521558..6bf996ae 100644 --- a/src/cmd/venti/srv/syncindex.c +++ b/src/cmd/venti/srv/syncindex.c @@ -56,7 +56,7 @@ threadmain(int argc, char *argv[]) if(verbose) printindex(2, mainindex); if(syncindex(mainindex) < 0) - sysfatal("failed to sync index=%s: %r\n", mainindex->name); + sysfatal("failed to sync index=%s: %r", mainindex->name); flushicache(); flushdcache(); diff --git a/src/cmd/venti/srv/venti.c b/src/cmd/venti/srv/venti.c index 81495376..ce95c1fb 100644 --- a/src/cmd/venti/srv/venti.c +++ b/src/cmd/venti/srv/venti.c @@ -109,6 +109,9 @@ threadmain(int argc, char *argv[]) fprint(2, "conf..."); if(initventi(configfile, &config) < 0) sysfatal("can't init server: %r"); + /* + * load bloom filter + */ if(mainindex->bloom && loadbloom(mainindex->bloom) < 0) sysfatal("can't load bloom filter: %r"); @@ -139,15 +142,22 @@ threadmain(int argc, char *argv[]) if(mem == 0xffffffffUL) mem = 1 * 1024 * 1024; + + /* + * lump cache + */ if(0) fprint(2, "initialize %d bytes of lump cache for %d lumps\n", mem, mem / (8 * 1024)); initlumpcache(mem, mem / (8 * 1024)); + /* + * index cache + */ initicache(icmem); initicachewrite(); /* - * need a block for every arena and every process + * block cache: need a block for every arena and every process */ minbcmem = maxblocksize * (mainindex->narenas + mainindex->nsects*4 + 16); @@ -186,6 +196,8 @@ threadmain(int argc, char *argv[]) ventiserver(nil); else vtproc(ventiserver, nil); + + threadexits(nil); } static void diff --git a/src/cmd/venti/srv/wrarena.c b/src/cmd/venti/srv/wrarena.c index 9308ca88..fee8e427 100644 --- a/src/cmd/venti/srv/wrarena.c +++ b/src/cmd/venti/srv/wrarena.c @@ -75,7 +75,7 @@ rdarena(Arena *arena, u64int offset) e = arena->base + arena->size; if(offset != ~(u64int)0) { if(offset >= e - a) - sysfatal("bad offset %#llx >= %#llx\n", offset, e - a); + sysfatal("bad offset %#llx >= %#llx", offset, e - a); aa = offset; } else aa = 0; @@ -87,8 +87,8 @@ rdarena(Arena *arena, u64int offset) break; if(a < aa || ci.type == VtCorruptType){ if(ci.type == VtCorruptType) - fprint(2, "corrupt at %#llx: +%d\n", - a, ClumpSize+ci.size); + fprint(2, "%s: corrupt clump read at %#llx: +%d\n", + argv0, a, ClumpSize+ci.size); continue; } lump = loadclump(arena, a, 0, &cl, score, 0); @@ -187,7 +187,7 @@ threadmain(int argc, char *argv[]) sysfatal("corrupted arena header: %r"); if(aoffset+head.size > part->size) - sysfatal("arena is truncated: want %llud bytes have %llud\n", + sysfatal("arena is truncated: want %llud bytes have %llud", head.size, part->size); partblocksize(part, head.blocksize); -- cgit v1.2.3