diff options
Diffstat (limited to 'src/cmd/vac')
-rw-r--r-- | src/cmd/vac/cache.c | 876 | ||||
-rw-r--r-- | src/cmd/vac/dat.h | 156 | ||||
-rw-r--r-- | src/cmd/vac/error.c | 20 | ||||
-rw-r--r-- | src/cmd/vac/error.h | 14 | ||||
-rw-r--r-- | src/cmd/vac/file.c | 1214 | ||||
-rw-r--r-- | src/cmd/vac/fns.h | 46 | ||||
-rw-r--r-- | src/cmd/vac/fs.c | 188 | ||||
-rw-r--r-- | src/cmd/vac/mkfile | 36 | ||||
-rw-r--r-- | src/cmd/vac/pack.c | 609 | ||||
-rw-r--r-- | src/cmd/vac/rtest.c | 71 | ||||
-rw-r--r-- | src/cmd/vac/source.c | 390 | ||||
-rw-r--r-- | src/cmd/vac/srcload.c | 302 | ||||
-rw-r--r-- | src/cmd/vac/stdinc.h | 8 | ||||
-rw-r--r-- | src/cmd/vac/util.c | 71 | ||||
-rw-r--r-- | src/cmd/vac/vac-orig.c | 1213 | ||||
-rw-r--r-- | src/cmd/vac/vac.c | 1024 | ||||
-rw-r--r-- | src/cmd/vac/vac.h | 126 | ||||
-rw-r--r-- | src/cmd/vac/vacfs.c | 849 | ||||
-rw-r--r-- | src/cmd/vac/vactest.c | 182 | ||||
-rw-r--r-- | src/cmd/vac/vtdump.c | 391 | ||||
-rw-r--r-- | src/cmd/vac/vtread.c | 126 | ||||
-rw-r--r-- | src/cmd/vac/wtest.c | 47 |
22 files changed, 7959 insertions, 0 deletions
diff --git a/src/cmd/vac/cache.c b/src/cmd/vac/cache.c new file mode 100644 index 00000000..fc34d688 --- /dev/null +++ b/src/cmd/vac/cache.c @@ -0,0 +1,876 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" + +typedef struct Label Label; + +enum { + BadHeap = ~0, +}; + +/* + * the plan is to store data to the cache in c->size blocks + * with the block zero extended to fill it out. When writing to + * venti, the block will be zero truncated. The walker will also check + * that the block fits within psize or dsize as the case may be. + */ + +struct Cache +{ + VtLock *lk; + VtSession *z; + u32int now; /* ticks for usage timestamps */ + int size; /* max. size of any block; allocated to each block */ + Lump **heads; /* hash table for finding address */ + int nheap; /* number of available victims */ + Lump **heap; /* heap for locating victims */ + long nblocks; /* number of blocks allocated */ + Lump *blocks; /* array of block descriptors */ + u8int *mem; /* memory for all block descriptors */ + Lump *free; /* free list of lumps */ + + long hashSize; +}; + +/* + * the tag for a block is hash(index, parent tag) + */ + +struct Label { + uchar gen[4]; + uchar state; + uchar type; /* top bit indicates it is part of a directory */ + uchar tag[4]; /* tag of file it is in */ +}; + + +static char ENoDir[] = "directory entry is not allocated"; + +static void fixHeap(int si, Lump *b); +static int upHeap(int i, Lump *b); +static int downHeap(int i, Lump *b); +static char *lumpState(int); +static void lumpSetState(Lump *u, int state); + +Cache * +cacheAlloc(VtSession *z, int blockSize, long nblocks) +{ + int i; + Cache *c; + Lump *b; + + c = vtMemAllocZ(sizeof(Cache)); + + c->lk = vtLockAlloc(); + c->z = z; + c->size = blockSize; + c->nblocks = nblocks; + c->hashSize = nblocks; + c->heads = vtMemAllocZ(c->hashSize*sizeof(Lump*)); + c->heap = vtMemAllocZ(nblocks*sizeof(Lump*)); + c->blocks = vtMemAllocZ(nblocks*sizeof(Lump)); + c->mem = vtMemAllocZ(nblocks * blockSize); + for(i = 0; i < nblocks; i++){ + b = &c->blocks[i]; + b->lk = vtLockAlloc(); + b->c = c; + b->data = &c->mem[i * blockSize]; + b->addr = i+1; + b->state = LumpFree; + b->heap = BadHeap; + b->next = c->free; + c->free = b; + } + c->nheap = 0; + + return c; +} + +long +cacheGetSize(Cache *c) +{ + return c->nblocks; +} + +int +cacheGetBlockSize(Cache *c) +{ + return c->size; +} + +int +cacheSetSize(Cache *c, long nblocks) +{ + USED(c); + USED(nblocks); + return 0; +} + +void +cacheFree(Cache *c) +{ + int i; + + for(i = 0; i < c->nblocks; i++){ + assert(c->blocks[i].ref == 0); + vtLockFree(c->blocks[i].lk); + } + vtMemFree(c->heads); + vtMemFree(c->blocks); + vtMemFree(c->mem); + vtMemFree(c); +} + +static u32int +hash(Cache *c, uchar score[VtScoreSize], int type) +{ + u32int h; + uchar *p = score + VtScoreSize-4; + + h = (p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]; + h += type; + return h % c->hashSize; +} + +static void +findLump(Cache *c, Lump *bb) +{ + Lump *b, *last; + int h; + + last = nil; + h = hash(c, bb->score, bb->type); + for(b = c->heads[h]; b != nil; b = b->next){ + if(last != b->prev) + vtFatal("bad prev link"); + if(b == bb) + return; + last = b; + } + vtFatal("block missing from hash table"); +} + +void +cacheCheck(Cache *c) +{ + u32int size, now; + int i, k, refed, free; + static uchar zero[VtScoreSize]; + Lump *p; + + size = c->size; + now = c->now; + + free = 0; + for(p=c->free; p; p=p->next) + free++; + for(i = 0; i < c->nheap; i++){ + if(c->heap[i]->heap != i) + vtFatal("mis-heaped at %d: %d", i, c->heap[i]->heap); + if(i > 0 && c->heap[(i - 1) >> 1]->used2 - now > c->heap[i]->used2 - now) + vtFatal("bad heap ordering"); + k = (i << 1) + 1; + if(k < c->nheap && c->heap[i]->used2 - now > c->heap[k]->used2 - now) + vtFatal("bad heap ordering"); + k++; + if(k < c->nheap && c->heap[i]->used2 - now > c->heap[k]->used2 - now) + vtFatal("bad heap ordering"); + } + + refed = 0; + for(i = 0; i < c->nblocks; i++){ + if(c->blocks[i].data != &c->mem[i * size]) + vtFatal("mis-blocked at %d", i); + if(c->blocks[i].ref && c->blocks[i].heap == BadHeap){ + refed++; + } + if(memcmp(zero, c->blocks[i].score, VtScoreSize)) + findLump(c, &c->blocks[i]); + } +if(refed > 0)fprint(2, "cacheCheck: nheap %d refed %d free %d\n", c->nheap, refed, free); + assert(c->nheap + refed + free == c->nblocks); + refed = 0; + for(i = 0; i < c->nblocks; i++){ + if(c->blocks[i].ref) { +if(1)fprint(2, "%d %V %d %s\n", c->blocks[i].type, c->blocks[i].score, c->blocks[i].ref, lumpState(c->blocks[i].state)); + refed++; + } + } +if(refed > 0)fprint(2, "cacheCheck: in used %d\n", refed); +} + +/* + * delete an arbitrary block from the heap + */ +static void +delHeap(Lump *db) +{ + fixHeap(db->heap, db->c->heap[--db->c->nheap]); + db->heap = BadHeap; +} + +static void +fixHeap(int si, Lump *b) +{ + int i; + + i = upHeap(si, b); + if(i == si) + downHeap(i, b); +} + +static int +upHeap(int i, Lump *b) +{ + Lump *bb; + u32int now; + int p; + Cache *c; + + c = b->c; + now = c->now; + for(; i != 0; i = p){ + p = (i - 1) >> 1; + bb = c->heap[p]; + if(b->used2 - now >= bb->used2 - now) + break; + c->heap[i] = bb; + bb->heap = i; + } + c->heap[i] = b; + b->heap = i; + + return i; +} + +static int +downHeap(int i, Lump *b) +{ + Lump *bb; + u32int now; + int k; + Cache *c; + + c = b->c; + now = c->now; + for(; ; i = k){ + k = (i << 1) + 1; + if(k >= c->nheap) + break; + if(k + 1 < c->nheap && c->heap[k]->used2 - now > c->heap[k + 1]->used2 - now) + k++; + bb = c->heap[k]; + if(b->used2 - now <= bb->used2 - now) + break; + c->heap[i] = bb; + bb->heap = i; + } + c->heap[i] = b; + b->heap = i; + return i; +} + + +/* called with c->lk held */ +Lump * +cacheBumpLump(Cache *c) +{ + Lump *b; + + /* + * missed: locate the block with the oldest second to last use. + * remove it from the heap, and fix up the heap. + */ + if(c->free) { + b = c->free; + c->free = b->next; + } else { + for(;;){ + if(c->nheap == 0) { + cacheCheck(c); + assert(0); + return nil; + } + b = c->heap[0]; + delHeap(b); + if(b->ref == 0) + break; + } + + /* + * unchain the block from hash chain + */ + if(b->prev == nil) + c->heads[hash(c, b->score, b->type)] = b->next; + else + b->prev->next = b->next; + if(b->next != nil) + b->next->prev = b->prev; + + } + + /* + * the new block has no last use, so assume it happens sometime in the middle + */ + b->used = (b->used2 + c->now) / 2; + b->asize = 0; + + return b; +} + +Lump * +cacheAllocLump(Cache *c, int type, int size, int dir) +{ + Lump *b; + ulong h; + + assert(size <= c->size); + +again: + vtLock(c->lk); + b = cacheBumpLump(c); + if(b == nil) { + vtUnlock(c->lk); +fprint(2, "cache is full\n"); + /* XXX should be better */ + sleep(100); + goto again; + } + + vtLock(b->lk); + + assert(b->ref == 0); + b->ref++; + b->used2 = b->used; + b->used = c->now++; + + /* convert addr into score */ + memset(b->score, 0, VtScoreSize-4); + b->score[VtScoreSize-4] = b->addr>>24; + b->score[VtScoreSize-3] = b->addr>>16; + b->score[VtScoreSize-2] = b->addr>>8; + b->score[VtScoreSize-1] = b->addr; + + b->dir = dir; + b->type = type; + b->gen = 0; + b->asize = size; + b->state = LumpFree; + + h = hash(c, b->score, b->type); + + /* chain onto correct hash */ + b->next = c->heads[h]; + c->heads[h] = b; + if(b->next != nil) + b->next->prev = b; + b->prev = nil; + + vtUnlock(c->lk); + + vtZeroExtend(type, b->data, 0, size); + lumpSetState(b, LumpActive); + + return b; +} + +int +scoreIsLocal(uchar score[VtScoreSize]) +{ + static uchar zero[VtScoreSize]; + + return memcmp(score, zero, VtScoreSize-4) == 0; +} + +Lump * +cacheGetLump(Cache *c, uchar score[VtScoreSize], int type, int size) +{ + Lump *b; + ulong h; + int n; + static uchar zero[VtScoreSize]; + + assert(size <= c->size); + + h = hash(c, score, type); + +again: + /* + * look for the block in the cache + */ + vtLock(c->lk); + for(b = c->heads[h]; b != nil; b = b->next){ + if(memcmp(b->score, score, VtScoreSize) == 0 && b->type == type) + goto found; + } + + /* should not be looking for a temp block */ + if(scoreIsLocal(score)) { + if(memcmp(score, zero, VtScoreSize) == 0) + vtSetError("looking for zero score"); + else + vtSetError("missing local block"); + vtUnlock(c->lk); + return nil; + } + + b = cacheBumpLump(c); + if(b == nil) { + vtUnlock(c->lk); + sleep(100); + goto again; + } + + /* chain onto correct hash */ + b->next = c->heads[h]; + c->heads[h] = b; + if(b->next != nil) + b->next->prev = b; + b->prev = nil; + + memmove(b->score, score, VtScoreSize); + b->type = type; + b->state = LumpFree; + +found: + b->ref++; + b->used2 = b->used; + b->used = c->now++; + if(b->heap != BadHeap) + fixHeap(b->heap, b); + + vtUnlock(c->lk); + + vtLock(b->lk); + if(b->state != LumpFree) + return b; + + n = vtRead(c->z, score, type, b->data, size); + if(n < 0) { + lumpDecRef(b, 1); + return nil; + } + if(!vtSha1Check(score, b->data, n)) { + vtSetError("vtSha1Check failed"); + lumpDecRef(b, 1); + return nil; + } + vtZeroExtend(type, b->data, n, size); + b->asize = size; + lumpSetState(b, LumpVenti); + + return b; +} + +static char * +lumpState(int state) +{ + switch(state) { + default: + return "Unknown!!"; + case LumpFree: + return "Free"; + case LumpActive: + return "Active"; + case LumpSnap: + return "Snap"; + case LumpZombie: + return "Zombie"; + case LumpVenti: + return "Venti"; + } +} + +static void +lumpSetState(Lump *u, int state) +{ +// if(u->state != LumpFree) +// fprint(2, "%V: %s -> %s\n", u->score, lumpState(u->state), lumpState(state)); + u->state = state; +} + +int +lumpGetScore(Lump *u, int offset, uchar score[VtScoreSize]) +{ + uchar *sp; + VtRoot root; + VtEntry dir; + + vtLock(u->lk); + + switch(u->type) { + default: + vtSetError("bad type"); + goto Err; + case VtPointerType0: + case VtPointerType1: + case VtPointerType2: + case VtPointerType3: + case VtPointerType4: + case VtPointerType5: + case VtPointerType6: + if((offset+1)*VtScoreSize > u->asize) + sp = nil; + else + sp = u->data + offset*VtScoreSize; + break; + case VtRootType: + if(u->asize < VtRootSize) { + vtSetError("runt root block"); + goto Err; + } + if(!vtRootUnpack(&root, u->data)) + goto Err; + sp = root.score; + break; + case VtDirType: + if((offset+1)*VtEntrySize > u->asize) { + vtSetError(ENoDir); + goto Err; + } + if(!vtEntryUnpack(&dir, u->data, offset)) + goto Err; + if(!dir.flags & VtEntryActive) { + vtSetError(ENoDir); + goto Err; + } + sp = dir.score; + break; + } + + if(sp == nil) + memmove(score, vtZeroScore, VtScoreSize); + else + memmove(score, sp, VtScoreSize); + + vtUnlock(u->lk); + return !scoreIsLocal(score); +Err: + vtUnlock(u->lk); + return 0; +} + +Lump * +lumpWalk(Lump *u, int offset, int type, int size, int readOnly, int lock) +{ + Lump *v, *vv; + Cache *c; + uchar score[VtScoreSize], *sp; + VtRoot root; + VtEntry dir; + int split, isdir; + + c = u->c; + vtLock(u->lk); + +Again: + v = nil; + vv = nil; + + isdir = u->dir; + switch(u->type) { + default: + vtSetError("bad type"); + goto Err; + case VtPointerType0: + case VtPointerType1: + case VtPointerType2: + case VtPointerType3: + case VtPointerType4: + case VtPointerType5: + case VtPointerType6: + if((offset+1)*VtScoreSize > u->asize) + sp = nil; + else + sp = u->data + offset*VtScoreSize; + break; + case VtRootType: + if(u->asize < VtRootSize) { + vtSetError("runt root block"); + goto Err; + } + if(!vtRootUnpack(&root, u->data)) + goto Err; + sp = root.score; + break; + case VtDirType: + if((offset+1)*VtEntrySize > u->asize) { + vtSetError(ENoDir); + goto Err; + } + if(!vtEntryUnpack(&dir, u->data, offset)) + goto Err; + if(!(dir.flags & VtEntryActive)) { + vtSetError(ENoDir); + goto Err; + } + isdir = (dir.flags & VtEntryDir) != 0; +// sp = dir.score; + sp = u->data + offset*VtEntrySize + 20; + break; + } + + if(sp == nil) + memmove(score, vtZeroScore, VtScoreSize); + else + memmove(score, sp, VtScoreSize); + + vtUnlock(u->lk); + + +if(0)fprint(2, "lumpWalk: %V:%s %d:%d-> %V:%d\n", u->score, lumpState(u->state), u->type, offset, score, type); + v = cacheGetLump(c, score, type, size); + if(v == nil) + return nil; + + split = 1; + if(readOnly) + split = 0; + + switch(v->state) { + default: + assert(0); + case LumpFree: +fprint(2, "block is free %V!\n", v->score); + vtSetError("phase error"); + goto Err2; + case LumpActive: + if(v->gen < u->gen) { +print("LumpActive gen\n"); + lumpSetState(v, LumpSnap); + v->gen = u->gen; + } else + split = 0; + break; + case LumpSnap: + case LumpVenti: + break; + } + + /* easy case */ + if(!split) { + if(!lock) + vtUnlock(v->lk); + return v; + } + + if(sp == nil) { + vtSetError("bad offset"); + goto Err2; + } + + vv = cacheAllocLump(c, v->type, size, isdir); + /* vv is locked */ + vv->gen = u->gen; + memmove(vv->data, v->data, v->asize); +if(0)fprint(2, "split %V into %V\n", v->score, vv->score); + + lumpDecRef(v, 1); + v = nil; + + vtLock(u->lk); + if(u->state != LumpActive) { + vtSetError("bad parent state: can not happen"); + goto Err; + } + + /* check that nothing changed underfoot */ + if(memcmp(sp, score, VtScoreSize) != 0) { + lumpDecRef(vv, 1); +fprint(2, "lumpWalk: parent changed under foot\n"); + goto Again; + } + + /* XXX - hold Active blocks up - will go eventually */ + lumpIncRef(vv); + + /* change the parent */ + memmove(sp, vv->score, VtScoreSize); + + vtUnlock(u->lk); + + if(!lock) + vtUnlock(vv->lk); + return vv; +Err: + vtUnlock(u->lk); + lumpDecRef(v, 0); + lumpDecRef(vv, 1); + return nil; +Err2: + lumpDecRef(v, 1); + return nil; + +} + +void +lumpFreeEntry(Lump *u, int entry) +{ + uchar score[VtScoreSize]; + int type; + ulong gen; + VtEntry dir; + Cache *c; + + c = u->c; + vtLock(u->lk); + if(u->state == LumpVenti) + goto Exit; + + switch(u->type) { + default: + fprint(2, "freeing bad lump type: %d\n", u->type); + return; + case VtPointerType0: + if((entry+1)*VtScoreSize > u->asize) + goto Exit; + memmove(score, u->data + entry*VtScoreSize, VtScoreSize); + memmove(u->data + entry*VtScoreSize, vtZeroScore, VtScoreSize); + type = u->dir?VtDirType:VtDataType; + break; + case VtPointerType1: + case VtPointerType2: + case VtPointerType3: + case VtPointerType4: + case VtPointerType5: + case VtPointerType6: + if((entry+1)*VtScoreSize > u->asize) + goto Exit; + memmove(score, u->data + entry*VtScoreSize, VtScoreSize); + memmove(u->data + entry*VtScoreSize, vtZeroScore, VtScoreSize); + type = u->type-1; + break; + case VtDirType: + if((entry+1)*VtEntrySize > u->asize) + goto Exit; + if(!vtEntryUnpack(&dir, u->data, entry)) + goto Exit; + if(!dir.flags & VtEntryActive) + goto Exit; + gen = dir.gen; + if(gen != ~0) + gen++; + if(dir.depth == 0) + type = (dir.flags&VtEntryDir)?VtDirType:VtDataType; + else + type = VtPointerType0 + dir.depth - 1; + memmove(score, dir.score, VtScoreSize); + memset(&dir, 0, sizeof(dir)); + dir.gen = gen; + vtEntryPack(&dir, u->data, entry); + break; + case VtDataType: + type = VtErrType; + break; + } + vtUnlock(u->lk); + if(type == VtErrType || !scoreIsLocal(score)) + return; + + u = cacheGetLump(c, score, type, c->size); + if(u == nil) + return; + lumpDecRef(u, 1); + /* XXX remove extra reference */ + lumpDecRef(u, 0); + return; +Exit: + vtUnlock(u->lk); + return; + +} + +void +lumpCleanup(Lump *u) +{ + int i, n; + + switch(u->type) { + default: + return; + case VtPointerType0: + case VtPointerType1: + case VtPointerType2: + case VtPointerType3: + case VtPointerType4: + case VtPointerType5: + case VtPointerType6: + n = u->asize/VtScoreSize; + break; + case VtDirType: + n = u->asize/VtEntrySize; + break; + } + + for(i=0; i<n; i++) + lumpFreeEntry(u, i); +} + + +void +lumpDecRef(Lump *b, int unlock) +{ + int i; + Cache *c; + + if(b == nil) + return; + + if(unlock) + vtUnlock(b->lk); + + c = b->c; + vtLock(c->lk); + if(--b->ref > 0) { + vtUnlock(c->lk); + return; + } + assert(b->ref == 0); + + switch(b->state) { + default: + fprint(2, "bad state: %s\n", lumpState(b->state)); + assert(0); + case LumpActive: + /* hack - but will do for now */ + b->ref++; + vtUnlock(c->lk); + lumpCleanup(b); + vtLock(c->lk); + b->ref--; + lumpSetState(b, LumpFree); + break; + case LumpZombie: + lumpSetState(b, LumpFree); + break; + case LumpFree: + case LumpVenti: + break; + } + + /* + * reinsert in the free heap + */ + if(b->heap == BadHeap) { + i = upHeap(c->nheap++, b); + c->heap[i] = b; + b->heap = i; + } + + vtUnlock(c->lk); +} + +Lump * +lumpIncRef(Lump *b) +{ + Cache *c; + + c = b->c; + + vtLock(c->lk); + assert(b->ref > 0); + b->ref++; + vtUnlock(c->lk); + return b; +} diff --git a/src/cmd/vac/dat.h b/src/cmd/vac/dat.h new file mode 100644 index 00000000..a468668d --- /dev/null +++ b/src/cmd/vac/dat.h @@ -0,0 +1,156 @@ +typedef struct Source Source; +typedef struct VacFile VacFile; +typedef struct MetaBlock MetaBlock; +typedef struct MetaEntry MetaEntry; +typedef struct Lump Lump; +typedef struct Cache Cache; +typedef struct Super Super; + +enum { + NilBlock = (~0UL), + MaxBlock = (1UL<<31), +}; + + +struct VacFS { + int ref; + + /* need a read write lock? */ + + uchar score[VtScoreSize]; + VacFile *root; + + VtSession *z; + int readOnly; + int bsize; /* maximum block size */ + uvlong qid; /* next qid */ + Cache *cache; +}; + + +struct Source { + VtLock *lk; + + Cache *cache; /* immutable */ + int readOnly; /* immutable */ + + Lump *lump; /* lump containing venti dir entry */ + ulong block; /* block number within parent: immutable */ + int entry; /* which entry in the block: immutable */ + + /* most of a VtEntry, except the score */ + ulong gen; /* generation: immutable */ + int dir; /* dir flags: immutable */ + int depth; /* number of levels of pointer blocks */ + int psize; /* pointer block size: immutable */ + int dsize; /* data block size: immutable */ + uvlong size; /* size in bytes of file */ + + int epb; /* dir entries per block = dize/VtEntrySize: immutable */ +}; + +struct MetaEntry { + uchar *p; + ushort size; +}; + +struct MetaBlock { + int maxsize; /* size of block */ + int size; /* size used */ + int free; /* free space within used size */ + int maxindex; /* entries allocated for table */ + int nindex; /* amount of table used */ + int unbotch; + uchar *buf; +}; + +/* + * contains a one block buffer + * to avoid problems of the block changing underfoot + * and to enable an interface that supports unget. + */ +struct VacDirEnum { + VacFile *file; + + ulong block; /* current block */ + MetaBlock mb; /* parsed version of block */ + int index; /* index in block */ +}; + +/* Lump states */ +enum { + LumpFree, + LumpVenti, /* on venti server: score > 2^32: just a cached copy */ + LumpActive, /* active */ + LumpActiveRO, /* active: read only block */ + LumpActiveA, /* active: achrived */ + LumpSnap, /* snapshot: */ + LumpSnapRO, /* snapshot: read only */ + LumpSnapA, /* snapshot: achived */ + LumpZombie, /* block with no pointer to it: waiting to be freed */ + + LumpMax +}; + +/* + * Each lump has a state and generation + * The following invariants are maintained + * Each lump has no more than than one parent per generation + * For Active*, no child has a parent of a greater generation + * For Snap*, there is a snap parent of given generation and there are + * no parents of greater gen - implies no children of a greater gen + * For *RO, the lump is fixed - no change ca be made - all pointers + * are valid venti addresses + * For *A, the lump is on the venti server + * There are no pointers to Zombie lumps + * + * Transitions + * Archiver at generation g + * Mutator at generation h + * + * Want to modify a lump + * Venti: create new Active(h) + * Active(x): x == h: do nothing + * Acitve(x): x < h: change to Snap(h-1) + add Active(h) + * ActiveRO(x): change to SnapRO(h-1) + add Active(h) + * ActiveA(x): add Active(h) + * Snap*(x): should not occur + * Zombie(x): should not occur + * Want to archive + * Active(x): x != g: should never happen + * Active(x): x == g fix children and free them: move to ActoveRO(g); + * ActiveRO(x): x != g: should never happen + * ActiveRO(x): x == g: wait until it hits ActiveA or SnapA + * ActiveA(x): done + * Active(x): x < g: should never happen + * Snap(x): x >= g: fix children, freeing all SnapA(y) x == y; + * SnapRO(x): wait until it hits SnapA + * + */ + + +struct Lump { + int ref; + + Cache *c; + + VtLock *lk; + + int state; + ulong gen; + + uchar *data; + uchar score[VtScoreSize]; /* score of packet */ + uchar vscore[VtScoreSize]; /* venti score - when archived */ + u8int type; /* type of packet */ + int dir; /* part of a directory - extension of type */ + u16int asize; /* allocated size of block */ + Lump *next; /* doubly linked hash chains */ + Lump *prev; + u32int heap; /* index in heap table */ + u32int used; /* last reference times */ + u32int used2; + + u32int addr; /* mutable block address */ +}; + diff --git a/src/cmd/vac/error.c b/src/cmd/vac/error.c new file mode 100644 index 00000000..5c8ff29d --- /dev/null +++ b/src/cmd/vac/error.c @@ -0,0 +1,20 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" +#include "error.h" + +char ENoDir[] = "directory entry is not allocated"; +char EBadDir[] = "corrupted directory entry"; +char EBadMeta[] = "corrupted meta data"; +char ENotDir[] = "not a directory"; +char ENotFile[] = "not a file"; +char EIO[] = "i/o error"; +char EBadOffset[] = "illegal offset"; +char ETooBig[] = "file too big"; +char EReadOnly[] = "read only"; +char ERemoved[] = "file has been removed"; +char ENilBlock[] = "illegal block address"; +char ENotEmpty[] = "directory not empty"; +char EExists[] = "file already exists"; +char ERoot[] = "cannot remove root"; diff --git a/src/cmd/vac/error.h b/src/cmd/vac/error.h new file mode 100644 index 00000000..742228cf --- /dev/null +++ b/src/cmd/vac/error.h @@ -0,0 +1,14 @@ +extern char ENoDir[]; +extern char EBadDir[]; +extern char EBadMeta[]; +extern char ENilBlock[]; +extern char ENotDir[]; +extern char ENotFile[]; +extern char EIO[]; +extern char EBadOffset[]; +extern char ETooBig[]; +extern char EReadOnly[]; +extern char ERemoved[]; +extern char ENotEmpty[]; +extern char EExists[]; +extern char ERoot[]; diff --git a/src/cmd/vac/file.c b/src/cmd/vac/file.c new file mode 100644 index 00000000..900422d2 --- /dev/null +++ b/src/cmd/vac/file.c @@ -0,0 +1,1214 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" +#include "error.h" + +/* + * locking order is upwards. A thread can hold the lock for a VacFile + * and then acquire the lock of its parent + */ + +struct VacFile { + /* meta data for file: protected by the lk in the parent */ + int ref; /* holds this data structure up */ + VacFS *fs; /* immutable */ + + int removed; /* file has been removed */ + int dirty; /* dir is dirty with respect to meta data in block */ + ulong block; /* block offset withing msource for this file's meta data */ + + VacDir dir; /* meta data for this file */ + + VacFile *up; /* parent file */ + VacFile *next; /* sibling */ + + /* data for file */ + VtLock *lk; /* lock for source and msource */ + Source *source; + Source *msource; /* for directories: meta data for children */ + VacFile *down; /* children */ +}; + +static int vfMetaFlush(VacFile*); +static ulong msAlloc(Source *ms, ulong, int n); + +static void +vfRUnlock(VacFile *vf) +{ + vtRUnlock(vf->lk); +} + + +static int +vfRLock(VacFile *vf) +{ + vtRLock(vf->lk); + if(vf->source == nil) { + vfRUnlock(vf); + vtSetError(ERemoved); + return 0; + } + return 1; +} + +static void +vfUnlock(VacFile *vf) +{ + vtUnlock(vf->lk); +} + +static int +vfLock(VacFile *vf) +{ + vtLock(vf->lk); + if(vf->source == nil) { + vfUnlock(vf); + vtSetError(ERemoved); + return 0; + } + return 1; +} + +static void +vfMetaLock(VacFile *vf) +{ + assert(vf->up->msource != nil); + vtLock(vf->up->lk); +} + +static void +vfMetaUnlock(VacFile *vf) +{ + vtUnlock(vf->up->lk); +} + + +static void +vfRAccess(VacFile* vf) +{ + vfMetaLock(vf); + vf->dir.atime = time(0L); + vf->dirty = 1; + vfMetaUnlock(vf); + vfMetaFlush(vf); +} + +static void +vfWAccess(VacFile* vf, char *mid) +{ + vfMetaLock(vf); + vf->dir.atime = vf->dir.mtime = time(0L); + if(strcmp(vf->dir.mid, mid) != 0) { + vtMemFree(vf->dir.mid); + vf->dir.mid = vtStrDup(mid); + } + vf->dir.mcount++; + vf->dirty = 1; + vfMetaUnlock(vf); + vfMetaFlush(vf); +} + +void +vdCleanup(VacDir *dir) +{ + vtMemFree(dir->elem); + dir->elem = nil; + vtMemFree(dir->uid); + dir->uid = nil; + vtMemFree(dir->gid); + dir->gid = nil; + vtMemFree(dir->mid); + dir->mid = nil; +} + +void +vdCopy(VacDir *dst, VacDir *src) +{ + *dst = *src; + dst->elem = vtStrDup(src->elem); + dst->uid = vtStrDup(src->uid); + dst->gid = vtStrDup(src->gid); + dst->mid = vtStrDup(src->mid); +} + +static int +mbSearch(MetaBlock *mb, char *elem, int *ri, MetaEntry *me) +{ + int i; + int b, t, x; + + /* binary search within block */ + b = 0; + t = mb->nindex; + while(b < t) { + i = (b+t)>>1; + if(!meUnpack(me, mb, i)) + return 0; + if(mb->unbotch) + x = meCmpNew(me, elem); + else + x = meCmp(me, elem); + + if(x == 0) { + *ri = i; + return 1; + } + + if(x < 0) + b = i+1; + else /* x > 0 */ + t = i; + } + + assert(b == t); + + *ri = b; /* b is the index to insert this entry */ + memset(me, 0, sizeof(*me)); + + return 1; +} + +static void +mbInit(MetaBlock *mb, uchar *p, int n) +{ + memset(mb, 0, sizeof(MetaBlock)); + mb->maxsize = n; + mb->buf = p; + mb->maxindex = n/100; + mb->size = MetaHeaderSize + mb->maxindex*MetaIndexSize; +} + +static int +vfMetaFlush(VacFile *vf) +{ + VacFile *vfp; + Lump *u; + MetaBlock mb; + MetaEntry me, nme; + uchar *p; + int i, n, moved; + +//print("vfMetaFlush %s\n", vf->dir.elem); + + /* assume name has not changed for the moment */ + + vfMetaLock(vf); + + vfp = vf->up; + moved = 0; + + u = sourceGetLump(vfp->msource, vf->block, 0, 1); + if(u == nil) + goto Err; + + if(!mbUnpack(&mb, u->data, u->asize)) + goto Err; + if(!mbSearch(&mb, vf->dir.elem, &i, &me) || me.p == nil) + goto Err; + + nme = me; + n = vdSize(&vf->dir); +//print("old size %d new size %d\n", me.size, n); + if(n <= nme.size) { + nme.size = n; + } else { + /* try expand entry? */ + p = mbAlloc(&mb, n); +//print("alloced %ld\n", p - mb.buf); + if(p == nil) { +assert(0); + /* much more work */ + } + nme.p = p; + nme.size = n; + } + + mbDelete(&mb, i, &me); + memset(me.p, 0, me.size); + if(!moved) { + vdPack(&vf->dir, &nme); + mbInsert(&mb, i, &nme); + } + + mbPack(&mb); + lumpDecRef(u, 1); + + vf->dirty = 0; + + vfMetaUnlock(vf); + return 1; + +Err: + lumpDecRef(u, 1); + vfMetaUnlock(vf); + return 0; +} + +static VacFile * +vfAlloc(VacFS *fs) +{ + VacFile *vf; + + vf = vtMemAllocZ(sizeof(VacFile)); + vf->lk = vtLockAlloc(); + vf->ref = 1; + vf->fs = fs; + return vf; +} + +static void +vfFree(VacFile *vf) +{ + sourceFree(vf->source); + vtLockFree(vf->lk); + sourceFree(vf->msource); + vdCleanup(&vf->dir); + + vtMemFree(vf); +} + +/* the file is locked already */ +static VacFile * +dirLookup(VacFile *vf, char *elem) +{ + int i, j, nb; + MetaBlock mb; + MetaEntry me; + Lump *u; + Source *meta; + VacFile *nvf; + + meta = vf->msource; + u = nil; + nb = sourceGetNumBlocks(meta); + for(i=0; i<nb; i++) { + u = sourceGetLump(meta, i, 1, 1); + if(u == nil) + goto Err; + if(!mbUnpack(&mb, u->data, u->asize)) + goto Err; + if(!mbSearch(&mb, elem, &j, &me)) + goto Err; + if(me.p != nil) { + nvf = vfAlloc(vf->fs); + if(!vdUnpack(&nvf->dir, &me)) { + vfFree(nvf); + goto Err; + } + lumpDecRef(u, 1); + nvf->block = i; + return nvf; + } + + lumpDecRef(u, 1); + u = nil; + } + vtSetError("file does not exist"); + /* fall through */ +Err: + lumpDecRef(u, 1); + return nil; +} + +VacFile * +vfRoot(VacFS *fs, uchar *score) +{ + VtEntry e; + Lump *u, *v; + Source *r, *r0, *r1, *r2; + MetaBlock mb; + MetaEntry me; + VacFile *root, *mr; + + root = nil; + mr = nil; + r0 = nil; + r1 = nil; + r2 = nil; + v = nil; + r = nil; + + u = cacheGetLump(fs->cache, score, VtDirType, fs->bsize); + if(u == nil) + goto Err; + if(!fs->readOnly) { + v = cacheAllocLump(fs->cache, VtDirType, fs->bsize, 1); + if(v == nil) { + vtUnlock(u->lk); + goto Err; + } + v->gen = u->gen; + v->asize = u->asize; + v->state = LumpActive; + memmove(v->data, u->data, v->asize); + lumpDecRef(u, 1); + u = v; + v = nil; + } + vtUnlock(u->lk); + vtEntryUnpack(&e, u->data, 2); + if(e.flags == 0){ /* just one entry */ + r = sourceAlloc(fs->cache, u, 0, 0, fs->readOnly); + if(r == nil) + goto Err; + r0 = sourceOpen(r, 0, fs->readOnly); + if(r0 == nil) + goto Err; + r1 = sourceOpen(r, 1, fs->readOnly); + if(r1 == nil) + goto Err; + r2 = sourceOpen(r, 2, fs->readOnly); + if(r2 == nil) + goto Err; + sourceFree(r); + r = nil; + }else{ + r0 = sourceAlloc(fs->cache, u, 0, 0, fs->readOnly); + if(r0 == nil) + goto Err; + r1 = sourceAlloc(fs->cache, u, 0, 1, fs->readOnly); + if(r1 == nil) + goto Err; + r2 = sourceAlloc(fs->cache, u, 0, 2, fs->readOnly); + if(r2 == nil) + goto Err; + } + lumpDecRef(u, 0); + u = sourceGetLump(r2, 0, 1, 0); + if(u == nil) + goto Err; + + mr = vfAlloc(fs); + mr->msource = r2; + r2 = nil; + + root = vfAlloc(fs); + root->up = mr; + root->source = r0; + r0 = nil; + root->msource = r1; + r1 = nil; + + mr->down = root; + + if(!mbUnpack(&mb, u->data, u->asize)) + goto Err; + + if(!meUnpack(&me, &mb, 0)) + goto Err; + if(!vdUnpack(&root->dir, &me)) + goto Err; + + vfRAccess(root); + lumpDecRef(u, 0); + sourceFree(r2); + + return root; +Err: + lumpDecRef(u, 0); + lumpDecRef(v, 0); + if(r0) + sourceFree(r0); + if(r1) + sourceFree(r1); + if(r2) + sourceFree(r2); + if(r) + sourceFree(r); + if(mr) + vfFree(mr); + if(root) + vfFree(root); + + return nil; +} + +VacFile * +vfWalk(VacFile *vf, char *elem) +{ + VacFile *nvf; + + vfRAccess(vf); + + if(elem[0] == 0) { + vtSetError("illegal path element"); + return nil; + } + if(!vfIsDir(vf)) { + vtSetError("not a directory"); + return nil; + } + + if(strcmp(elem, ".") == 0) { + return vfIncRef(vf); + } + + if(strcmp(elem, "..") == 0) { + if(vfIsRoot(vf)) + return vfIncRef(vf); + return vfIncRef(vf->up); + } + + if(!vfLock(vf)) + return nil; + + for(nvf = vf->down; nvf; nvf=nvf->next) { + if(strcmp(elem, nvf->dir.elem) == 0 && !nvf->removed) { + nvf->ref++; + goto Exit; + } + } + + nvf = dirLookup(vf, elem); + if(nvf == nil) + goto Err; + nvf->source = sourceOpen(vf->source, nvf->dir.entry, vf->fs->readOnly); + if(nvf->source == nil) + goto Err; + if(nvf->dir.mode & ModeDir) { + nvf->msource = sourceOpen(vf->source, nvf->dir.mentry, vf->fs->readOnly); + if(nvf->msource == nil) + goto Err; + } + + /* link in and up parent ref count */ + nvf->next = vf->down; + vf->down = nvf; + nvf->up = vf; + vfIncRef(vf); +Exit: + vfUnlock(vf); + return nvf; +Err: + vfUnlock(vf); + if(nvf != nil) + vfFree(nvf); + return nil; +} + +VacFile * +vfOpen(VacFS *fs, char *path) +{ + VacFile *vf, *nvf; + char *p, elem[VtMaxStringSize]; + int n; + + vf = fs->root; + vfIncRef(vf); + while(*path != 0) { + for(p = path; *p && *p != '/'; p++) + ; + n = p - path; + if(n > 0) { + if(n > VtMaxStringSize) { + vtSetError("path element too long"); + goto Err; + } + memmove(elem, path, n); + elem[n] = 0; + nvf = vfWalk(vf, elem); + if(nvf == nil) + goto Err; + vfDecRef(vf); + vf = nvf; + } + if(*p == '/') + p++; + path = p; + } + return vf; +Err: + vfDecRef(vf); + return nil; +} + +VacFile * +vfCreate(VacFile *vf, char *elem, ulong mode, char *user) +{ + VacFile *nvf; + VacDir *dir; + int n, i; + uchar *p; + Source *pr, *r, *mr; + int isdir; + MetaBlock mb; + MetaEntry me; + Lump *u; + + if(!vfLock(vf)) + return nil; + + r = nil; + mr = nil; + u = nil; + + for(nvf = vf->down; nvf; nvf=nvf->next) { + if(strcmp(elem, nvf->dir.elem) == 0 && !nvf->removed) { + nvf = nil; + vtSetError(EExists); + goto Err; + } + } + + nvf = dirLookup(vf, elem); + if(nvf != nil) { + vtSetError(EExists); + goto Err; + } + + nvf = vfAlloc(vf->fs); + isdir = mode & ModeDir; + + pr = vf->source; + r = sourceCreate(pr, pr->psize, pr->dsize, isdir, 0); + if(r == nil) + goto Err; + if(isdir) { + mr = sourceCreate(pr, pr->psize, pr->dsize, 0, r->block*pr->epb + r->entry); + if(mr == nil) + goto Err; + } + + dir = &nvf->dir; + dir->elem = vtStrDup(elem); + dir->entry = r->block*pr->epb + r->entry; + dir->gen = r->gen; + if(isdir) { + dir->mentry = mr->block*pr->epb + mr->entry; + dir->mgen = mr->gen; + } + dir->size = 0; + dir->qid = vf->fs->qid++; + dir->uid = vtStrDup(user); + dir->gid = vtStrDup(vf->dir.gid); + dir->mid = vtStrDup(user); + dir->mtime = time(0L); + dir->mcount = 0; + dir->ctime = dir->mtime; + dir->atime = dir->mtime; + dir->mode = mode; + + n = vdSize(dir); + nvf->block = msAlloc(vf->msource, 0, n); + if(nvf->block == NilBlock) + goto Err; + u = sourceGetLump(vf->msource, nvf->block, 0, 1); + if(u == nil) + goto Err; + if(!mbUnpack(&mb, u->data, u->asize)) + goto Err; + p = mbAlloc(&mb, n); + if(p == nil) + goto Err; + + if(!mbSearch(&mb, elem, &i, &me)) + goto Err; + assert(me.p == nil); + me.p = p; + me.size = n; + + vdPack(dir, &me); + mbInsert(&mb, i, &me); + mbPack(&mb); + lumpDecRef(u, 1); + + nvf->source = r; + nvf->msource = mr; + + /* link in and up parent ref count */ + nvf->next = vf->down; + vf->down = nvf; + nvf->up = vf; + vfIncRef(vf); + + vfWAccess(vf, user); + + vfUnlock(vf); + return nvf; + +Err: + lumpDecRef(u, 1); + if(r) + sourceRemove(r); + if(mr) + sourceRemove(mr); + if(nvf) + vfFree(nvf); + vfUnlock(vf); + return 0; +} + + +int +vfRead(VacFile *vf, void *buf, int cnt, vlong offset) +{ + Source *s; + uvlong size; + ulong bn; + int off, dsize, n, nn; + Lump *u; + uchar *b; + +if(0)fprint(2, "vfRead: %s %d, %lld\n", vf->dir.elem, cnt, offset); + + if(!vfRLock(vf)) + return -1; + + s = vf->source; + + dsize = s->dsize; + size = sourceGetSize(s); + + if(offset < 0) { + vtSetError(EBadOffset); + goto Err; + } + + vfRAccess(vf); + + if(offset >= size) + offset = size; + + if(cnt > size-offset) + cnt = size-offset; + bn = offset/dsize; + off = offset%dsize; + b = buf; + while(cnt > 0) { + u = sourceGetLump(s, bn, 1, 0); + if(u == nil) + goto Err; + if(u->asize <= off) { + lumpDecRef(u, 0); + goto Err; + } + n = cnt; + if(n > dsize-off) + n = dsize-off; + nn = u->asize-off; + if(nn > n) + nn = n; + memmove(b, u->data+off, nn); + memset(b+nn, 0, n-nn); + off = 0; + bn++; + cnt -= n; + b += n; + lumpDecRef(u, 0); + } + vfRUnlock(vf); + return b-(uchar*)buf; +Err: + vfRUnlock(vf); + return -1; +} + +int +vfWrite(VacFile *vf, void *buf, int cnt, vlong offset, char *user) +{ + Source *s; + ulong bn; + int off, dsize, n; + Lump *u; + uchar *b; + + USED(user); + + if(!vfLock(vf)) + return -1; + + if(vf->fs->readOnly) { + vtSetError(EReadOnly); + goto Err; + } + + if(vf->dir.mode & ModeDir) { + vtSetError(ENotFile); + goto Err; + } +if(0)fprint(2, "vfWrite: %s %d, %lld\n", vf->dir.elem, cnt, offset); + + s = vf->source; + dsize = s->dsize; + + if(offset < 0) { + vtSetError(EBadOffset); + goto Err; + } + + vfWAccess(vf, user); + + bn = offset/dsize; + off = offset%dsize; + b = buf; + while(cnt > 0) { + n = cnt; + if(n > dsize-off) + n = dsize-off; + if(!sourceSetDepth(s, offset+n)) + goto Err; + u = sourceGetLump(s, bn, 0, 0); + if(u == nil) + goto Err; + if(u->asize < dsize) { + vtSetError("runt block"); + lumpDecRef(u, 0); + goto Err; + } + memmove(u->data+off, b, n); + off = 0; + cnt -= n; + b += n; + offset += n; + bn++; + lumpDecRef(u, 0); + if(!sourceSetSize(s, offset)) + goto Err; + } + vfLock(vf); + return b-(uchar*)buf; +Err: + vfLock(vf); + return -1; +} + +int +vfGetDir(VacFile *vf, VacDir *dir) +{ + if(!vfRLock(vf)) + return 0; + + vfMetaLock(vf); + vdCopy(dir, &vf->dir); + vfMetaUnlock(vf); + + if(!vfIsDir(vf)) + dir->size = sourceGetSize(vf->source); + vfRUnlock(vf); + + return 1; +} + +uvlong +vfGetId(VacFile *vf) +{ + /* immutable */ + return vf->dir.qid; +} + +ulong +vfGetMcount(VacFile *vf) +{ + ulong mcount; + + vfMetaLock(vf); + mcount = vf->dir.mcount; + vfMetaUnlock(vf); + return mcount; +} + + +int +vfIsDir(VacFile *vf) +{ + /* immutable */ + return (vf->dir.mode & ModeDir) != 0; +} + +int +vfIsRoot(VacFile *vf) +{ + return vf == vf->fs->root; +} + +int +vfGetSize(VacFile *vf, uvlong *size) +{ + if(!vfRLock(vf)) + return 0; + *size = sourceGetSize(vf->source); + vfRUnlock(vf); + + return 1; +} + +static int +vfMetaRemove(VacFile *vf, char *user) +{ + Lump *u; + MetaBlock mb; + MetaEntry me; + int i; + VacFile *vfp; + + vfp = vf->up; + + vfWAccess(vfp, user); + + vfMetaLock(vf); + + u = sourceGetLump(vfp->msource, vf->block, 0, 1); + if(u == nil) + goto Err; + + if(!mbUnpack(&mb, u->data, u->asize)) + goto Err; + if(!mbSearch(&mb, vf->dir.elem, &i, &me) || me.p == nil) + goto Err; +print("deleting %d entry\n", i); + mbDelete(&mb, i, &me); + memset(me.p, 0, me.size); + mbPack(&mb); + + lumpDecRef(u, 1); + + vf->removed = 1; + vf->block = NilBlock; + + vfMetaUnlock(vf); + return 1; + +Err: + lumpDecRef(u, 1); + vfMetaUnlock(vf); + return 0; +} + + +static int +vfCheckEmpty(VacFile *vf) +{ + int i, n; + Lump *u; + MetaBlock mb; + Source *r; + + r = vf->msource; + n = sourceGetNumBlocks(r); + for(i=0; i<n; i++) { + u = sourceGetLump(r, i, 1, 1); + if(u == nil) + goto Err; + if(!mbUnpack(&mb, u->data, u->asize)) + goto Err; + if(mb.nindex > 0) { + vtSetError(ENotEmpty); + goto Err; + } + lumpDecRef(u, 1); + } + return 1; +Err: + lumpDecRef(u, 1); + return 0; +} + +int +vfRemove(VacFile *vf, char *user) +{ + /* can not remove the root */ + if(vfIsRoot(vf)) { + vtSetError(ERoot); + return 0; + } + + if(!vfLock(vf)) + return 0; + + if(vfIsDir(vf) && !vfCheckEmpty(vf)) + goto Err; + + assert(vf->down == nil); + + sourceRemove(vf->source); + vf->source = nil; + if(vf->msource) { + sourceRemove(vf->msource); + vf->msource = nil; + } + + vfUnlock(vf); + + if(!vfMetaRemove(vf, user)) + return 0; + + return 1; + +Err: + vfUnlock(vf); + return 0; +} + +VacFile * +vfIncRef(VacFile *vf) +{ + vfMetaLock(vf); + assert(vf->ref > 0); + vf->ref++; + vfMetaUnlock(vf); + return vf; +} + +void +vfDecRef(VacFile *vf) +{ + VacFile *p, *q, **qq; + + if(vf->up == nil) { + vfFree(vf); + return; + } + + vfMetaLock(vf); + vf->ref--; + if(vf->ref > 0) { + vfMetaUnlock(vf); + return; + } + assert(vf->ref == 0); + assert(vf->down == nil); + + p = vf->up; + qq = &p->down; + for(q = *qq; q; qq=&q->next,q=*qq) + if(q == vf) + break; + assert(q != nil); + *qq = vf->next; + + vfMetaUnlock(vf); + vfFree(vf); + + vfDecRef(p); +} + +int +vfGetVtEntry(VacFile *vf, VtEntry *e) +{ + int res; + + if(!vfRLock(vf)) + return 0; + res = sourceGetVtEntry(vf->source, e); + vfRUnlock(vf); + return res; +} + +int +vfGetBlockScore(VacFile *vf, ulong bn, uchar score[VtScoreSize]) +{ + Lump *u; + int ret, off; + Source *r; + + if(!vfRLock(vf)) + return 0; + + r = vf->source; + + u = sourceWalk(r, bn, 1, &off); + if(u == nil){ + vfRUnlock(vf); + return 0; + } + + ret = lumpGetScore(u, off, score); + lumpDecRef(u, 0); + vfRUnlock(vf); + + return ret; +} + +VacFile * +vfGetParent(VacFile *vf) +{ + if(vfIsRoot(vf)) + return vfIncRef(vf); + return vfIncRef(vf->up); +} + +static VacDirEnum * +vdeAlloc(VacFile *vf) +{ + VacDirEnum *ds; + + if(!(vf->dir.mode & ModeDir)) { + vtSetError(ENotDir); + vfDecRef(vf); + return nil; + } + + ds = vtMemAllocZ(sizeof(VacDirEnum)); + ds->file = vf; + + return ds; +} + +VacDirEnum * +vdeOpen(VacFS *fs, char *path) +{ + VacFile *vf; + + vf = vfOpen(fs, path); + if(vf == nil) + return nil; + + return vdeAlloc(vf); +} + +VacDirEnum * +vfDirEnum(VacFile *vf) +{ + return vdeAlloc(vfIncRef(vf)); +} + +static int +dirEntrySize(Source *s, ulong elem, ulong gen, uvlong *size) +{ + Lump *u; + ulong bn; + VtEntry e; + + bn = elem/s->epb; + elem -= bn*s->epb; + + u = sourceGetLump(s, bn, 1, 1); + if(u == nil) + goto Err; + if(u->asize < (elem+1)*VtEntrySize) { + vtSetError(ENoDir); + goto Err; + } + vtEntryUnpack(&e, u->data, elem); + if(!(e.flags & VtEntryActive) || e.gen != gen) { +fprint(2, "gen mismatch\n"); + vtSetError(ENoDir); + goto Err; + } + + *size = e.size; + lumpDecRef(u, 1); + return 1; + +Err: + lumpDecRef(u, 1); + return 0; +} + +int +vdeRead(VacDirEnum *ds, VacDir *dir, int n) +{ + ulong nb; + int i; + Source *meta, *source; + MetaBlock mb; + MetaEntry me; + Lump *u; + + vfRAccess(ds->file); + + if(!vfRLock(ds->file)) + return -1; + + i = 0; + u = nil; + source = ds->file->source; + meta = ds->file->msource; + nb = (sourceGetSize(meta) + meta->dsize - 1)/meta->dsize; + + if(ds->block >= nb) + goto Exit; + u = sourceGetLump(meta, ds->block, 1, 1); + if(u == nil) + goto Err; + if(!mbUnpack(&mb, u->data, u->asize)) + goto Err; + + for(i=0; i<n; i++) { + while(ds->index >= mb.nindex) { + lumpDecRef(u, 1); + u = nil; + ds->index = 0; + ds->block++; + if(ds->block >= nb) + goto Exit; + u = sourceGetLump(meta, ds->block, 1, 1); + if(u == nil) + goto Err; + if(!mbUnpack(&mb, u->data, u->asize)) + goto Err; + } + if(!meUnpack(&me, &mb, ds->index)) + goto Err; + if(dir != nil) { + if(!vdUnpack(&dir[i], &me)) + goto Err; + if(!(dir[i].mode & ModeDir)) + if(!dirEntrySize(source, dir[i].entry, dir[i].gen, &dir[i].size)) + goto Err; + } + ds->index++; + } +Exit: + lumpDecRef(u, 1); + vfRUnlock(ds->file); + return i; +Err: + lumpDecRef(u, 1); + vfRUnlock(ds->file); + n = i; + for(i=0; i<n ; i++) + vdCleanup(&dir[i]); + return -1; +} + +void +vdeFree(VacDirEnum *ds) +{ + if(ds == nil) + return; + vfDecRef(ds->file); + vtMemFree(ds); +} + +static ulong +msAlloc(Source *ms, ulong start, int n) +{ + ulong nb, i; + Lump *u; + MetaBlock mb; + + nb = sourceGetNumBlocks(ms); + u = nil; + if(start > nb) + start = nb; + for(i=start; i<nb; i++) { + u = sourceGetLump(ms, i, 1, 1); + if(u == nil) + goto Err; + if(!mbUnpack(&mb, u->data, ms->dsize)) + goto Err; + if(mb.maxsize - mb.size + mb.free >= n && mb.nindex < mb.maxindex) + break; + lumpDecRef(u, 1); + u = nil; + } + /* add block to meta file */ + if(i == nb) { + if(!sourceSetDepth(ms, (i+1)*ms->dsize)) + goto Err; + u = sourceGetLump(ms, i, 0, 1); + if(u == nil) + goto Err; + sourceSetSize(ms, (nb+1)*ms->dsize); + mbInit(&mb, u->data, u->asize); + mbPack(&mb); + } + lumpDecRef(u, 1); + return i; +Err: + lumpDecRef(u, 1); + return NilBlock; +} + diff --git a/src/cmd/vac/fns.h b/src/cmd/vac/fns.h new file mode 100644 index 00000000..f013f126 --- /dev/null +++ b/src/cmd/vac/fns.h @@ -0,0 +1,46 @@ +Source *sourceAlloc(Cache*, Lump *u, ulong block, int elem, int readonly); +Source *sourceOpen(Source*, ulong entry, int readOnly); +Source *sourceCreate(Source*, int psize, int dsize, int isdir, ulong entry); +Lump *sourceGetLump(Source*, ulong block, int readOnly, int lock); +Lump *sourceWalk(Source *r, ulong block, int readOnly, int *); +int sourceSetDepth(Source *r, uvlong size); +int sourceSetSize(Source *r, uvlong size); +uvlong sourceGetSize(Source *r); +int sourceSetDirSize(Source *r, ulong size); +ulong sourceGetDirSize(Source *r); +void sourceRemove(Source*); +void sourceFree(Source*); +int sourceGetVtEntry(Source *r, VtEntry *dir); +ulong sourceGetNumBlocks(Source *r); + +Lump *lumpWalk(Lump *u, int offset, int type, int size, int readOnly, int lock); +int lumpGetScore(Lump *u, int offset, uchar score[VtScoreSize]); +void lumpDecRef(Lump*, int unlock); +Lump *lumpIncRef(Lump*); +void lumpFreeEntry(Lump *u, int entry); + +Cache *cacheAlloc(VtSession *z, int blockSize, long nblocks); +Lump *cacheAllocLump(Cache *c, int type, int size, int dir); +void cacheFree(Cache *c); +long cacheGetSize(Cache*); +int cacheSetSize(Cache*, long); +int cacheGetBlockSize(Cache *c); +Lump *cacheGetLump(Cache *c, uchar score[VtScoreSize], int type, int size); +void cacheCheck(Cache*); + +int mbUnpack(MetaBlock *mb, uchar *p, int n); +void mbInsert(MetaBlock *mb, int i, MetaEntry*); +void mbDelete(MetaBlock *mb, int i, MetaEntry*); +void mbPack(MetaBlock *mb); +uchar *mbAlloc(MetaBlock *mb, int n); + +int meUnpack(MetaEntry*, MetaBlock *mb, int i); +int meCmp(MetaEntry*, char *s); +int meCmpNew(MetaEntry*, char *s); + +int vdSize(VacDir *dir); +int vdUnpack(VacDir *dir, MetaEntry*); +void vdPack(VacDir *dir, MetaEntry*); + +VacFile *vfRoot(VacFS *fs, uchar *score); + diff --git a/src/cmd/vac/fs.c b/src/cmd/vac/fs.c new file mode 100644 index 00000000..568e2ece --- /dev/null +++ b/src/cmd/vac/fs.c @@ -0,0 +1,188 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" + +static char EBadVacFormat[] = "bad format for vac file"; + +static VacFS * +vfsAlloc(VtSession *z, int bsize, long ncache) +{ + VacFS *fs; + + fs = vtMemAllocZ(sizeof(VacFS)); + fs->ref = 1; + fs->z = z; + fs->bsize = bsize; + fs->cache = cacheAlloc(z, bsize, ncache); + return fs; +} + +static int +readScore(int fd, uchar score[VtScoreSize]) +{ + char buf[44]; + int i, n, c; + + n = readn(fd, buf, sizeof(buf)); + if(n < sizeof(buf)) { + vtSetError("short read"); + return 0; + } + if(strncmp(buf, "vac:", 4) != 0) { + vtSetError("not a vac file"); + return 0; + } + memset(score, 0, VtScoreSize); + for(i=4; i<sizeof(buf); i++) { + if(buf[i] >= '0' && buf[i] <= '9') + c = buf[i] - '0'; + else if(buf[i] >= 'a' && buf[i] <= 'f') + c = buf[i] - 'a' + 10; + else if(buf[i] >= 'A' && buf[i] <= 'F') + c = buf[i] - 'A' + 10; + else { + vtSetError("bad format for venti score"); + return 0; + } + if((i & 1) == 0) + c <<= 4; + + score[(i>>1)-2] |= c; + } + return 1; +} + +VacFS * +vfsOpen(VtSession *z, char *file, int readOnly, long ncache) +{ + VacFS *fs; + int n, fd; + VtRoot rt; + uchar score[VtScoreSize], buf[VtRootSize]; + VacFile *root; + + fd = open(file, OREAD); + if(fd < 0) { + vtOSError(); + return nil; + } + + if(!readScore(fd, score)) { + close(fd); + return nil; + } + close(fd); + + n = vtRead(z, score, VtRootType, buf, VtRootSize); + if(n < 0) + return nil; + if(n != VtRootSize) { + vtSetError("vtRead on root too short"); + return nil; + } + + if(!vtSha1Check(score, buf, VtRootSize)) { + vtSetError("vtSha1Check failed on root block"); + return nil; + } + + if(!vtRootUnpack(&rt, buf)) + return nil; + + if(strcmp(rt.type, "vac") != 0) { + vtSetError("not a vac root"); + return nil; + } + + fs = vfsAlloc(z, rt.blockSize, ncache); + memmove(fs->score, score, VtScoreSize); + fs->readOnly = readOnly; + root = vfRoot(fs, rt.score); + if(root == nil) + goto Err; + fs->root = root; + + return fs; +Err: + if(root) + vfDecRef(root); + vfsClose(fs); + return nil; +} + +VacFS * +vacFsCreate(VtSession *z, int bsize, long ncache) +{ + VacFS *fs; + + fs = vfsAlloc(z, bsize, ncache); + return fs; +} + +int +vfsIsReadOnly(VacFS *fs) +{ + return fs->readOnly != 0; +} + +VacFile * +vfsGetRoot(VacFS *fs) +{ + return vfIncRef(fs->root); +} + +int +vfsGetBlockSize(VacFS *fs) +{ + return fs->bsize; +} + +int +vfsGetScore(VacFS *fs, uchar score[VtScoreSize]) +{ + memmove(fs, score, VtScoreSize); + return 1; +} + +long +vfsGetCacheSize(VacFS *fs) +{ + return cacheGetSize(fs->cache); +} + +int +vfsSetCacheSize(VacFS *fs, long size) +{ + return cacheSetSize(fs->cache, size); +} + +int +vfsSnapshot(VacFS *fs, char *src, char *dst) +{ + USED(fs); + USED(src); + USED(dst); + return 1; +} + +int +vfsSync(VacFS*) +{ + return 1; +} + +int +vfsClose(VacFS *fs) +{ + if(fs->root) + vfDecRef(fs->root); + fs->root = nil; + cacheCheck(fs->cache); + cacheFree(fs->cache); + memset(fs, 0, sizeof(VacFS)); + vtMemFree(fs); + return 1; +} + + diff --git a/src/cmd/vac/mkfile b/src/cmd/vac/mkfile new file mode 100644 index 00000000..ac33ab32 --- /dev/null +++ b/src/cmd/vac/mkfile @@ -0,0 +1,36 @@ +PLAN9=../../.. +<$PLAN9/src/mkhdr + +LIBFILES=\ + cache\ + error\ + file\ + fs\ + source\ + pack\ + +LIB=${LIBFILES:%=%.$O} + +HFILES=\ + $PLAN9/include/venti.h\ + stdinc.h\ + error.h\ + vac.h\ + dat.h\ + fns.h\ + +TARG=vac vtdump + +CFILES=${TARG:%=%.c} ${LIBFILES:%=%.c} srcload.c vactest.c + +UPDATE=\ + mkfile\ + $CFILES\ + $HFILES\ + ${TARG:%=/386/bin/%} + +default:V: all + +test:V: $O.srcload $O.wtest $O.rtest $O.vtdump $O.vtread + +<$PLAN9/src/mkmany diff --git a/src/cmd/vac/pack.c b/src/cmd/vac/pack.c new file mode 100644 index 00000000..b16834d9 --- /dev/null +++ b/src/cmd/vac/pack.c @@ -0,0 +1,609 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" +#include "error.h" + +typedef struct MetaChunk MetaChunk; + +struct MetaChunk { + ushort offset; + ushort size; + ushort index; +}; + +static int stringUnpack(char **s, uchar **p, int *n); + +/* + * integer conversion routines + */ +#define U8GET(p) ((p)[0]) +#define U16GET(p) (((p)[0]<<8)|(p)[1]) +#define U32GET(p) (((p)[0]<<24)|((p)[1]<<16)|((p)[2]<<8)|(p)[3]) +#define U48GET(p) (((uvlong)U16GET(p)<<32)|(uvlong)U32GET((p)+2)) +#define U64GET(p) (((uvlong)U32GET(p)<<32)|(uvlong)U32GET((p)+4)) + +#define U8PUT(p,v) (p)[0]=(v) +#define U16PUT(p,v) (p)[0]=(v)>>8;(p)[1]=(v) +#define U32PUT(p,v) (p)[0]=(v)>>24;(p)[1]=(v)>>16;(p)[2]=(v)>>8;(p)[3]=(v) +#define U48PUT(p,v,t32) t32=(v)>>32;U16PUT(p,t32);t32=(v);U32PUT((p)+2,t32) +#define U64PUT(p,v,t32) t32=(v)>>32;U32PUT(p,t32);t32=(v);U32PUT((p)+4,t32) + +static int +stringUnpack(char **s, uchar **p, int *n) +{ + int nn; + + if(*n < 2) + return 0; + + nn = U16GET(*p); + *p += 2; + *n -= 2; + if(nn > *n) + return 0; + *s = vtMemAlloc(nn+1); + memmove(*s, *p, nn); + (*s)[nn] = 0; + *p += nn; + *n -= nn; + return 1; +} + +static int +stringPack(char *s, uchar *p) +{ + int n; + + n = strlen(s); + U16PUT(p, n); + memmove(p+2, s, n); + return n+2; +} + + +int +mbUnpack(MetaBlock *mb, uchar *p, int n) +{ + u32int magic; + + mb->maxsize = n; + mb->buf = p; + + if(n == 0) { + memset(mb, 0, sizeof(MetaBlock)); + return 1; + } + + magic = U32GET(p); + if(magic != MetaMagic && magic != MetaMagic+1) { + vtSetError("bad meta block magic"); + return 0; + } + mb->size = U16GET(p+4); + mb->free = U16GET(p+6); + mb->maxindex = U16GET(p+8); + mb->nindex = U16GET(p+10); + mb->unbotch = (magic == MetaMagic+1); + + if(mb->size > n) { + vtSetError("bad meta block size"); + return 0; + } + p += MetaHeaderSize; + n -= MetaHeaderSize; + + USED(p); + if(n < mb->maxindex*MetaIndexSize) { + vtSetError("truncated meta block 2"); + return 0; + } + return 1; +} + +void +mbPack(MetaBlock *mb) +{ + uchar *p; + + p = mb->buf; + + U32PUT(p, MetaMagic); + U16PUT(p+4, mb->size); + U16PUT(p+6, mb->free); + U16PUT(p+8, mb->maxindex); + U16PUT(p+10, mb->nindex); +} + + +void +mbDelete(MetaBlock *mb, int i, MetaEntry *me) +{ + uchar *p; + int n; + + assert(i < mb->nindex); + + if(me->p - mb->buf + me->size == mb->size) + mb->size -= me->size; + else + mb->free += me->size; + + p = mb->buf + MetaHeaderSize + i*MetaIndexSize; + n = (mb->nindex-i-1)*MetaIndexSize; + memmove(p, p+MetaIndexSize, n); + memset(p+n, 0, MetaIndexSize); + mb->nindex--; +} + +void +mbInsert(MetaBlock *mb, int i, MetaEntry *me) +{ + uchar *p; + int o, n; + + assert(mb->nindex < mb->maxindex); + + o = me->p - mb->buf; + n = me->size; + if(o+n > mb->size) { + mb->free -= mb->size - o; + mb->size = o + n; + } else + mb->free -= n; + + p = mb->buf + MetaHeaderSize + i*MetaIndexSize; + n = (mb->nindex-i)*MetaIndexSize; + memmove(p+MetaIndexSize, p, n); + U16PUT(p, me->p - mb->buf); + U16PUT(p+2, me->size); + mb->nindex++; +} + +int +meUnpack(MetaEntry *me, MetaBlock *mb, int i) +{ + uchar *p; + int eo, en; + + if(i < 0 || i >= mb->nindex) { + vtSetError("bad meta entry index"); + return 0; + } + + p = mb->buf + MetaHeaderSize + i*MetaIndexSize; + eo = U16GET(p); + en = U16GET(p+2); + +if(0)print("eo = %d en = %d\n", eo, en); + if(eo < MetaHeaderSize + mb->maxindex*MetaIndexSize) { + vtSetError("corrupted entry in meta block"); + return 0; + } + + if(eo+en > mb->size) { + vtSetError("truncated meta block"); + return 0; + } + + p = mb->buf + eo; + + /* make sure entry looks ok and includes an elem name */ + if(en < 8 || U32GET(p) != DirMagic || en < 8 + U16GET(p+6)) { + vtSetError("corrupted meta block entry"); + return 0; + } + + me->p = p; + me->size = en; + + return 1; +} + +/* assumes a small amount of checking has been done in mbEntry */ +int +meCmp(MetaEntry *me, char *s) +{ + int n; + uchar *p; + + p = me->p; + + p += 6; + n = U16GET(p); + p += 2; + + assert(n + 8 < me->size); + + while(n > 0) { + if(*s == 0) + return -1; + if(*p < (uchar)*s) + return -1; + if(*p > (uchar)*s) + return 1; + p++; + s++; + n--; + } + return *s != 0; +} + +int +meCmpNew(MetaEntry *me, char *s) +{ + int n; + uchar *p; + + p = me->p; + + p += 6; + n = U16GET(p); + p += 2; + + assert(n + 8 < me->size); + + while(n > 0) { + if(*s == 0) + return 1; + if(*p < (uchar)*s) + return -1; + if(*p > (uchar)*s) + return 1; + p++; + s++; + n--; + } + return -(*s != 0); +} + +static int +offsetCmp(void *s0, void *s1) +{ + MetaChunk *mc0, *mc1; + + mc0 = s0; + mc1 = s1; + if(mc0->offset < mc1->offset) + return -1; + if(mc0->offset > mc1->offset) + return 1; + return 0; +} + +static MetaChunk * +metaChunks(MetaBlock *mb) +{ + MetaChunk *mc; + int oo, o, n, i; + uchar *p; + + mc = vtMemAlloc(mb->nindex*sizeof(MetaChunk)); + p = mb->buf + MetaHeaderSize; + for(i = 0; i<mb->nindex; i++) { + mc[i].offset = U16GET(p); + mc[i].size = U16GET(p+2); + mc[i].index = i; + p += MetaIndexSize; + } + + qsort(mc, mb->nindex, sizeof(MetaChunk), offsetCmp); + + /* check block looks ok */ + oo = MetaHeaderSize + mb->maxindex*MetaIndexSize; + o = oo; + n = 0; + for(i=0; i<mb->nindex; i++) { + o = mc[i].offset; + n = mc[i].size; + if(o < oo) + goto Err; + oo += n; + } + if(o+n <= mb->size) + goto Err; + if(mb->size - oo != mb->free) + goto Err; + + return mc; +Err: + vtMemFree(mc); + return nil; +} + +static void +mbCompact(MetaBlock *mb, MetaChunk *mc) +{ + int oo, o, n, i; + + oo = MetaHeaderSize + mb->maxindex*MetaIndexSize; + + for(i=0; i<mb->nindex; i++) { + o = mc[i].offset; + n = mc[i].size; + if(o != oo) { + memmove(mb->buf + oo, mb->buf + o, n); + U16PUT(mb->buf + MetaHeaderSize + mc[i].index*MetaIndexSize, oo); + } + oo += n; + } + + mb->size = oo; + mb->free = 0; +} + +uchar * +mbAlloc(MetaBlock *mb, int n) +{ + int i, o; + MetaChunk *mc; + + /* off the end */ + if(mb->maxsize - mb->size >= n) + return mb->buf + mb->size; + + /* check if possible */ + if(mb->maxsize - mb->size + mb->free < n) + return nil; + + mc = metaChunks(mb); + + /* look for hole */ + o = MetaHeaderSize + mb->maxindex*MetaIndexSize; + for(i=0; i<mb->nindex; i++) { + if(mc[i].offset - o >= n) { + vtMemFree(mc); + return mb->buf + o; + } + o = mc[i].offset + mc[i].size; + } + + if(mb->maxsize - o >= n) { + vtMemFree(mc); + return mb->buf + o; + } + + /* compact and return off the end */ + mbCompact(mb, mc); + vtMemFree(mc); + + assert(mb->maxsize - mb->size >= n); + return mb->buf + mb->size; +} + +int +vdSize(VacDir *dir) +{ + int n; + + /* constant part */ + + n = 4 + /* magic */ + 2 + /* version */ + 4 + /* entry */ + 4 + /* guid */ + 4 + /* mentry */ + 4 + /* mgen */ + 8 + /* qid */ + 4 + /* mtime */ + 4 + /* mcount */ + 4 + /* ctime */ + 4 + /* atime */ + 4 + /* mode */ + 0; + + /* strings */ + n += 2 + strlen(dir->elem); + n += 2 + strlen(dir->uid); + n += 2 + strlen(dir->gid); + n += 2 + strlen(dir->mid); + + /* optional sections */ + if(dir->qidSpace) { + n += 3 + /* option header */ + 8 + /* qidOffset */ + 8; /* qid Max */ + } + + return n; +} + +void +vdPack(VacDir *dir, MetaEntry *me) +{ + uchar *p; + ulong t32; + + p = me->p; + + U32PUT(p, DirMagic); + U16PUT(p+4, 9); /* version */ + p += 6; + + p += stringPack(dir->elem, p); + + U32PUT(p, dir->entry); + U32PUT(p+4, dir->gen); + U32PUT(p+8, dir->mentry); + U32PUT(p+12, dir->mgen); + U64PUT(p+16, dir->qid, t32); + p += 24; + + p += stringPack(dir->uid, p); + p += stringPack(dir->gid, p); + p += stringPack(dir->mid, p); + + U32PUT(p, dir->mtime); + U32PUT(p+4, dir->mcount); + U32PUT(p+8, dir->ctime); + U32PUT(p+12, dir->atime); + U32PUT(p+16, dir->mode); + p += 5*4; + + if(dir->qidSpace) { + U8PUT(p, DirQidSpaceEntry); + U16PUT(p+1, 2*8); + p += 3; + U64PUT(p, dir->qidOffset, t32); + U64PUT(p+8, dir->qidMax, t32); + } + + assert(p == me->p + me->size); +} + + +int +vdUnpack(VacDir *dir, MetaEntry *me) +{ + int t, nn, n, version; + uchar *p; + + p = me->p; + n = me->size; + + memset(dir, 0, sizeof(VacDir)); + +if(0)print("vdUnpack\n"); + /* magic */ + if(n < 4 || U32GET(p) != DirMagic) + goto Err; + p += 4; + n -= 4; + +if(0)print("vdUnpack: got magic\n"); + /* version */ + if(n < 2) + goto Err; + version = U16GET(p); + if(version < 7 || version > 9) + goto Err; + p += 2; + n -= 2; + +if(0)print("vdUnpack: got version\n"); + + /* elem */ + if(!stringUnpack(&dir->elem, &p, &n)) + goto Err; + +if(0)print("vdUnpack: got elem\n"); + + /* entry */ + if(n < 4) + goto Err; + dir->entry = U32GET(p); + p += 4; + n -= 4; + +if(0)print("vdUnpack: got entry\n"); + + if(version < 9) { + dir->gen = 0; + dir->mentry = dir->entry+1; + dir->mgen = 0; + } else { + if(n < 3*4) + goto Err; + dir->gen = U32GET(p); + dir->mentry = U32GET(p+4); + dir->mgen = U32GET(p+8); + p += 3*4; + n -= 3*4; + } + +if(0)print("vdUnpack: got gen etc\n"); + + /* size is gotten from DirEntry */ + + /* qid */ + if(n < 8) + goto Err; + dir->qid = U64GET(p); + p += 8; + n -= 8; + +if(0)print("vdUnpack: got qid\n"); + /* skip replacement */ + if(version == 7) { + if(n < VtScoreSize) + goto Err; + p += VtScoreSize; + n -= VtScoreSize; + } + + /* uid */ + if(!stringUnpack(&dir->uid, &p, &n)) + goto Err; + + /* gid */ + if(!stringUnpack(&dir->gid, &p, &n)) + goto Err; + + /* mid */ + if(!stringUnpack(&dir->mid, &p, &n)) + goto Err; + +if(0)print("vdUnpack: got ids\n"); + if(n < 5*4) + goto Err; + dir->mtime = U32GET(p); + dir->mcount = U32GET(p+4); + dir->ctime = U32GET(p+8); + dir->atime = U32GET(p+12); + dir->mode = U32GET(p+16); + p += 5*4; + n -= 5*4; + +if(0)print("vdUnpack: got times\n"); + /* optional meta data */ + while(n > 0) { + if(n < 3) + goto Err; + t = p[0]; + nn = U16GET(p+1); + p += 3; + n -= 3; + if(n < nn) + goto Err; + switch(t) { + case DirPlan9Entry: + /* not valid in version >= 9 */ + if(version >= 9) + break; + if(dir->plan9 || nn != 12) + goto Err; + dir->plan9 = 1; + dir->p9path = U64GET(p); + dir->p9version = U32GET(p+8); + if(dir->mcount == 0) + dir->mcount = dir->p9version; + break; + case DirGenEntry: + /* not valid in version >= 9 */ + if(version >= 9) + break; + break; + case DirQidSpaceEntry: + if(dir->qidSpace || nn != 16) + goto Err; + dir->qidSpace = 1; + dir->qidOffset = U64GET(p); + dir->qidMax = U64GET(p+8); + break; + } + p += nn; + n -= nn; + } +if(0)print("vdUnpack: got options\n"); + + if(p != me->p + me->size) + goto Err; + +if(0)print("vdUnpack: correct size\n"); + return 1; +Err: +if(0)print("vdUnpack: XXXXXXXXXXXX EbadMeta\n"); + vtSetError(EBadMeta); + vdCleanup(dir); + return 0; +} diff --git a/src/cmd/vac/rtest.c b/src/cmd/vac/rtest.c new file mode 100644 index 00000000..d45cd0fe --- /dev/null +++ b/src/cmd/vac/rtest.c @@ -0,0 +1,71 @@ +#include "stdinc.h" + +enum { + Nblock = 300000, + BlockSize = 8*1024, +}; + +uchar data[Nblock*VtScoreSize]; +int rflag; +int nblock = 10000; +int perm[Nblock]; + +void +main(int argc, char *argv[]) +{ + VtSession *z; + int i, j, t; + int start; + uchar buf[BlockSize]; + + srand(time(0)); + + ARGBEGIN{ + case 'r': + rflag++; + break; + case 'n': + nblock = atoi(ARGF()); + break; + }ARGEND + + for(i=0; i<nblock; i++) + perm[i] = i; + + if(rflag) { + for(i=0; i<nblock; i++) { + j = nrand(nblock); + t = perm[j]; + perm[j] = perm[i]; + perm[i] = t; + } + } + + if(readn(0, data, VtScoreSize*nblock) < VtScoreSize*nblock) + sysfatal("read failed: %r"); + + vtAttach(); + + z = vtDial("iolaire2"); + if(z == nil) + sysfatal("cound not connect to venti"); + if(!vtConnect(z, 0)) + vtFatal("vtConnect: %s", vtGetError()); + + print("starting\n"); + + start = times(0); + + if(rflag && nblock > 10000) + nblock = 10000; + + for(i=0; i<nblock; i++) { + if(vtRead(z, data+perm[i]*VtScoreSize, VtDataType, buf, BlockSize) < 0) + vtFatal("vtRead failed: %d: %s", i, vtGetError()); + } + + print("time = %f\n", (times(0) - start)*0.001); + + vtClose(z); + vtDetach(); +} diff --git a/src/cmd/vac/source.c b/src/cmd/vac/source.c new file mode 100644 index 00000000..e7245a2f --- /dev/null +++ b/src/cmd/vac/source.c @@ -0,0 +1,390 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" +#include "error.h" + +static int sizeToDepth(uvlong s, int psize, int dsize); + +static int +sizeToDepth(uvlong s, int psize, int dsize) +{ + int np; + int d; + + /* determine pointer depth */ + np = psize/VtScoreSize; + s = (s + dsize - 1)/dsize; + for(d = 0; s > 1; d++) + s = (s + np - 1)/np; + return d; +} + +/* assumes u is lock? */ +Source * +sourceAlloc(Cache *c, Lump *u, ulong block, int entry, int readOnly) +{ + Source *r; + VtEntry d; + + if(u->asize < (entry+1)*VtEntrySize) { + vtSetError(ENoDir); + return nil; + } + + if(!vtEntryUnpack(&d, u->data, entry)) + return nil; + + if(!(d.flags & VtEntryActive)) { +fprint(2, "bad flags %#ux %V\n", d.flags, d.score); + vtSetError(ENoDir); + return nil; + } + + /* HACK for backwards compatiblity - should go away at some point */ + if(d.depth == 0) { +if(d.size > d.dsize) fprint(2, "depth == 0! size = %ulld\n", d.size); + d.depth = sizeToDepth(d.size, d.psize, d.dsize); + } + + if(d.depth < sizeToDepth(d.size, d.psize, d.dsize)) { + vtSetError(EBadDir); + return nil; + } + + r = vtMemAllocZ(sizeof(Source)); + r->lk = vtLockAlloc(); + r->cache = c; + r->readOnly = readOnly; + r->lump = lumpIncRef(u); + r->block = block; + r->entry = entry; + r->gen = d.gen; + r->dir = (d.flags & VtEntryDir) != 0; + r->depth = d.depth; + r->psize = d.psize; + r->dsize = d.dsize; + r->size = d.size; + + r->epb = r->dsize/VtEntrySize; + + return r; +} + +Source * +sourceOpen(Source *r, ulong entry, int readOnly) +{ + ulong bn; + Lump *u; + +if(0)fprint(2, "sourceOpen: %V:%d: %lud\n", r->lump->score, r->entry, entry); + if(r->readOnly && !readOnly) { + vtSetError(EReadOnly); + return nil; + } + + bn = entry/r->epb; + + u = sourceGetLump(r, bn, readOnly, 1); + if(u == nil) + return nil; + + r = sourceAlloc(r->cache, u, bn, entry%r->epb, readOnly); + lumpDecRef(u, 1); + return r; +} + +Source * +sourceCreate(Source *r, int psize, int dsize, int isdir, ulong entry) +{ + Source *rr; + int i; + Lump *u; + ulong bn; + VtEntry dir; + + if(r->readOnly) { + vtSetError(EReadOnly); + return nil; + } + + if(entry == 0) { + /* + * look at a random block to see if we can find an empty entry + */ + entry = sourceGetDirSize(r); + entry = r->epb*lnrand(entry/r->epb+1); + } + + /* + * need to loop since multiple threads could be trying to allocate + */ + for(;;) { + bn = entry/r->epb; + sourceSetDepth(r, (uvlong)(bn+1)*r->dsize); + u = sourceGetLump(r, bn, 0, 1); + if(u == nil) + return nil; + for(i=entry%r->epb; i<r->epb; i++) { + vtEntryUnpack(&dir, u->data, i); + if((dir.flags&VtEntryActive) == 0 && dir.gen != ~0) + goto Found; + } + lumpDecRef(u, 1); + entry = sourceGetDirSize(r); + } +Found: + /* found an entry */ + dir.psize = psize; + dir.dsize = dsize; + dir.flags = VtEntryActive; + if(isdir) + dir.flags |= VtEntryDir; + dir.depth = 0; + dir.size = 0; + memmove(dir.score, vtZeroScore, VtScoreSize); + vtEntryPack(&dir, u->data, i); + + sourceSetDirSize(r, bn*r->epb + i + 1); + rr = sourceAlloc(r->cache, u, bn, i, 0); + + lumpDecRef(u, 1); + return rr; +} + +void +sourceRemove(Source *r) +{ + lumpFreeEntry(r->lump, r->entry); + sourceFree(r); +} + +int +sourceSetDepth(Source *r, uvlong size) +{ + Lump *u, *v; + VtEntry dir; + int depth; + + if(r->readOnly){ + vtSetError(EReadOnly); + return 0; + } + + depth = sizeToDepth(size, r->psize, r->dsize); + + assert(depth >= 0); + + if(depth > VtPointerDepth) { + vtSetError(ETooBig); + return 0; + } + + vtLock(r->lk); + + if(r->depth >= depth) { + vtUnlock(r->lk); + return 1; + } + + u = r->lump; + vtLock(u->lk); + if(!vtEntryUnpack(&dir, u->data, r->entry)) { + vtUnlock(u->lk); + vtUnlock(r->lk); + return 0; + } + while(dir.depth < depth) { + v = cacheAllocLump(r->cache, VtPointerType0+r->depth, r->psize, r->dir); + if(v == nil) + break; + memmove(v->data, dir.score, VtScoreSize); + memmove(dir.score, v->score, VtScoreSize); + dir.depth++; + vtUnlock(v->lk); + } + vtEntryPack(&dir, u->data, r->entry); + vtUnlock(u->lk); + + r->depth = dir.depth; + vtUnlock(r->lk); + + return dir.depth == depth; +} + +int +sourceGetVtEntry(Source *r, VtEntry *dir) +{ + Lump *u; + + u = r->lump; + vtLock(u->lk); + if(!vtEntryUnpack(dir, u->data, r->entry)) { + vtUnlock(u->lk); + return 0; + } + vtUnlock(u->lk); + return 1; +} + +uvlong +sourceGetSize(Source *r) +{ + uvlong size; + + vtLock(r->lk); + size = r->size; + vtUnlock(r->lk); + + return size; +} + + +int +sourceSetSize(Source *r, uvlong size) +{ + Lump *u; + VtEntry dir; + int depth; + + if(r->readOnly) { + vtSetError(EReadOnly); + return 0; + } + + if(size > VtMaxFileSize || size > ((uvlong)MaxBlock)*r->dsize) { + vtSetError(ETooBig); + return 0; + } + + vtLock(r->lk); + depth = sizeToDepth(size, r->psize, r->dsize); + if(size < r->size) { + vtUnlock(r->lk); + return 1; + } + if(depth > r->depth) { + vtSetError(EBadDir); + vtUnlock(r->lk); + return 0; + } + + u = r->lump; + vtLock(u->lk); + vtEntryUnpack(&dir, u->data, r->entry); + dir.size = size; + vtEntryPack(&dir, u->data, r->entry); + vtUnlock(u->lk); + r->size = size; + vtUnlock(r->lk); + return 1; +} + +int +sourceSetDirSize(Source *r, ulong ds) +{ + uvlong size; + + size = (uvlong)r->dsize*(ds/r->epb); + size += VtEntrySize*(ds%r->epb); + return sourceSetSize(r, size); +} + +ulong +sourceGetDirSize(Source *r) +{ + ulong ds; + uvlong size; + + size = sourceGetSize(r); + ds = r->epb*(size/r->dsize); + ds += (size%r->dsize)/VtEntrySize; + return ds; +} + +ulong +sourceGetNumBlocks(Source *r) +{ + return (sourceGetSize(r)+r->dsize-1)/r->dsize; +} + +Lump * +sourceWalk(Source *r, ulong block, int readOnly, int *off) +{ + int depth; + int i, np; + Lump *u, *v; + int elem[VtPointerDepth+1]; + ulong b; + + if(r->readOnly && !readOnly) { + vtSetError(EReadOnly); + return nil; + } + + vtLock(r->lk); + np = r->psize/VtScoreSize; + b = block; + for(i=0; i<r->depth; i++) { + elem[i] = b % np; + b /= np; + } + if(b != 0) { + vtUnlock(r->lk); + vtSetError(EBadOffset); + return nil; + } + elem[i] = r->entry; + u = lumpIncRef(r->lump); + depth = r->depth; + *off = elem[0]; + vtUnlock(r->lk); + + for(i=depth; i>0; i--) { + v = lumpWalk(u, elem[i], VtPointerType0+i-1, r->psize, readOnly, 0); + lumpDecRef(u, 0); + if(v == nil) + return nil; + u = v; + } + + return u; +} + +Lump * +sourceGetLump(Source *r, ulong block, int readOnly, int lock) +{ + int type, off; + Lump *u, *v; + + if(r->readOnly && !readOnly) { + vtSetError(EReadOnly); + return nil; + } + if(block == NilBlock) { + vtSetError(ENilBlock); + return nil; + } +if(0)fprint(2, "sourceGetLump: %V:%d %lud\n", r->lump->score, r->entry, block); + u = sourceWalk(r, block, readOnly, &off); + if(u == nil) + return nil; + if(r->dir) + type = VtDirType; + else + type = VtDataType; + v = lumpWalk(u, off, type, r->dsize, readOnly, lock); + lumpDecRef(u, 0); + return v; +} + +void +sourceFree(Source *k) +{ + if(k == nil) + return; + lumpDecRef(k->lump, 0); + vtLockFree(k->lk); + memset(k, ~0, sizeof(*k)); + vtMemFree(k); +} diff --git a/src/cmd/vac/srcload.c b/src/cmd/vac/srcload.c new file mode 100644 index 00000000..4f48efc6 --- /dev/null +++ b/src/cmd/vac/srcload.c @@ -0,0 +1,302 @@ +#include "stdinc.h" +#include <bio.h> +#include "vac.h" +#include "dat.h" +#include "fns.h" +#include "error.h" + +int num = 1000; +int length = 20*1024; +int block= 1024; +int bush = 4; +int iter = 10000; +Biobuf *bout; +int maxdepth; + +Source *mkroot(Cache*); +void new(Source*, int trace, int); +int delete(Source*); +void dump(Source*, int indent, ulong nentry); +void dumpone(Source *s); +int count(Source *s, int); +void stats(Source *s); + +void +main(int argc, char *argv[]) +{ + int i; + Cache *c; + char *host = nil; + VtSession *z; + int csize = 10000; + Source *r; + ulong t; + + t = time(0); + fprint(1, "time = %lud\n", t); + + srand(t); + + ARGBEGIN{ + case 'i': + iter = atoi(ARGF()); + break; + case 'n': + num = atoi(ARGF()); + break; + case 'l': + length = atoi(ARGF()); + break; + case 'b': + block = atoi(ARGF()); + break; + case 'h': + host = ARGF(); + break; + case 'u': + bush = atoi(ARGF()); + break; + case 'c': + csize = atoi(ARGF()); + break; + }ARGEND; + + vtAttach(); + + bout = vtMemAllocZ(sizeof(Biobuf)); + Binit(bout, 1, OWRITE); + + fmtinstall('V', vtScoreFmt); + fmtinstall('R', vtErrFmt); + + z = vtDial(host); + if(z == nil) + vtFatal("could not connect to server: %s", vtGetError()); + + if(!vtConnect(z, 0)) + sysfatal("vtConnect: %r"); + + c = cacheAlloc(z, block, csize); + r = mkroot(c); + for(i=0; i<num; i++) + new(r, 0, 0); + + for(i=0; i<iter; i++) { +if(i % 10000 == 0) +stats(r); + new(r, 0, 0); + delete(r); + } + + fprint(2, "count = %d top = %lud\n", count(r, 0), sourceGetDirSize(r)); +// cacheCheck(c); +fprint(2, "deleting\n"); + for(i=0; i<num; i++) + delete(r); + +// dump(r, 0, 0); + + lumpDecRef(r->lump, 0); + sourceRemove(r); + cacheCheck(c); + + vtClose(z); + vtDetach(); + + exits(0); +} + + +Source * +mkroot(Cache *c) +{ + Lump *u; + VtEntry *dir; + Source *r; + + u = cacheAllocLump(c, VtDirType, cacheGetBlockSize(c), 1); + dir = (VtEntry*)u->data; + vtPutUint16(dir->psize, cacheGetBlockSize(c)); + vtPutUint16(dir->dsize, cacheGetBlockSize(c)); + dir->flag = VtEntryActive|VtEntryDir; + memmove(dir->score, vtZeroScore, VtScoreSize); + + r = sourceAlloc(c, u, 0, 0); + vtUnlock(u->lk); + if(r == nil) + sysfatal("could not create root source: %R"); + return r; +} + +void +new(Source *s, int trace, int depth) +{ + int i, n; + Source *ss; + + if(depth > maxdepth) + maxdepth = depth; + + n = sourceGetDirSize(s); + for(i=0; i<n; i++) { + ss = sourceOpen(s, nrand(n), 0); + if(ss == nil) + continue; + if(ss->dir && frand() < 1./bush) { + if(trace) { + int j; + for(j=0; j<trace; j++) + Bprint(bout, " "); + Bprint(bout, "decend %d\n", i); + } + new(ss, trace?trace+1:0, depth+1); + sourceFree(ss); + return; + } + sourceFree(ss); + } + ss = sourceCreate(s, s->psize, s->dsize, 1+frand()>.5, 0); + if(ss == nil) + fprint(2, "could not create directory: %R\n"); + if(trace) { + int j; + for(j=1; j<trace; j++) + Bprint(bout, " "); + Bprint(bout, "create %d %V\n", ss->entry, ss->lump->score); + } + sourceFree(ss); +} + +int +delete(Source *s) +{ + int i, n; + Source *ss; + + assert(s->dir); + + n = sourceGetDirSize(s); + /* check if empty */ + for(i=0; i<n; i++) { + ss = sourceOpen(s, i, 1); + if(ss != nil) { + sourceFree(ss); + break; + } + } + if(i == n) + return 0; + + for(;;) { + ss = sourceOpen(s, nrand(n), 0); + if(ss == nil) + continue; + if(ss->dir && delete(ss)) { + sourceFree(ss); + return 1; + } + if(1) + break; + sourceFree(ss); + } + + + sourceRemove(ss); + return 1; +} + +void +dumpone(Source *s) +{ + ulong i, n; + Source *ss; + + Bprint(bout, "gen %4lud depth %d %V", s->gen, s->depth, s->lump->score); + if(!s->dir) { + Bprint(bout, " data size: %llud\n", s->size); + return; + } + n = sourceGetDirSize(s); + Bprint(bout, " dir size: %lud\n", n); + for(i=0; i<n; i++) { + ss = sourceOpen(s, i, 1); + if(ss == nil) { +fprint(2, "%lud: %R\n", i); + continue; + } + Bprint(bout, "\t%lud %d %llud %V\n", i, ss->dir, ss->size, ss->lump->score); + sourceFree(ss); + } + return; +} + + +void +dump(Source *s, int ident, ulong entry) +{ + ulong i, n; + Source *ss; + + for(i=0; i<ident; i++) + Bprint(bout, " "); + Bprint(bout, "%4lud: gen %4lud depth %d", entry, s->gen, s->depth); + if(!s->dir) { + Bprint(bout, " data size: %llud\n", s->size); + return; + } + n = sourceGetDirSize(s); + Bprint(bout, " dir size: %lud\n", n); + for(i=0; i<n; i++) { + ss = sourceOpen(s, i, 1); + if(ss == nil) + continue; + dump(ss, ident+1, i); + sourceFree(ss); + } + return; +} + +int +count(Source *s, int rec) +{ + ulong i, n; + int c; + Source *ss; + + if(!s->dir) + return 0; + n = sourceGetDirSize(s); + c = 0; + for(i=0; i<n; i++) { + ss = sourceOpen(s, i, 1); + if(ss == nil) + continue; + if(rec) + c += count(ss, rec); + c++; + sourceFree(ss); + } + return c; +} + +void +stats(Source *s) +{ + int n, i, c, cc, max; + Source *ss; + + cc = 0; + max = 0; + n = sourceGetDirSize(s); + for(i=0; i<n; i++) { + ss = sourceOpen(s, i, 1); + if(ss == nil) + continue; + cc++; + c = count(ss, 1); + if(c > max) + max = c; + sourceFree(ss); + } +fprint(2, "count = %d top = %d depth=%d maxcount %d\n", cc, n, maxdepth, max); +} diff --git a/src/cmd/vac/stdinc.h b/src/cmd/vac/stdinc.h new file mode 100644 index 00000000..aca405fc --- /dev/null +++ b/src/cmd/vac/stdinc.h @@ -0,0 +1,8 @@ +#include <u.h> +#include <libc.h> + +#include "venti.h" + +typedef uvlong u64int; +typedef uchar u8int; +typedef ushort u16int; diff --git a/src/cmd/vac/util.c b/src/cmd/vac/util.c new file mode 100644 index 00000000..f52acf60 --- /dev/null +++ b/src/cmd/vac/util.c @@ -0,0 +1,71 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" + +int +vtGetUint16(uchar *p) +{ + return (p[0]<<8)|p[1]; +} + +ulong +vtGetUint32(uchar *p) +{ + return (p[0]<<24)|(p[1]<<16)|(p[2]<<8)|p[3]; +} + +uvlong +vtGetUint48(uchar *p) +{ + return ((uvlong)p[0]<<40)|((uvlong)p[1]<<32)| + (p[2]<<24)|(p[3]<<16)|(p[4]<<8)|p[5]; +} + +uvlong +vtGetUint64(uchar *p) +{ + return ((uvlong)p[0]<<56)|((uvlong)p[1]<<48)|((uvlong)p[2]<<40)| + ((uvlong)p[3]<<32)|(p[4]<<24)|(p[5]<<16)|(p[6]<<8)|p[7]; +} + + +void +vtPutUint16(uchar *p, int x) +{ + p[0] = x>>8; + p[1] = x; +} + +void +vtPutUint32(uchar *p, ulong x) +{ + p[0] = x>>24; + p[1] = x>>16; + p[2] = x>>8; + p[3] = x; +} + +void +vtPutUint48(uchar *p, uvlong x) +{ + p[0] = x>>40; + p[1] = x>>32; + p[2] = x>>24; + p[3] = x>>16; + p[4] = x>>8; + p[5] = x; +} + +void +vtPutUint64(uchar *p, uvlong x) +{ + p[0] = x>>56; + p[1] = x>>48; + p[2] = x>>40; + p[3] = x>>32; + p[4] = x>>24; + p[5] = x>>16; + p[6] = x>>8; + p[7] = x; +} diff --git a/src/cmd/vac/vac-orig.c b/src/cmd/vac/vac-orig.c new file mode 100644 index 00000000..c38365be --- /dev/null +++ b/src/cmd/vac/vac-orig.c @@ -0,0 +1,1213 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" + +typedef struct Sink Sink; +typedef struct MetaSink MetaSink; +typedef struct DirSink DirSink; + +struct Sink { + VtSession *z; + VtEntry dir; + uchar *buf; + uchar *pbuf[VtPointerDepth+1]; +}; + +struct DirSink { + Sink *sink; + MetaSink *msink; + ulong nentry; + uchar *buf; + uchar *p; /* current pointer */ + uchar *ep; /* end pointer */ +}; + +struct MetaSink { + Sink *sink; + uchar *buf; + int maxindex; + int nindex; + uchar *rp; /* start of current record */ + uchar *p; /* current pointer */ + uchar *ep; /* end pointer */ +}; + +static void usage(void); +static int strpCmp(void*, void*); +static void warn(char *fmt, ...); +static void cleanup(void); +static u64int unittoull(char *s); +static int vac(VtSession *z, char *argv[]); +static void vacFile(DirSink *dsink, char *lname, char *sname, VacFile*); +static void vacStdin(DirSink *dsink, char *name, VacFile *vf); +static void vacData(DirSink *dsink, int fd, char *lname, VacFile*, Dir*); +static void vacDir(DirSink *dsink, int fd, char *lname, char *sname, VacFile*); +static int vacMerge(DirSink *dsink, char *lname, char *sname); + +Sink *sinkAlloc(VtSession *z, int psize, int dsize); +void sinkWrite(Sink *k, uchar *data, int n); +void sinkWriteScore(Sink *k, uchar *score, int n); +void sinkClose(Sink *k); +void sinkFree(Sink *k); + +DirSink *dirSinkAlloc(VtSession *z, int psize, int dsize); +void dirSinkWrite(DirSink *k, VtEntry*); +void dirSinkWriteSink(DirSink *k, Sink*); +int dirSinkWriteFile(DirSink *k, VacFile *vf); +void dirSinkClose(DirSink *k); +void dirSinkFree(DirSink *k); + +MetaSink *metaSinkAlloc(VtSession *z, int psize, int dsize); +void metaSinkPutc(MetaSink *k, int c); +void metaSinkPutString(MetaSink *k, char *s); +void metaSinkPutUint32(MetaSink *k, ulong x); +void metaSinkPutUint64(MetaSink *k, uvlong x); +void metaSinkWrite(MetaSink *k, uchar *data, int n); +void metaSinkWriteDir(MetaSink *ms, VacDir *vd); +void metaSinkEOR(MetaSink *k); +void metaSinkClose(MetaSink *k); +void metaSinkFree(MetaSink *k); +void plan9ToVacDir(VacDir*, Dir*, ulong entry, uvlong qid); + +enum { + Version = 8, + BlockSize = 8*1024, + MaxExclude = 1000, +}; + +struct { + ulong file; + ulong sfile; + ulong data; + ulong sdata; + ulong skip; + ulong meta; +} stats; + +int bsize = BlockSize; +int maxbsize; +char *oname, *dfile; +int verbose; +uvlong fileid = 1; +int qdiff; +char *exclude[MaxExclude]; +int nexclude; +int nowrite; +int merge; +char *isi; + +void +main(int argc, char *argv[]) +{ + VtSession *z; + char *p; + char *host = nil; + int statsFlag = 0; + + atexit(cleanup); + + ARGBEGIN{ + default: + usage(); + case 'b': + p = ARGF(); + if(p == 0) + usage(); + bsize = unittoull(p); + if(bsize == ~0) + usage(); + break; + case 'd': + dfile = ARGF(); + if(dfile == nil) + usage(); + break; + case 'e': + if(nexclude >= MaxExclude) + sysfatal("too many exclusions\n"); + exclude[nexclude] = ARGF(); + if(exclude[nexclude] == nil) + usage(); + nexclude++; + break; + case 'f': + oname = ARGF(); + if(oname == 0) + usage(); + break; + case 'h': + host = ARGF(); + if(host == nil) + usage(); + break; + case 'i': + isi = ARGF(); + if(isi == nil) + usage(); + break; + case 'n': + nowrite++; + break; + case 'm': + merge++; + break; + case 'q': + qdiff++; + break; + case 's': + statsFlag++; + break; + case 'v': + verbose++; + break; + }ARGEND; + + if(bsize < 512) + bsize = 512; + if(bsize > VtMaxLumpSize) + bsize = VtMaxLumpSize; + maxbsize = bsize; + + vtAttach(); + + fmtinstall('V', vtScoreFmt); + fmtinstall('R', vtErrFmt); + + z = vtDial(host, 0); + if(z == nil) + vtFatal("could not connect to server: %R"); + + if(!vtConnect(z, 0)) + vtFatal("vtConnect: %R"); + + qsort(exclude, nexclude, sizeof(char*), strpCmp); + + vac(z, argv); + if(!vtSync(z)) + fprint(2, "warning: could not ask server to flush pending writes: %R\n"); + + if(statsFlag) + fprint(2, "files %ld:%ld data %ld:%ld:%ld meta %ld\n", stats.file, stats.sfile, + stats.data, stats.skip, stats.sdata, stats.meta); +//packetStats(); + vtClose(z); + vtDetach(); + + exits(0); +} + +void +static usage(void) +{ + fprint(2, "usage: %s [-amqsv] [-h host] [-d vacfile] [-b blocksize] [-i name] [-e exclude] [-f vacfile] file ... \n", argv0); + exits("usage"); +} + +static +int strpCmp(void *p0, void *p1) +{ + return strcmp(*(char**)p0, *(char**)p1); +} + + +int +readBlock(int fd, uchar *buf, int n) +{ + int m, t = 0; + + while(t < n){ + m = read(fd, buf+t, n-t); + if(m < 0) + return -1; + if(m == 0) + break; + t += m; + } + return t; +} + +int +vacWrite(VtSession *z, uchar score[VtScoreSize], int type, uchar *buf, int n) +{ +assert(n > 0); + if(nowrite) { + vtSha1(score, buf, n); + return 1; + } + if(!vtWrite(z, score, type, buf, n)) + return 0; + if(!vtSha1Check(score, buf, n)) { + uchar score2[VtScoreSize]; + + vtSha1(score2, buf, n); +fprint(2, "vtSha1Check: n = %d %V %V\n", n, score, score2); + vtSetError("vtSha1Check failed"); + return 0; + } + return 1; +} + + +static int +vac(VtSession *z, char *argv[]) +{ + DirSink *dsink, *ds; + MetaSink *ms; + VtRoot root; + uchar score[VtScoreSize], buf[VtRootSize]; + char cwd[2048]; + int cd, i; + char *cp2, *cp; + VacFS *fs; + VacFile *vff; + int fd; + Dir *dir; + VacDir vd; + + if(getwd(cwd, sizeof(cwd)) == 0) + sysfatal("can't find current directory: %r\n"); + + dsink = dirSinkAlloc(z, bsize, bsize); + + fs = nil; + if(dfile != nil) { + fs = vfsOpen(z, dfile, 1, 10000); + if(fs == nil) + fprint(2, "could not open diff: %s: %s\n", dfile, vtGetError()); + } + + + if(oname != nil) { + fd = create(oname, OWRITE, 0666); + if(fd < 0) + sysfatal("could not create file: %s: %r", oname); + } else + fd = 1; + + dir = dirfstat(fd); + if(dir == nil) + sysfatal("dirfstat failed: %r"); + + for(; *argv; argv++) { + cp2 = *argv; + cd = 0; + for (cp = *argv; *cp; cp++) + if (*cp == '/') + cp2 = cp; + if (cp2 != *argv) { + *cp2 = '\0'; + chdir(*argv); + *cp2 = '/'; + cp2++; + cd = 1; + } + vff = nil; + if(fs) + vff = vfOpen(fs, cp2); + vacFile(dsink, argv[0], cp2, vff); + if(vff) + vfDecRef(vff); + if(cd && chdir(cwd) < 0) + sysfatal("can't cd back to %s: %r\n", cwd); + } + + if(isi) { + vff = nil; + if(fs) + vff = vfOpen(fs, isi); + vacStdin(dsink, isi, vff); + if(vff) + vfDecRef(vff); + } + + dirSinkClose(dsink); + + /* build meta information for the root */ + ms = metaSinkAlloc(z, bsize, bsize); + /* fake into a directory */ + dir->mode |= (dir->mode&0444)>>2; + dir->qid.type |= QTDIR; + dir->mode |= DMDIR; + plan9ToVacDir(&vd, dir, 0, fileid++); + if(strcmp(vd.elem, "/") == 0){ + vtMemFree(vd.elem); + vd.elem = vtStrDup("root"); + } + metaSinkWriteDir(ms, &vd); + vdCleanup(&vd); + metaSinkClose(ms); + + ds = dirSinkAlloc(z, bsize, bsize); + dirSinkWriteSink(ds, dsink->sink); + dirSinkWriteSink(ds, dsink->msink->sink); + dirSinkWriteSink(ds, ms->sink); + dirSinkClose(ds); + + memset(&root, 0, sizeof(root)); + root.version = VtRootVersion; + strncpy(root.name, dir->name, sizeof(root.name)); + root.name[sizeof(root.name)-1] = 0; + free(dir); + sprint(root.type, "vac"); + memmove(root.score, ds->sink->dir.score, VtScoreSize); + root.blockSize = maxbsize; + if(fs != nil) + vfsGetScore(fs, root.prev); + + metaSinkFree(ms); + dirSinkFree(ds); + dirSinkFree(dsink); + if(fs != nil) + vfsClose(fs); + + vtRootPack(&root, buf); + if(!vacWrite(z, score, VtRootType, buf, VtRootSize)) + vtFatal("vacWrite failed: %s", vtGetError()); + + fprint(fd, "vac:"); + for(i=0; i<VtScoreSize; i++) + fprint(fd, "%.2x", score[i]); + fprint(fd, "\n"); + + /* avoid remove at cleanup */ + oname = nil; + return 1; +} + +static int +isExcluded(char *name) +{ + int bot, top, i, x; + + bot = 0; + top = nexclude; + while(bot < top) { + i = (bot+top)>>1; + x = strcmp(exclude[i], name); + if(x == 0) + return 1; + if(x < 0) + bot = i + 1; + else /* x > 0 */ + top = i; + } + return 0; +} + +static void +vacFile(DirSink *dsink, char *lname, char *sname, VacFile *vf) +{ + int fd; + Dir *dir; + VacDir vd; + ulong entry; + + if(isExcluded(lname)) { + warn("excluding: %s", lname); + return; + } + + if(merge && vacMerge(dsink, lname, sname)) + return; + + fd = open(sname, OREAD); + if(fd < 0) { + warn("could not open file: %s: %s", lname, vtOSError()); + return; + } + + if(verbose) + fprint(2, "%s\n", lname); + + dir = dirfstat(fd); + if(dir == nil) { + warn("can't stat %s: %r", lname); + close(fd); + return; + } + + entry = dsink->nentry; + + if(dir->mode & DMDIR) + vacDir(dsink, fd, lname, sname, vf); + else + vacData(dsink, fd, lname, vf, dir); + + plan9ToVacDir(&vd, dir, entry, fileid++); + metaSinkWriteDir(dsink->msink, &vd); + vdCleanup(&vd); + + free(dir); + close(fd); +} + +static void +vacStdin(DirSink *dsink, char *name, VacFile *vf) +{ + Dir *dir; + VacDir vd; + ulong entry; + + if(verbose) + fprint(2, "%s\n", "<stdio>"); + + dir = dirfstat(0); + if(dir == nil) { + warn("can't stat <stdio>: %r"); + return; + } + + entry = dsink->nentry; + + vacData(dsink, 0, "<stdin>", vf, dir); + + plan9ToVacDir(&vd, dir, entry, fileid++); + vd.elem = vtStrDup(name); + metaSinkWriteDir(dsink->msink, &vd); + vdCleanup(&vd); + + free(dir); +} + +static ulong +vacDataSkip(Sink *sink, VacFile *vf, int fd, ulong blocks, uchar *buf, char *lname) +{ + int n; + ulong i; + uchar score[VtScoreSize]; + + /* skip blocks for append only files */ + if(seek(fd, (blocks-1)*bsize, 0) != (blocks-1)*bsize) { + warn("error seeking: %s", lname); + goto Err; + } + n = readBlock(fd, buf, bsize); + if(n < bsize) { + warn("error checking append only file: %s", lname); + goto Err; + } + if(!vfGetBlockScore(vf, blocks-1, score) || !vtSha1Check(score, buf, n)) { + warn("last block of append file did not match: %s", lname); + goto Err; + } + + for(i=0; i<blocks; i++) { + if(!vfGetBlockScore(vf, i, score)) { + warn("could not get score: %s: %lud", lname, i); + seek(fd, i*bsize, 0); + return i; + } + stats.skip++; + sinkWriteScore(sink, score, bsize); + } + + return i; +Err: + seek(fd, 0, 0); + return 0; +} + +static void +vacData(DirSink *dsink, int fd, char *lname, VacFile *vf, Dir *dir) +{ + uchar *buf; + Sink *sink; + int n; + uchar score[VtScoreSize]; + ulong block, same; + VacDir vd; + ulong vfblocks; + + vfblocks = 0; + if(vf != nil && qdiff) { + vfGetDir(vf, &vd); + if(vd.mtime == dir->mtime) + if(vd.size == dir->length) + if(!vd.plan9 || /* vd.p9path == dir->qid.path && */ vd.p9version == dir->qid.vers) + if(dirSinkWriteFile(dsink, vf)) { + stats.sfile++; + vdCleanup(&vd); + return; + } + + /* look for an append only file */ + if((dir->mode&DMAPPEND) != 0) + if(vd.size < dir->length) + if(vd.plan9) + if(vd.p9path == dir->qid.path) + vfblocks = vd.size/bsize; + + vdCleanup(&vd); + } + stats.file++; + + buf = vtMemAlloc(bsize); + sink = sinkAlloc(dsink->sink->z, bsize, bsize); + block = 0; + same = stats.sdata+stats.skip; + + if(vfblocks > 1) + block += vacDataSkip(sink, vf, fd, vfblocks, buf, lname); + +if(0) fprint(2, "vacData: %s: %ld\n", lname, block); + for(;;) { + n = readBlock(fd, buf, bsize); + if(0 && n < 0) + warn("file truncated due to read error: %s: %s", lname, vtOSError()); + if(n <= 0) + break; + if(vf != nil && vfGetBlockScore(vf, block, score) && vtSha1Check(score, buf, n)) { + stats.sdata++; + sinkWriteScore(sink, score, n); + } else + sinkWrite(sink, buf, n); + block++; + } + same = stats.sdata+stats.skip - same; + + if(same && (dir->mode&DMAPPEND) != 0) + if(0)fprint(2, "%s: total %lud same %lud:%lud diff %lud\n", + lname, block, same, vfblocks, block-same); + + sinkClose(sink); + dirSinkWriteSink(dsink, sink); + sinkFree(sink); + free(buf); +} + + +static void +vacDir(DirSink *dsink, int fd, char *lname, char *sname, VacFile *vf) +{ + Dir *dirs; + char *ln, *sn; + int i, nd; + DirSink *ds; + VacFile *vvf; + char *name; + + ds = dirSinkAlloc(dsink->sink->z, bsize, bsize); + while((nd = dirread(fd, &dirs)) > 0){ + for(i = 0; i < nd; i++){ + name = dirs[i].name; + /* check for bad file names */ + if(name[0] == 0 || strcmp(name, ".") == 0 || strcmp(name, "..") == 0) + continue; + ln = vtMemAlloc(strlen(lname) + strlen(name) + 2); + sn = vtMemAlloc(strlen(sname) + strlen(name) + 2); + sprint(ln, "%s/%s", lname, name); + sprint(sn, "%s/%s", sname, name); + if(vf != nil) + vvf = vfWalk(vf, name); + else + vvf = nil; + vacFile(ds, ln, sn, vvf); + if(vvf != nil) + vfDecRef(vvf); + vtMemFree(ln); + vtMemFree(sn); + } + free(dirs); + } + dirSinkClose(ds); + dirSinkWriteSink(dsink, ds->sink); + dirSinkWriteSink(dsink, ds->msink->sink); + dirSinkFree(ds); +} + +static int +vacMergeFile(DirSink *dsink, VacFile *vf, VacDir *dir, uvlong offset, uvlong *max) +{ + uchar buf[VtEntrySize]; + VtEntry dd, md; + int e; + + if(vfRead(vf, buf, VtEntrySize, (uvlong)dir->entry*VtEntrySize) != VtEntrySize) { + warn("could not read venti dir entry: %s\n", dir->elem); + return 0; + } + vtEntryUnpack(&dd, buf, 0); + + if(dir->mode & ModeDir) { + e = dir->mentry; + if(e == 0) + e = dir->entry + 1; + + if(vfRead(vf, buf, VtEntrySize, e*VtEntrySize) != VtEntrySize) { + warn("could not read venti dir entry: %s\n", dir->elem); + return 0; + } + vtEntryUnpack(&md, buf, 0); + } + + /* max might incorrect in some old dumps */ + if(dir->qid >= *max) { + warn("qid out of range: %s", dir->elem); + *max = dir->qid; + } + + dir->qid += offset; + dir->entry = dsink->nentry; + + if(dir->qidSpace) { + dir->qidOffset += offset; + } else { + dir->qidSpace = 1; + dir->qidOffset = offset; + dir->qidMax = *max; + } + + dirSinkWrite(dsink, &dd); + if(dir->mode & ModeDir) + dirSinkWrite(dsink, &md); + metaSinkWriteDir(dsink->msink, dir); + + return 1; +} + +static int +vacMerge(DirSink *dsink, char *lname, char *sname) +{ + char *p; + VacFS *fs; + VacFile *vf; + VacDirEnum *d; + VacDir dir; + uvlong max; + + p = strrchr(sname, '.'); + if(p == 0 || strcmp(p, ".vac")) + return 0; + + d = nil; + fs = vfsOpen(dsink->sink->z, sname, 1, 100); + if(fs == nil) + return 0; + + vf = vfOpen(fs, "/"); + if(vf == nil) + goto Done; + max = vfGetId(vf); + d = vdeOpen(fs, "/"); + if(d == nil) + goto Done; + + if(verbose) + fprint(2, "merging: %s\n", lname); + + if(maxbsize < vfsGetBlockSize(fs)) + maxbsize = vfsGetBlockSize(fs); + + for(;;) { + if(vdeRead(d, &dir, 1) < 1) + break; + vacMergeFile(dsink, vf, &dir, fileid, &max); + vdCleanup(&dir); + } + fileid += max; + +Done: + if(d != nil) + vdeFree(d); + if(vf != nil) + vfDecRef(vf); + vfsClose(fs); + return 1; +} + +Sink * +sinkAlloc(VtSession *z, int psize, int dsize) +{ + Sink *k; + int i; + + if(psize < 512 || psize > VtMaxLumpSize) + vtFatal("sinkAlloc: bad psize"); + if(dsize < 512 || dsize > VtMaxLumpSize) + vtFatal("sinkAlloc: bad psize"); + + psize = VtScoreSize*(psize/VtScoreSize); + + k = vtMemAllocZ(sizeof(Sink)); + k->z = z; + k->dir.flags = VtEntryActive; + k->dir.psize = psize; + k->dir.dsize = dsize; + k->buf = vtMemAllocZ(VtPointerDepth*k->dir.psize + VtScoreSize); + for(i=0; i<=VtPointerDepth; i++) + k->pbuf[i] = k->buf + i*k->dir.psize; + return k; +} + +void +sinkWriteScore(Sink *k, uchar score[VtScoreSize], int n) +{ + int i; + uchar *p; + VtEntry *d; + + memmove(k->pbuf[0], score, VtScoreSize); + + d = &k->dir; + + for(i=0; i<VtPointerDepth; i++) { + k->pbuf[i] += VtScoreSize; + if(k->pbuf[i] < k->buf + d->psize*(i+1)) + break; + if(i == VtPointerDepth-1) + vtFatal("file too big"); + p = k->buf+i*d->psize; + stats.meta++; + if(!vacWrite(k->z, k->pbuf[i+1], VtPointerType0+i, p, d->psize)) + vtFatal("vacWrite failed: %s", vtGetError()); + k->pbuf[i] = p; + } + + /* round size up to multiple of dsize */ + d->size = d->dsize * ((d->size + d->dsize-1)/d->dsize); + + d->size += n; +} + +void +sinkWrite(Sink *k, uchar *p, int n) +{ + int type; + uchar score[VtScoreSize]; + + if(n > k->dir.dsize) + vtFatal("sinkWrite: size too big"); + + if(k->dir.flags & VtEntryDir) { + type = VtDirType; + stats.meta++; + } else { + type = VtDataType; + stats.data++; + } + if(!vacWrite(k->z, score, type, p, n)) + vtFatal("vacWrite failed: %s", vtGetError()); + + sinkWriteScore(k, score, n); +} + +static int +sizeToDepth(uvlong s, int psize, int dsize) +{ + int np; + int d; + + /* determine pointer depth */ + np = psize/VtScoreSize; + s = (s + dsize - 1)/dsize; + for(d = 0; s > 1; d++) + s = (s + np - 1)/np; + return d; +} + +void +sinkClose(Sink *k) +{ + int i, n; + uchar *p; + VtEntry *kd; + + kd = &k->dir; + + /* empty */ + if(kd->size == 0) { + memmove(kd->score, vtZeroScore, VtScoreSize); + return; + } + + for(n=VtPointerDepth-1; n>0; n--) + if(k->pbuf[n] > k->buf + kd->psize*n) + break; + + kd->depth = sizeToDepth(kd->size, kd->psize, kd->dsize); + + /* skip full part of tree */ + for(i=0; i<n && k->pbuf[i] == k->buf + kd->psize*i; i++) + ; + + /* is the tree completely full */ + if(i == n && k->pbuf[n] == k->buf + kd->psize*n + VtScoreSize) { + memmove(kd->score, k->pbuf[n] - VtScoreSize, VtScoreSize); + return; + } + n++; + + /* clean up the edge */ + for(; i<n; i++) { + p = k->buf+i*kd->psize; + stats.meta++; + if(!vacWrite(k->z, k->pbuf[i+1], VtPointerType0+i, p, k->pbuf[i]-p)) + vtFatal("vacWrite failed: %s", vtGetError()); + k->pbuf[i+1] += VtScoreSize; + } + memmove(kd->score, k->pbuf[i] - VtScoreSize, VtScoreSize); +} + +void +sinkFree(Sink *k) +{ + vtMemFree(k->buf); + vtMemFree(k); +} + +DirSink * +dirSinkAlloc(VtSession *z, int psize, int dsize) +{ + DirSink *k; + int ds; + + ds = VtEntrySize*(dsize/VtEntrySize); + + k = vtMemAllocZ(sizeof(DirSink)); + k->sink = sinkAlloc(z, psize, ds); + k->sink->dir.flags |= VtEntryDir; + k->msink = metaSinkAlloc(z, psize, dsize); + k->buf = vtMemAlloc(ds); + k->p = k->buf; + k->ep = k->buf + ds; + return k; +} + +void +dirSinkWrite(DirSink *k, VtEntry *dir) +{ + if(k->p + VtEntrySize > k->ep) { + sinkWrite(k->sink, k->buf, k->p - k->buf); + k->p = k->buf; + } + vtEntryPack(dir, k->p, 0); + k->nentry++; + k->p += VtEntrySize; +} + +void +dirSinkWriteSink(DirSink *k, Sink *sink) +{ + dirSinkWrite(k, &sink->dir); +} + +int +dirSinkWriteFile(DirSink *k, VacFile *vf) +{ + VtEntry dir; + + if(!vfGetVtEntry(vf, &dir)) + return 0; + dirSinkWrite(k, &dir); + return 1; +} + +void +dirSinkClose(DirSink *k) +{ + metaSinkClose(k->msink); + if(k->p != k->buf) + sinkWrite(k->sink, k->buf, k->p - k->buf); + sinkClose(k->sink); +} + +void +dirSinkFree(DirSink *k) +{ + sinkFree(k->sink); + metaSinkFree(k->msink); + vtMemFree(k->buf); + vtMemFree(k); +} + +MetaSink * +metaSinkAlloc(VtSession *z, int psize, int dsize) +{ + MetaSink *k; + + k = vtMemAllocZ(sizeof(MetaSink)); + k->sink = sinkAlloc(z, psize, dsize); + k->buf = vtMemAlloc(dsize); + k->maxindex = dsize/100; /* 100 byte entries seems reasonable */ + if(k->maxindex < 1) + k->maxindex = 1; + k->rp = k->p = k->buf + MetaHeaderSize + k->maxindex*MetaIndexSize; + k->ep = k->buf + dsize; + return k; +} + +/* hack to get base to compare routine - not reentrant */ +uchar *blockBase; + +int +dirCmp(void *p0, void *p1) +{ + uchar *q0, *q1; + int n0, n1, r; + + /* name is first element of entry */ + q0 = p0; + q0 = blockBase + (q0[0]<<8) + q0[1]; + n0 = (q0[6]<<8) + q0[7]; + q0 += 8; + + q1 = p1; + q1 = blockBase + (q1[0]<<8) + q1[1]; + n1 = (q1[6]<<8) + q1[7]; + q1 += 8; + + if(n0 == n1) + return memcmp(q0, q1, n0); + else if (n0 < n1) { + r = memcmp(q0, q1, n0); + return (r==0)?1:r; + } else { + r = memcmp(q0, q1, n1); + return (r==0)?-1:r; + } +} + +void +metaSinkFlush(MetaSink *k) +{ + uchar *p; + int n; + MetaBlock mb; + + if(k->nindex == 0) + return; + assert(k->nindex <= k->maxindex); + + p = k->buf; + n = k->rp - p; + + mb.size = n; + mb.free = 0; + mb.nindex = k->nindex; + mb.maxindex = k->maxindex; + mb.buf = p; + mbPack(&mb); + + p += MetaHeaderSize; + + /* XXX this is not reentrant! */ + blockBase = k->buf; + qsort(p, k->nindex, MetaIndexSize, dirCmp); + p += k->nindex*MetaIndexSize; + + memset(p, 0, (k->maxindex-k->nindex)*MetaIndexSize); + p += (k->maxindex-k->nindex)*MetaIndexSize; + + sinkWrite(k->sink, k->buf, n); + + /* move down partial entry */ + n = k->p - k->rp; + memmove(p, k->rp, n); + k->rp = p; + k->p = p + n; + k->nindex = 0; +} + +void +metaSinkPutc(MetaSink *k, int c) +{ + if(k->p+1 > k->ep) + metaSinkFlush(k); + if(k->p+1 > k->ep) + vtFatal("directory entry too large"); + k->p[0] = c; + k->p++; +} + +void +metaSinkPutString(MetaSink *k, char *s) +{ + int n = strlen(s); + metaSinkPutc(k, n>>8); + metaSinkPutc(k, n); + metaSinkWrite(k, (uchar*)s, n); +} + +void +metaSinkPutUint32(MetaSink *k, ulong x) +{ + metaSinkPutc(k, x>>24); + metaSinkPutc(k, x>>16); + metaSinkPutc(k, x>>8); + metaSinkPutc(k, x); +} + +void +metaSinkPutUint64(MetaSink *k, uvlong x) +{ + metaSinkPutUint32(k, x>>32); + metaSinkPutUint32(k, x); +} + +void +metaSinkWrite(MetaSink *k, uchar *data, int n) +{ + if(k->p + n > k->ep) + metaSinkFlush(k); + if(k->p + n > k->ep) + vtFatal("directory entry too large"); + + memmove(k->p, data, n); + k->p += n; +} + +void +metaSinkWriteDir(MetaSink *ms, VacDir *dir) +{ + metaSinkPutUint32(ms, DirMagic); + metaSinkPutc(ms, Version>>8); + metaSinkPutc(ms, Version); + metaSinkPutString(ms, dir->elem); + metaSinkPutUint32(ms, dir->entry); + metaSinkPutUint64(ms, dir->qid); + metaSinkPutString(ms, dir->uid); + metaSinkPutString(ms, dir->gid); + metaSinkPutString(ms, dir->mid); + metaSinkPutUint32(ms, dir->mtime); + metaSinkPutUint32(ms, dir->mcount); + metaSinkPutUint32(ms, dir->ctime); + metaSinkPutUint32(ms, dir->atime); + metaSinkPutUint32(ms, dir->mode); + + if(dir->plan9) { + metaSinkPutc(ms, DirPlan9Entry); /* plan9 extra info */ + metaSinkPutc(ms, 0); /* plan9 extra size */ + metaSinkPutc(ms, 12); /* plan9 extra size */ + metaSinkPutUint64(ms, dir->p9path); + metaSinkPutUint32(ms, dir->p9version); + } + + if(dir->qidSpace != 0) { + metaSinkPutc(ms, DirQidSpaceEntry); + metaSinkPutc(ms, 0); + metaSinkPutc(ms, 16); + metaSinkPutUint64(ms, dir->qidOffset); + metaSinkPutUint64(ms, dir->qidMax); + } + + if(dir->gen != 0) { + metaSinkPutc(ms, DirGenEntry); + metaSinkPutc(ms, 0); + metaSinkPutc(ms, 4); + metaSinkPutUint32(ms, dir->gen); + } + + metaSinkEOR(ms); +} + + +void +plan9ToVacDir(VacDir *vd, Dir *dir, ulong entry, uvlong qid) +{ + memset(vd, 0, sizeof(VacDir)); + + vd->elem = vtStrDup(dir->name); + vd->entry = entry; + vd->qid = qid; + vd->uid = vtStrDup(dir->uid); + vd->gid = vtStrDup(dir->gid); + vd->mid = vtStrDup(dir->muid); + vd->mtime = dir->mtime; + vd->mcount = 0; + vd->ctime = dir->mtime; /* ctime: not available on plan 9 */ + vd->atime = dir->atime; + + vd->mode = dir->mode & 0777; + if(dir->mode & DMDIR) + vd->mode |= ModeDir; + if(dir->mode & DMAPPEND) + vd->mode |= ModeAppend; + if(dir->mode & DMEXCL) + vd->mode |= ModeExclusive; + + vd->plan9 = 1; + vd->p9path = dir->qid.path; + vd->p9version = dir->qid.vers; +} + + +void +metaSinkEOR(MetaSink *k) +{ + uchar *p; + int o, n; + + p = k->buf + MetaHeaderSize; + p += k->nindex * MetaIndexSize; + o = k->rp-k->buf; /* offset from start of block */ + n = k->p-k->rp; /* size of entry */ + p[0] = o >> 8; + p[1] = o; + p[2] = n >> 8; + p[3] = n; + k->rp = k->p; + k->nindex++; + if(k->nindex == k->maxindex) + metaSinkFlush(k); +} + +void +metaSinkClose(MetaSink *k) +{ + metaSinkFlush(k); + sinkClose(k->sink); +} + +void +metaSinkFree(MetaSink *k) +{ + sinkFree(k->sink); + vtMemFree(k->buf); + vtMemFree(k); +} + +static void +warn(char *fmt, ...) +{ + va_list arg; + + va_start(arg, fmt); + fprint(2, "%s: ", argv0); + vfprint(2, fmt, arg); + fprint(2, "\n"); + va_end(arg); +} + +static void +cleanup(void) +{ + if(oname != nil) + remove(oname); +} + +#define TWID64 ((u64int)~(u64int)0) + +static u64int +unittoull(char *s) +{ + char *es; + u64int n; + + if(s == nil) + return TWID64; + n = strtoul(s, &es, 0); + if(*es == 'k' || *es == 'K'){ + n *= 1024; + es++; + }else if(*es == 'm' || *es == 'M'){ + n *= 1024*1024; + es++; + }else if(*es == 'g' || *es == 'G'){ + n *= 1024*1024*1024; + es++; + } + if(*es != '\0') + return TWID64; + return n; +} diff --git a/src/cmd/vac/vac.c b/src/cmd/vac/vac.c new file mode 100644 index 00000000..8153c37b --- /dev/null +++ b/src/cmd/vac/vac.c @@ -0,0 +1,1024 @@ +#include <u.h> +#include <libc.h> +#include <venti.h> + +int bsize; +char *host; +VtConn *z; + +void +usage(void) +{ + fprint(2, "usage: vac [-b blocksize] [-h host] file\n"); +} + +void +main(int argc, char *argv[]) +{ + ARGBEGIN{ + default: + usage(); + case 'b': + bsize = unittoull(EARGF(usage())); + break; + case 'h': + host = EARGF(usage()); + break; + }ARGEND + + if(bsize < 512) + bsize = 512; + if(bsize > VtMaxLumpSize) + bsize = VtMaxLumpSize; + maxbsize = bsize; + + vtAttach(); + + fmtinstall('V', vtScoreFmt); + fmtinstall('R', vtErrFmt); + + z = vtDial(host, 0); + if(z == nil) + vtFatal("could not connect to server: %R"); + + if(!vtConnect(z, 0)) + vtFatal("vtConnect: %R"); + + qsort(exclude, nexclude, sizeof(char*), strpCmp); + + vac(z, argv); + if(!vtSync(z)) + fprint(2, "warning: could not ask server to flush pending writes: %R\n"); + + if(statsFlag) + fprint(2, "files %ld:%ld data %ld:%ld:%ld meta %ld\n", stats.file, stats.sfile, + stats.data, stats.skip, stats.sdata, stats.meta); +//packetStats(); + vtClose(z); + vtDetach(); + + exits(0); +} + +static int +vac(VtSession *z, char *argv[]) +{ + DirSink *dsink, *ds; + MetaSink *ms; + VtRoot root; + uchar score[VtScoreSize], buf[VtRootSize]; + char cwd[2048]; + int cd, i; + char *cp2, *cp; + VacFS *fs; + VacFile *vff; + int fd; + Dir *dir; + VacDir vd; + + if(getwd(cwd, sizeof(cwd)) == 0) + sysfatal("can't find current directory: %r\n"); + + dsink = dirSinkAlloc(z, bsize, bsize); + + fs = nil; + if(dfile != nil) { + fs = vfsOpen(z, dfile, 1, 10000); + if(fs == nil) + fprint(2, "could not open diff: %s: %s\n", dfile, vtGetError()); + } + + + if(oname != nil) { + fd = create(oname, OWRITE, 0666); + if(fd < 0) + sysfatal("could not create file: %s: %r", oname); + } else + fd = 1; + + dir = dirfstat(fd); + if(dir == nil) + sysfatal("dirfstat failed: %r"); + + for(; *argv; argv++) { + cp2 = *argv; + cd = 0; + for (cp = *argv; *cp; cp++) + if (*cp == '/') + cp2 = cp; + if (cp2 != *argv) { + *cp2 = '\0'; + chdir(*argv); + *cp2 = '/'; + cp2++; + cd = 1; + } + vff = nil; + if(fs) + vff = vfOpen(fs, cp2); + vacFile(dsink, argv[0], cp2, vff); + if(vff) + vfDecRef(vff); + if(cd && chdir(cwd) < 0) + sysfatal("can't cd back to %s: %r\n", cwd); + } + + if(isi) { + vff = nil; + if(fs) + vff = vfOpen(fs, isi); + vacStdin(dsink, isi, vff); + if(vff) + vfDecRef(vff); + } + + dirSinkClose(dsink); + + /* build meta information for the root */ + ms = metaSinkAlloc(z, bsize, bsize); + /* fake into a directory */ + dir->mode |= (dir->mode&0444)>>2; + dir->qid.type |= QTDIR; + dir->mode |= DMDIR; + plan9ToVacDir(&vd, dir, 0, fileid++); + if(strcmp(vd.elem, "/") == 0){ + vtMemFree(vd.elem); + vd.elem = vtStrDup("root"); + } + metaSinkWriteDir(ms, &vd); + vdCleanup(&vd); + metaSinkClose(ms); + + ds = dirSinkAlloc(z, bsize, bsize); + dirSinkWriteSink(ds, dsink->sink); + dirSinkWriteSink(ds, dsink->msink->sink); + dirSinkWriteSink(ds, ms->sink); + dirSinkClose(ds); + + memset(&root, 0, sizeof(root)); + root.version = VtRootVersion; + strncpy(root.name, dir->name, sizeof(root.name)); + root.name[sizeof(root.name)-1] = 0; + free(dir); + sprint(root.type, "vac"); + memmove(root.score, ds->sink->dir.score, VtScoreSize); + root.blockSize = maxbsize; + if(fs != nil) + vfsGetScore(fs, root.prev); + + metaSinkFree(ms); + dirSinkFree(ds); + dirSinkFree(dsink); + if(fs != nil) + vfsClose(fs); + + vtRootPack(&root, buf); + if(!vacWrite(z, score, VtRootType, buf, VtRootSize)) + vtFatal("vacWrite failed: %s", vtGetError()); + + fprint(fd, "vac:"); + for(i=0; i<VtScoreSize; i++) + fprint(fd, "%.2x", score[i]); + fprint(fd, "\n"); + + /* avoid remove at cleanup */ + oname = nil; + return 1; +} + +static int +isExcluded(char *name) +{ + int bot, top, i, x; + + bot = 0; + top = nexclude; + while(bot < top) { + i = (bot+top)>>1; + x = strcmp(exclude[i], name); + if(x == 0) + return 1; + if(x < 0) + bot = i + 1; + else /* x > 0 */ + top = i; + } + return 0; +} + +static void +vacFile(DirSink *dsink, char *lname, char *sname, VacFile *vf) +{ + int fd; + Dir *dir; + VacDir vd; + ulong entry; + + if(isExcluded(lname)) { + warn("excluding: %s", lname); + return; + } + + if(merge && vacMerge(dsink, lname, sname)) + return; + + fd = open(sname, OREAD); + if(fd < 0) { + warn("could not open file: %s: %s", lname, vtOSError()); + return; + } + + if(verbose) + fprint(2, "%s\n", lname); + + dir = dirfstat(fd); + if(dir == nil) { + warn("can't stat %s: %r", lname); + close(fd); + return; + } + + entry = dsink->nentry; + + if(dir->mode & DMDIR) + vacDir(dsink, fd, lname, sname, vf); + else + vacData(dsink, fd, lname, vf, dir); + + plan9ToVacDir(&vd, dir, entry, fileid++); + metaSinkWriteDir(dsink->msink, &vd); + vdCleanup(&vd); + + free(dir); + close(fd); +} + +static void +vacStdin(DirSink *dsink, char *name, VacFile *vf) +{ + Dir *dir; + VacDir vd; + ulong entry; + + if(verbose) + fprint(2, "%s\n", "<stdio>"); + + dir = dirfstat(0); + if(dir == nil) { + warn("can't stat <stdio>: %r"); + return; + } + + entry = dsink->nentry; + + vacData(dsink, 0, "<stdin>", vf, dir); + + plan9ToVacDir(&vd, dir, entry, fileid++); + vd.elem = vtStrDup(name); + metaSinkWriteDir(dsink->msink, &vd); + vdCleanup(&vd); + + free(dir); +} + +static ulong +vacDataSkip(Sink *sink, VacFile *vf, int fd, ulong blocks, uchar *buf, char *lname) +{ + int n; + ulong i; + uchar score[VtScoreSize]; + + /* skip blocks for append only files */ + if(seek(fd, (blocks-1)*bsize, 0) != (blocks-1)*bsize) { + warn("error seeking: %s", lname); + goto Err; + } + n = readBlock(fd, buf, bsize); + if(n < bsize) { + warn("error checking append only file: %s", lname); + goto Err; + } + if(!vfGetBlockScore(vf, blocks-1, score) || !vtSha1Check(score, buf, n)) { + warn("last block of append file did not match: %s", lname); + goto Err; + } + + for(i=0; i<blocks; i++) { + if(!vfGetBlockScore(vf, i, score)) { + warn("could not get score: %s: %lud", lname, i); + seek(fd, i*bsize, 0); + return i; + } + stats.skip++; + sinkWriteScore(sink, score, bsize); + } + + return i; +Err: + seek(fd, 0, 0); + return 0; +} + +static void +vacData(DirSink *dsink, int fd, char *lname, VacFile *vf, Dir *dir) +{ + uchar *buf; + Sink *sink; + int n; + uchar score[VtScoreSize]; + ulong block, same; + VacDir vd; + ulong vfblocks; + + vfblocks = 0; + if(vf != nil && qdiff) { + vfGetDir(vf, &vd); + if(vd.mtime == dir->mtime) + if(vd.size == dir->length) + if(!vd.plan9 || /* vd.p9path == dir->qid.path && */ vd.p9version == dir->qid.vers) + if(dirSinkWriteFile(dsink, vf)) { + stats.sfile++; + vdCleanup(&vd); + return; + } + + /* look for an append only file */ + if((dir->mode&DMAPPEND) != 0) + if(vd.size < dir->length) + if(vd.plan9) + if(vd.p9path == dir->qid.path) + vfblocks = vd.size/bsize; + + vdCleanup(&vd); + } + stats.file++; + + buf = vtMemAlloc(bsize); + sink = sinkAlloc(dsink->sink->z, bsize, bsize); + block = 0; + same = stats.sdata+stats.skip; + + if(vfblocks > 1) + block += vacDataSkip(sink, vf, fd, vfblocks, buf, lname); + +if(0) fprint(2, "vacData: %s: %ld\n", lname, block); + for(;;) { + n = readBlock(fd, buf, bsize); + if(0 && n < 0) + warn("file truncated due to read error: %s: %s", lname, vtOSError()); + if(n <= 0) + break; + if(vf != nil && vfGetBlockScore(vf, block, score) && vtSha1Check(score, buf, n)) { + stats.sdata++; + sinkWriteScore(sink, score, n); + } else + sinkWrite(sink, buf, n); + block++; + } + same = stats.sdata+stats.skip - same; + + if(same && (dir->mode&DMAPPEND) != 0) + if(0)fprint(2, "%s: total %lud same %lud:%lud diff %lud\n", + lname, block, same, vfblocks, block-same); + + sinkClose(sink); + dirSinkWriteSink(dsink, sink); + sinkFree(sink); + free(buf); +} + + +static void +vacDir(DirSink *dsink, int fd, char *lname, char *sname, VacFile *vf) +{ + Dir *dirs; + char *ln, *sn; + int i, nd; + DirSink *ds; + VacFile *vvf; + char *name; + + ds = dirSinkAlloc(dsink->sink->z, bsize, bsize); + while((nd = dirread(fd, &dirs)) > 0){ + for(i = 0; i < nd; i++){ + name = dirs[i].name; + /* check for bad file names */ + if(name[0] == 0 || strcmp(name, ".") == 0 || strcmp(name, "..") == 0) + continue; + ln = vtMemAlloc(strlen(lname) + strlen(name) + 2); + sn = vtMemAlloc(strlen(sname) + strlen(name) + 2); + sprint(ln, "%s/%s", lname, name); + sprint(sn, "%s/%s", sname, name); + if(vf != nil) + vvf = vfWalk(vf, name); + else + vvf = nil; + vacFile(ds, ln, sn, vvf); + if(vvf != nil) + vfDecRef(vvf); + vtMemFree(ln); + vtMemFree(sn); + } + free(dirs); + } + dirSinkClose(ds); + dirSinkWriteSink(dsink, ds->sink); + dirSinkWriteSink(dsink, ds->msink->sink); + dirSinkFree(ds); +} + +static int +vacMergeFile(DirSink *dsink, VacFile *vf, VacDir *dir, uvlong offset, uvlong *max) +{ + uchar buf[VtEntrySize]; + VtEntry dd, md; + int e; + + if(vfRead(vf, buf, VtEntrySize, (uvlong)dir->entry*VtEntrySize) != VtEntrySize) { + warn("could not read venti dir entry: %s\n", dir->elem); + return 0; + } + vtEntryUnpack(&dd, buf, 0); + + if(dir->mode & ModeDir) { + e = dir->mentry; + if(e == 0) + e = dir->entry + 1; + + if(vfRead(vf, buf, VtEntrySize, e*VtEntrySize) != VtEntrySize) { + warn("could not read venti dir entry: %s\n", dir->elem); + return 0; + } + vtEntryUnpack(&md, buf, 0); + } + + /* max might incorrect in some old dumps */ + if(dir->qid >= *max) { + warn("qid out of range: %s", dir->elem); + *max = dir->qid; + } + + dir->qid += offset; + dir->entry = dsink->nentry; + + if(dir->qidSpace) { + dir->qidOffset += offset; + } else { + dir->qidSpace = 1; + dir->qidOffset = offset; + dir->qidMax = *max; + } + + dirSinkWrite(dsink, &dd); + if(dir->mode & ModeDir) + dirSinkWrite(dsink, &md); + metaSinkWriteDir(dsink->msink, dir); + + return 1; +} + +static int +vacMerge(DirSink *dsink, char *lname, char *sname) +{ + char *p; + VacFS *fs; + VacFile *vf; + VacDirEnum *d; + VacDir dir; + uvlong max; + + p = strrchr(sname, '.'); + if(p == 0 || strcmp(p, ".vac")) + return 0; + + d = nil; + fs = vfsOpen(dsink->sink->z, sname, 1, 100); + if(fs == nil) + return 0; + + vf = vfOpen(fs, "/"); + if(vf == nil) + goto Done; + max = vfGetId(vf); + d = vdeOpen(fs, "/"); + if(d == nil) + goto Done; + + if(verbose) + fprint(2, "merging: %s\n", lname); + + if(maxbsize < vfsGetBlockSize(fs)) + maxbsize = vfsGetBlockSize(fs); + + for(;;) { + if(vdeRead(d, &dir, 1) < 1) + break; + vacMergeFile(dsink, vf, &dir, fileid, &max); + vdCleanup(&dir); + } + fileid += max; + +Done: + if(d != nil) + vdeFree(d); + if(vf != nil) + vfDecRef(vf); + vfsClose(fs); + return 1; +} + +Sink * +sinkAlloc(VtSession *z, int psize, int dsize) +{ + Sink *k; + int i; + + if(psize < 512 || psize > VtMaxLumpSize) + vtFatal("sinkAlloc: bad psize"); + if(dsize < 512 || dsize > VtMaxLumpSize) + vtFatal("sinkAlloc: bad psize"); + + psize = VtScoreSize*(psize/VtScoreSize); + + k = vtMemAllocZ(sizeof(Sink)); + k->z = z; + k->dir.flags = VtEntryActive; + k->dir.psize = psize; + k->dir.dsize = dsize; + k->buf = vtMemAllocZ(VtPointerDepth*k->dir.psize + VtScoreSize); + for(i=0; i<=VtPointerDepth; i++) + k->pbuf[i] = k->buf + i*k->dir.psize; + return k; +} + +void +sinkWriteScore(Sink *k, uchar score[VtScoreSize], int n) +{ + int i; + uchar *p; + VtEntry *d; + + memmove(k->pbuf[0], score, VtScoreSize); + + d = &k->dir; + + for(i=0; i<VtPointerDepth; i++) { + k->pbuf[i] += VtScoreSize; + if(k->pbuf[i] < k->buf + d->psize*(i+1)) + break; + if(i == VtPointerDepth-1) + vtFatal("file too big"); + p = k->buf+i*d->psize; + stats.meta++; + if(!vacWrite(k->z, k->pbuf[i+1], VtPointerType0+i, p, d->psize)) + vtFatal("vacWrite failed: %s", vtGetError()); + k->pbuf[i] = p; + } + + /* round size up to multiple of dsize */ + d->size = d->dsize * ((d->size + d->dsize-1)/d->dsize); + + d->size += n; +} + +void +sinkWrite(Sink *k, uchar *p, int n) +{ + int type; + uchar score[VtScoreSize]; + + if(n > k->dir.dsize) + vtFatal("sinkWrite: size too big"); + + if(k->dir.flags & VtEntryDir) { + type = VtDirType; + stats.meta++; + } else { + type = VtDataType; + stats.data++; + } + if(!vacWrite(k->z, score, type, p, n)) + vtFatal("vacWrite failed: %s", vtGetError()); + + sinkWriteScore(k, score, n); +} + +static int +sizeToDepth(uvlong s, int psize, int dsize) +{ + int np; + int d; + + /* determine pointer depth */ + np = psize/VtScoreSize; + s = (s + dsize - 1)/dsize; + for(d = 0; s > 1; d++) + s = (s + np - 1)/np; + return d; +} + +void +sinkClose(Sink *k) +{ + int i, n; + uchar *p; + VtEntry *kd; + + kd = &k->dir; + + /* empty */ + if(kd->size == 0) { + memmove(kd->score, vtZeroScore, VtScoreSize); + return; + } + + for(n=VtPointerDepth-1; n>0; n--) + if(k->pbuf[n] > k->buf + kd->psize*n) + break; + + kd->depth = sizeToDepth(kd->size, kd->psize, kd->dsize); + + /* skip full part of tree */ + for(i=0; i<n && k->pbuf[i] == k->buf + kd->psize*i; i++) + ; + + /* is the tree completely full */ + if(i == n && k->pbuf[n] == k->buf + kd->psize*n + VtScoreSize) { + memmove(kd->score, k->pbuf[n] - VtScoreSize, VtScoreSize); + return; + } + n++; + + /* clean up the edge */ + for(; i<n; i++) { + p = k->buf+i*kd->psize; + stats.meta++; + if(!vacWrite(k->z, k->pbuf[i+1], VtPointerType0+i, p, k->pbuf[i]-p)) + vtFatal("vacWrite failed: %s", vtGetError()); + k->pbuf[i+1] += VtScoreSize; + } + memmove(kd->score, k->pbuf[i] - VtScoreSize, VtScoreSize); +} + +void +sinkFree(Sink *k) +{ + vtMemFree(k->buf); + vtMemFree(k); +} + +DirSink * +dirSinkAlloc(VtSession *z, int psize, int dsize) +{ + DirSink *k; + int ds; + + ds = VtEntrySize*(dsize/VtEntrySize); + + k = vtMemAllocZ(sizeof(DirSink)); + k->sink = sinkAlloc(z, psize, ds); + k->sink->dir.flags |= VtEntryDir; + k->msink = metaSinkAlloc(z, psize, dsize); + k->buf = vtMemAlloc(ds); + k->p = k->buf; + k->ep = k->buf + ds; + return k; +} + +void +dirSinkWrite(DirSink *k, VtEntry *dir) +{ + if(k->p + VtEntrySize > k->ep) { + sinkWrite(k->sink, k->buf, k->p - k->buf); + k->p = k->buf; + } + vtEntryPack(dir, k->p, 0); + k->nentry++; + k->p += VtEntrySize; +} + +void +dirSinkWriteSink(DirSink *k, Sink *sink) +{ + dirSinkWrite(k, &sink->dir); +} + +int +dirSinkWriteFile(DirSink *k, VacFile *vf) +{ + VtEntry dir; + + if(!vfGetVtEntry(vf, &dir)) + return 0; + dirSinkWrite(k, &dir); + return 1; +} + +void +dirSinkClose(DirSink *k) +{ + metaSinkClose(k->msink); + if(k->p != k->buf) + sinkWrite(k->sink, k->buf, k->p - k->buf); + sinkClose(k->sink); +} + +void +dirSinkFree(DirSink *k) +{ + sinkFree(k->sink); + metaSinkFree(k->msink); + vtMemFree(k->buf); + vtMemFree(k); +} + +MetaSink * +metaSinkAlloc(VtSession *z, int psize, int dsize) +{ + MetaSink *k; + + k = vtMemAllocZ(sizeof(MetaSink)); + k->sink = sinkAlloc(z, psize, dsize); + k->buf = vtMemAlloc(dsize); + k->maxindex = dsize/100; /* 100 byte entries seems reasonable */ + if(k->maxindex < 1) + k->maxindex = 1; + k->rp = k->p = k->buf + MetaHeaderSize + k->maxindex*MetaIndexSize; + k->ep = k->buf + dsize; + return k; +} + +/* hack to get base to compare routine - not reentrant */ +uchar *blockBase; + +int +dirCmp(void *p0, void *p1) +{ + uchar *q0, *q1; + int n0, n1, r; + + /* name is first element of entry */ + q0 = p0; + q0 = blockBase + (q0[0]<<8) + q0[1]; + n0 = (q0[6]<<8) + q0[7]; + q0 += 8; + + q1 = p1; + q1 = blockBase + (q1[0]<<8) + q1[1]; + n1 = (q1[6]<<8) + q1[7]; + q1 += 8; + + if(n0 == n1) + return memcmp(q0, q1, n0); + else if (n0 < n1) { + r = memcmp(q0, q1, n0); + return (r==0)?1:r; + } else { + r = memcmp(q0, q1, n1); + return (r==0)?-1:r; + } +} + +void +metaSinkFlush(MetaSink *k) +{ + uchar *p; + int n; + MetaBlock mb; + + if(k->nindex == 0) + return; + assert(k->nindex <= k->maxindex); + + p = k->buf; + n = k->rp - p; + + mb.size = n; + mb.free = 0; + mb.nindex = k->nindex; + mb.maxindex = k->maxindex; + mb.buf = p; + mbPack(&mb); + + p += MetaHeaderSize; + + /* XXX this is not reentrant! */ + blockBase = k->buf; + qsort(p, k->nindex, MetaIndexSize, dirCmp); + p += k->nindex*MetaIndexSize; + + memset(p, 0, (k->maxindex-k->nindex)*MetaIndexSize); + p += (k->maxindex-k->nindex)*MetaIndexSize; + + sinkWrite(k->sink, k->buf, n); + + /* move down partial entry */ + n = k->p - k->rp; + memmove(p, k->rp, n); + k->rp = p; + k->p = p + n; + k->nindex = 0; +} + +void +metaSinkPutc(MetaSink *k, int c) +{ + if(k->p+1 > k->ep) + metaSinkFlush(k); + if(k->p+1 > k->ep) + vtFatal("directory entry too large"); + k->p[0] = c; + k->p++; +} + +void +metaSinkPutString(MetaSink *k, char *s) +{ + int n = strlen(s); + metaSinkPutc(k, n>>8); + metaSinkPutc(k, n); + metaSinkWrite(k, (uchar*)s, n); +} + +void +metaSinkPutUint32(MetaSink *k, ulong x) +{ + metaSinkPutc(k, x>>24); + metaSinkPutc(k, x>>16); + metaSinkPutc(k, x>>8); + metaSinkPutc(k, x); +} + +void +metaSinkPutUint64(MetaSink *k, uvlong x) +{ + metaSinkPutUint32(k, x>>32); + metaSinkPutUint32(k, x); +} + +void +metaSinkWrite(MetaSink *k, uchar *data, int n) +{ + if(k->p + n > k->ep) + metaSinkFlush(k); + if(k->p + n > k->ep) + vtFatal("directory entry too large"); + + memmove(k->p, data, n); + k->p += n; +} + +void +metaSinkWriteDir(MetaSink *ms, VacDir *dir) +{ + metaSinkPutUint32(ms, DirMagic); + metaSinkPutc(ms, Version>>8); + metaSinkPutc(ms, Version); + metaSinkPutString(ms, dir->elem); + metaSinkPutUint32(ms, dir->entry); + metaSinkPutUint64(ms, dir->qid); + metaSinkPutString(ms, dir->uid); + metaSinkPutString(ms, dir->gid); + metaSinkPutString(ms, dir->mid); + metaSinkPutUint32(ms, dir->mtime); + metaSinkPutUint32(ms, dir->mcount); + metaSinkPutUint32(ms, dir->ctime); + metaSinkPutUint32(ms, dir->atime); + metaSinkPutUint32(ms, dir->mode); + + if(dir->plan9) { + metaSinkPutc(ms, DirPlan9Entry); /* plan9 extra info */ + metaSinkPutc(ms, 0); /* plan9 extra size */ + metaSinkPutc(ms, 12); /* plan9 extra size */ + metaSinkPutUint64(ms, dir->p9path); + metaSinkPutUint32(ms, dir->p9version); + } + + if(dir->qidSpace != 0) { + metaSinkPutc(ms, DirQidSpaceEntry); + metaSinkPutc(ms, 0); + metaSinkPutc(ms, 16); + metaSinkPutUint64(ms, dir->qidOffset); + metaSinkPutUint64(ms, dir->qidMax); + } + + if(dir->gen != 0) { + metaSinkPutc(ms, DirGenEntry); + metaSinkPutc(ms, 0); + metaSinkPutc(ms, 4); + metaSinkPutUint32(ms, dir->gen); + } + + metaSinkEOR(ms); +} + + +void +plan9ToVacDir(VacDir *vd, Dir *dir, ulong entry, uvlong qid) +{ + memset(vd, 0, sizeof(VacDir)); + + vd->elem = vtStrDup(dir->name); + vd->entry = entry; + vd->qid = qid; + vd->uid = vtStrDup(dir->uid); + vd->gid = vtStrDup(dir->gid); + vd->mid = vtStrDup(dir->muid); + vd->mtime = dir->mtime; + vd->mcount = 0; + vd->ctime = dir->mtime; /* ctime: not available on plan 9 */ + vd->atime = dir->atime; + + vd->mode = dir->mode & 0777; + if(dir->mode & DMDIR) + vd->mode |= ModeDir; + if(dir->mode & DMAPPEND) + vd->mode |= ModeAppend; + if(dir->mode & DMEXCL) + vd->mode |= ModeExclusive; + + vd->plan9 = 1; + vd->p9path = dir->qid.path; + vd->p9version = dir->qid.vers; +} + + +void +metaSinkEOR(MetaSink *k) +{ + uchar *p; + int o, n; + + p = k->buf + MetaHeaderSize; + p += k->nindex * MetaIndexSize; + o = k->rp-k->buf; /* offset from start of block */ + n = k->p-k->rp; /* size of entry */ + p[0] = o >> 8; + p[1] = o; + p[2] = n >> 8; + p[3] = n; + k->rp = k->p; + k->nindex++; + if(k->nindex == k->maxindex) + metaSinkFlush(k); +} + +void +metaSinkClose(MetaSink *k) +{ + metaSinkFlush(k); + sinkClose(k->sink); +} + +void +metaSinkFree(MetaSink *k) +{ + sinkFree(k->sink); + vtMemFree(k->buf); + vtMemFree(k); +} + +static void +warn(char *fmt, ...) +{ + va_list arg; + + va_start(arg, fmt); + fprint(2, "%s: ", argv0); + vfprint(2, fmt, arg); + fprint(2, "\n"); + va_end(arg); +} + +static void +cleanup(void) +{ + if(oname != nil) + remove(oname); +} + +#define TWID64 ((u64int)~(u64int)0) + +static u64int +unittoull(char *s) +{ + char *es; + u64int n; + + if(s == nil) + return TWID64; + n = strtoul(s, &es, 0); + if(*es == 'k' || *es == 'K'){ + n *= 1024; + es++; + }else if(*es == 'm' || *es == 'M'){ + n *= 1024*1024; + es++; + }else if(*es == 'g' || *es == 'G'){ + n *= 1024*1024*1024; + es++; + } + if(*es != '\0') + return TWID64; + return n; +} diff --git a/src/cmd/vac/vac.h b/src/cmd/vac/vac.h new file mode 100644 index 00000000..549c441c --- /dev/null +++ b/src/cmd/vac/vac.h @@ -0,0 +1,126 @@ +typedef struct VacFS VacFS; +typedef struct VacDir VacDir; +typedef struct VacFile VacFile; +typedef struct VacDirEnum VacDirEnum; + +/* + * Mode bits + */ +enum { + ModeOtherExec = (1<<0), + ModeOtherWrite = (1<<1), + ModeOtherRead = (1<<2), + ModeGroupExec = (1<<3), + ModeGroupWrite = (1<<4), + ModeGroupRead = (1<<5), + ModeOwnerExec = (1<<6), + ModeOwnerWrite = (1<<7), + ModeOwnerRead = (1<<8), + ModeSticky = (1<<9), + ModeSetUid = (1<<10), + ModeSetGid = (1<<11), + ModeAppend = (1<<12), /* append only file */ + ModeExclusive = (1<<13), /* lock file - plan 9 */ + ModeLink = (1<<14), /* sym link */ + ModeDir = (1<<15), /* duplicate of DirEntry */ + ModeHidden = (1<<16), /* MS-DOS */ + ModeSystem = (1<<17), /* MS-DOS */ + ModeArchive = (1<<18), /* MS-DOS */ + ModeTemporary = (1<<19), /* MS-DOS */ + ModeSnapshot = (1<<20), /* read only snapshot */ +}; + +enum { + MetaMagic = 0x5656fc79, + MetaHeaderSize = 12, + MetaIndexSize = 4, + IndexEntrySize = 8, + DirMagic = 0x1c4d9072, +}; + +enum { + DirPlan9Entry = 1, /* not valid in version >= 9 */ + DirNTEntry, /* not valid in version >= 9 */ + DirQidSpaceEntry, + DirGenEntry, /* not valid in version >= 9 */ +}; + +struct VacDir { + char *elem; /* path element */ + ulong entry; /* entry in directory for data */ + ulong gen; /* generation of data entry */ + ulong mentry; /* entry in directory for meta */ + ulong mgen; /* generation of meta entry */ + uvlong size; /* size of file */ + uvlong qid; /* unique file id */ + + char *uid; /* owner id */ + char *gid; /* group id */ + char *mid; /* last modified by */ + ulong mtime; /* last modified time */ + ulong mcount; /* number of modifications: can wrap! */ + ulong ctime; /* directory entry last changed */ + ulong atime; /* last time accessed */ + ulong mode; /* various mode bits */ + + /* plan 9 */ + int plan9; + uvlong p9path; + ulong p9version; + + /* sub space of qid */ + int qidSpace; + uvlong qidOffset; /* qid offset */ + uvlong qidMax; /* qid maximum */ +}; + +VacFS *vfsOpen(VtSession *z, char *file, int readOnly, long ncache); +VacFS *vfsCreate(VtSession *z, int bsize, long ncache); +int vfsGetBlockSize(VacFS*); +int vfsIsReadOnly(VacFS*); +VacFile *vfsGetRoot(VacFS*); + +long vfsGetCacheSize(VacFS*); +int vfsSetCacheSize(VacFS*, long); +int vfsSnapshot(VacFS*, char *src, char *dst); +int vfsSync(VacFS*); +int vfsClose(VacFS*); +int vfsGetScore(VacFS*, uchar score[VtScoreSize]); + +/* + * other ideas + * + * VacFS *vfsSnapshot(VacFS*, char *src); + * int vfsGraft(VacFS*, char *name, VacFS*); + */ + +VacFile *vfOpen(VacFS*, char *path); +VacFile *vfCreate(VacFile*, char *elem, ulong perm, char *user); +VacFile *vfWalk(VacFile*, char *elem); +int vfRemove(VacFile*, char*); +int vfRead(VacFile*, void *, int n, vlong offset); +int vfWrite(VacFile*, void *, int n, vlong offset, char *user); +int vfReadPacket(VacFile*, Packet**, vlong offset); +int vfWritePacket(VacFile*, Packet*, vlong offset, char *user); +uvlong vfGetId(VacFile*); +ulong vfGetMcount(VacFile*); +int vfIsDir(VacFile*); +int vfGetBlockScore(VacFile*, ulong bn, uchar score[VtScoreSize]); +int vfGetSize(VacFile*, uvlong *size); +int vfGetDir(VacFile*, VacDir*); +int vfSetDir(VacFile*, VacDir*); +int vfGetVtEntry(VacFile*, VtEntry*); +VacFile *vfGetParent(VacFile*); +int vfSync(VacFile*); +VacFile *vfIncRef(VacFile*); +void vfDecRef(VacFile*); +VacDirEnum *vfDirEnum(VacFile*); +int vfIsRoot(VacFile *vf); + +void vdCleanup(VacDir *dir); +void vdCopy(VacDir *dst, VacDir *src); + +VacDirEnum *vdeOpen(VacFS*, char *path); +int vdeRead(VacDirEnum*, VacDir *, int n); +void vdeFree(VacDirEnum*); + diff --git a/src/cmd/vac/vacfs.c b/src/cmd/vac/vacfs.c new file mode 100644 index 00000000..7702b0dd --- /dev/null +++ b/src/cmd/vac/vacfs.c @@ -0,0 +1,849 @@ +#include "stdinc.h" +#include <auth.h> +#include <fcall.h> +#include "vac.h" + +typedef struct Fid Fid; +typedef struct DirBuf DirBuf; + +enum +{ + OPERM = 0x3, /* mask of all permission types in open mode */ +}; + +enum +{ + DirBufSize = 20, +}; + +struct Fid +{ + short busy; + short open; + int fid; + char *user; + Qid qid; + VacFile *file; + + DirBuf *db; + + Fid *next; +}; + +struct DirBuf +{ + VacDirEnum *vde; + VacDir buf[DirBufSize]; + int i, n; + int eof; +}; + +enum +{ + Pexec = 1, + Pwrite = 2, + Pread = 4, + Pother = 1, + Pgroup = 8, + Powner = 64, +}; + +Fid *fids; +uchar *data; +int mfd[2]; +char *user; +uchar mdata[8192+IOHDRSZ]; +int messagesize = sizeof mdata; +Fcall rhdr; +Fcall thdr; +VacFS *fs; +VtSession *session; +int noperm; + +Fid * newfid(int); +void error(char*); +void io(void); +void shutdown(void); +void usage(void); +int perm(Fid*, int); +int permf(VacFile*, char*, int); +ulong getl(void *p); +void init(char*, char*, long, int); +DirBuf *dirBufAlloc(VacFile*); +VacDir *dirBufGet(DirBuf*); +int dirBufUnget(DirBuf*); +void dirBufFree(DirBuf*); +int vacdirread(Fid *f, char *p, long off, long cnt); +int vdStat(VacDir *vd, uchar *p, int np); + +char *rflush(Fid*), *rversion(Fid*), + *rauth(Fid*), *rattach(Fid*), *rwalk(Fid*), + *ropen(Fid*), *rcreate(Fid*), + *rread(Fid*), *rwrite(Fid*), *rclunk(Fid*), + *rremove(Fid*), *rstat(Fid*), *rwstat(Fid*); + +char *(*fcalls[])(Fid*) = { + [Tflush] rflush, + [Tversion] rversion, + [Tattach] rattach, + [Tauth] rauth, + [Twalk] rwalk, + [Topen] ropen, + [Tcreate] rcreate, + [Tread] rread, + [Twrite] rwrite, + [Tclunk] rclunk, + [Tremove] rremove, + [Tstat] rstat, + [Twstat] rwstat, +}; + +char Eperm[] = "permission denied"; +char Enotdir[] = "not a directory"; +char Enotexist[] = "file does not exist"; +char Einuse[] = "file in use"; +char Eexist[] = "file exists"; +char Enotowner[] = "not owner"; +char Eisopen[] = "file already open for I/O"; +char Excl[] = "exclusive use file already open"; +char Ename[] = "illegal name"; +char Erdonly[] = "read only file system"; +char Eio[] = "i/o error"; +char Eempty[] = "directory is not empty"; +char Emode[] = "illegal mode"; + +int dflag; + +void +notifyf(void *a, char *s) +{ + USED(a); + if(strncmp(s, "interrupt", 9) == 0) + noted(NCONT); + noted(NDFLT); +} + +void +main(int argc, char *argv[]) +{ + char *defmnt; + int p[2]; + char buf[12]; + int fd; + int stdio = 0; + char *host = nil; + long ncache = 1000; + int readOnly = 1; + + defmnt = "/n/vac"; + ARGBEGIN{ + case 'd': + fmtinstall('F', fcallfmt); + dflag = 1; + break; + case 'c': + ncache = atoi(ARGF()); + break; + case 'i': + defmnt = 0; + stdio = 1; + mfd[0] = 0; + mfd[1] = 1; + break; + case 'h': + host = ARGF(); + break; + case 's': + defmnt = 0; + break; + case 'p': + noperm = 1; + break; + case 'm': + defmnt = ARGF(); + break; + default: + usage(); + }ARGEND + + if(argc != 1) + usage(); + + vtAttach(); + + init(argv[0], host, ncache, readOnly); + + if(pipe(p) < 0) + sysfatal("pipe failed: %r"); + if(!stdio){ + mfd[0] = p[0]; + mfd[1] = p[0]; + if(defmnt == 0){ + fd = create("#s/vacfs", OWRITE, 0666); + if(fd < 0) + sysfatal("create of /srv/vacfs failed: %r"); + sprint(buf, "%d", p[1]); + if(write(fd, buf, strlen(buf)) < 0) + sysfatal("writing /srv/vacfs: %r"); + } + } + + switch(rfork(RFFDG|RFPROC|RFNAMEG|RFNOTEG)){ + case -1: + sysfatal("fork: %r"); + case 0: + vtAttach(); + close(p[1]); + io(); + shutdown(); + break; + default: + close(p[0]); /* don't deadlock if child fails */ + if(defmnt && mount(p[1], -1, defmnt, MREPL|MCREATE, "") < 0) + sysfatal("mount failed: %r"); + } + vtDetach(); + exits(0); +} + +void +usage(void) +{ + fprint(2, "usage: %s [-sd] [-h host] [-c ncache] [-m mountpoint] vacfile\n", argv0); + exits("usage"); +} + +char* +rversion(Fid *unused) +{ + Fid *f; + + USED(unused); + + for(f = fids; f; f = f->next) + if(f->busy) + rclunk(f); + + if(rhdr.msize < 256) + return "version: message size too small"; + messagesize = rhdr.msize; + if(messagesize > sizeof mdata) + messagesize = sizeof mdata; + thdr.msize = messagesize; + if(strncmp(rhdr.version, "9P2000", 6) != 0) + return "unrecognized 9P version"; + thdr.version = "9P2000"; + return nil; +} + +char* +rflush(Fid *f) +{ + USED(f); + return 0; +} + +char* +rauth(Fid *f) +{ + USED(f); + return "vacfs: authentication not required"; +} + +char* +rattach(Fid *f) +{ + /* no authentication for the momment */ + VacFile *file; + + file = vfsGetRoot(fs); + if(file == nil) + return vtGetError(); + f->busy = 1; + f->file = file; + f->qid = (Qid){vfGetId(f->file), 0, QTDIR}; + thdr.qid = f->qid; + if(rhdr.uname[0]) + f->user = vtStrDup(rhdr.uname); + else + f->user = "none"; + return 0; +} + +VacFile* +_vfWalk(VacFile *file, char *name) +{ + VacFile *n; + + n = vfWalk(file, name); + if(n) + return n; + if(strcmp(name, "SLASH") == 0) + return vfWalk(file, "/"); + return nil; +} + +char* +rwalk(Fid *f) +{ + VacFile *file, *nfile; + Fid *nf; + int nqid, nwname; + Qid qid; + + if(f->busy == 0) + return Enotexist; + nf = nil; + if(rhdr.fid != rhdr.newfid){ + if(f->open) + return Eisopen; + if(f->busy == 0) + return Enotexist; + nf = newfid(rhdr.newfid); + if(nf->busy) + return Eisopen; + nf->busy = 1; + nf->open = 0; + nf->qid = f->qid; + nf->file = vfIncRef(f->file); + nf->user = vtStrDup(f->user); + f = nf; + } + + nwname = rhdr.nwname; + + /* easy case */ + if(nwname == 0) { + thdr.nwqid = 0; + return 0; + } + + file = f->file; + vfIncRef(file); + qid = f->qid; + + for(nqid = 0; nqid < nwname; nqid++){ + if((qid.type & QTDIR) == 0){ + vtSetError(Enotdir); + break; + } + if(!permf(file, f->user, Pexec)) { + vtSetError(Eperm); + break; + } + nfile = _vfWalk(file, rhdr.wname[nqid]); + if(nfile == nil) + break; + vfDecRef(file); + file = nfile; + qid.type = QTFILE; + if(vfIsDir(file)) + qid.type = QTDIR; + qid.vers = vfGetMcount(file); + qid.path = vfGetId(file); + thdr.wqid[nqid] = qid; + } + + thdr.nwqid = nqid; + + if(nqid == nwname){ + /* success */ + f->qid = thdr.wqid[nqid-1]; + vfDecRef(f->file); + f->file = file; + return 0; + } + + vfDecRef(file); + if(nf != nil) + rclunk(nf); + + /* only error on the first element */ + if(nqid == 0) + return vtGetError(); + + return 0; +} + +char * +ropen(Fid *f) +{ + int mode, trunc; + + if(f->open) + return Eisopen; + if(!f->busy) + return Enotexist; + mode = rhdr.mode; + thdr.iounit = messagesize - IOHDRSZ; + if(f->qid.type & QTDIR){ + if(mode != OREAD) + return Eperm; + if(!perm(f, Pread)) + return Eperm; + thdr.qid = f->qid; + f->db = nil; + f->open = 1; + return 0; + } + if(mode & ORCLOSE) + return Erdonly; + trunc = mode & OTRUNC; + mode &= OPERM; + if(mode==OWRITE || mode==ORDWR || trunc) + if(!perm(f, Pwrite)) + return Eperm; + if(mode==OREAD || mode==ORDWR) + if(!perm(f, Pread)) + return Eperm; + if(mode==OEXEC) + if(!perm(f, Pexec)) + return Eperm; + thdr.qid = f->qid; + thdr.iounit = messagesize - IOHDRSZ; + f->open = 1; + return 0; +} + +char* +rcreate(Fid* fid) +{ + VacFile *vf; + ulong mode; + + if(fid->open) + return Eisopen; + if(!fid->busy) + return Enotexist; + if(vfsIsReadOnly(fs)) + return Erdonly; + vf = fid->file; + if(!vfIsDir(vf)) + return Enotdir; + if(!permf(vf, fid->user, Pwrite)) + return Eperm; + + mode = rhdr.perm & 0777; + + if(rhdr.perm & DMDIR){ + if((rhdr.mode & OTRUNC) || (rhdr.perm & DMAPPEND)) + return Emode; + switch(rhdr.mode & OPERM){ + default: + return Emode; + case OEXEC: + case OREAD: + break; + case OWRITE: + case ORDWR: + return Eperm; + } + mode |= ModeDir; + } + vf = vfCreate(vf, rhdr.name, mode, "none"); + if(vf == nil) + return vtGetError(); + vfDecRef(fid->file); + + fid->file = vf; + fid->qid.type = QTFILE; + if(vfIsDir(vf)) + fid->qid.type = QTDIR; + fid->qid.vers = vfGetMcount(vf); + fid->qid.path = vfGetId(vf); + + thdr.qid = fid->qid; + thdr.iounit = messagesize - IOHDRSZ; + + return 0; +} + +char* +rread(Fid *f) +{ + char *buf; + vlong off; + int cnt; + VacFile *vf; + char *err; + int n; + + if(!f->busy) + return Enotexist; + vf = f->file; + thdr.count = 0; + off = rhdr.offset; + buf = thdr.data; + cnt = rhdr.count; + if(f->qid.type & QTDIR) + n = vacdirread(f, buf, off, cnt); + else + n = vfRead(vf, buf, cnt, off); + if(n < 0) { + err = vtGetError(); + if(err == nil) + err = "unknown error!"; + return err; + } + thdr.count = n; + return 0; +} + +char* +rwrite(Fid *f) +{ + char *buf; + vlong off; + int cnt; + VacFile *vf; + + if(!f->busy) + return Enotexist; + vf = f->file; + thdr.count = 0; + off = rhdr.offset; + buf = rhdr.data; + cnt = rhdr.count; + if(f->qid.type & QTDIR) + return "file is a directory"; + thdr.count = vfWrite(vf, buf, cnt, off, "none"); + if(thdr.count < 0) { +fprint(2, "write failed: %s\n", vtGetError()); + return vtGetError(); + } + return 0; +} + +char * +rclunk(Fid *f) +{ + f->busy = 0; + f->open = 0; + vtMemFree(f->user); + f->user = nil; + vfDecRef(f->file); + f->file = nil; + dirBufFree(f->db); + f->db = nil; + return 0; +} + +char * +rremove(Fid *f) +{ + VacFile *vf, *vfp; + char *err = nil; + + if(!f->busy) + return Enotexist; + vf = f->file; + vfp = vfGetParent(vf); + + if(!permf(vfp, f->user, Pwrite)) { + err = Eperm; + goto Exit; + } + + if(!vfRemove(vf, "none")) { +print("vfRemove failed\n"); + err = vtGetError(); + } + +Exit: + vfDecRef(vfp); + rclunk(f); + return err; +} + +char * +rstat(Fid *f) +{ + VacDir dir; + static uchar statbuf[1024]; + + if(!f->busy) + return Enotexist; + vfGetDir(f->file, &dir); + thdr.stat = statbuf; + thdr.nstat = vdStat(&dir, thdr.stat, sizeof statbuf); + vdCleanup(&dir); + return 0; +} + +char * +rwstat(Fid *f) +{ + if(!f->busy) + return Enotexist; + return Erdonly; +} + +int +vdStat(VacDir *vd, uchar *p, int np) +{ + Dir dir; + + memset(&dir, 0, sizeof(dir)); + + /* + * Where do path and version come from + */ + dir.qid.path = vd->qid; + dir.qid.vers = vd->mcount; + dir.mode = vd->mode & 0777; + if(vd->mode & ModeAppend){ + dir.qid.type |= QTAPPEND; + dir.mode |= DMAPPEND; + } + if(vd->mode & ModeExclusive){ + dir.qid.type |= QTEXCL; + dir.mode |= DMEXCL; + } + if(vd->mode & ModeDir){ + dir.qid.type |= QTDIR; + dir.mode |= DMDIR; + } + + dir.atime = vd->atime; + dir.mtime = vd->mtime; + dir.length = vd->size; + + dir.name = vd->elem; + dir.uid = vd->uid; + dir.gid = vd->gid; + dir.muid = vd->mid; + + return convD2M(&dir, p, np); +} + +DirBuf* +dirBufAlloc(VacFile *vf) +{ + DirBuf *db; + + db = vtMemAllocZ(sizeof(DirBuf)); + db->vde = vfDirEnum(vf); + return db; +} + +VacDir * +dirBufGet(DirBuf *db) +{ + VacDir *vd; + int n; + + if(db->eof) + return nil; + + if(db->i >= db->n) { + n = vdeRead(db->vde, db->buf, DirBufSize); + if(n < 0) + return nil; + db->i = 0; + db->n = n; + if(n == 0) { + db->eof = 1; + return nil; + } + } + + vd = db->buf + db->i; + db->i++; + + return vd; +} + +int +dirBufUnget(DirBuf *db) +{ + assert(db->i > 0); + db->i--; + return 1; +} + +void +dirBufFree(DirBuf *db) +{ + int i; + + if(db == nil) + return; + + for(i=db->i; i<db->n; i++) + vdCleanup(db->buf + i); + vdeFree(db->vde); + vtMemFree(db); +} + +int +vacdirread(Fid *f, char *p, long off, long cnt) +{ + int n, nb; + VacDir *vd; + + /* + * special case of rewinding a directory + * otherwise ignore the offset + */ + if(off == 0 && f->db) { + dirBufFree(f->db); + f->db = nil; + } + + if(f->db == nil) + f->db = dirBufAlloc(f->file); + + for(nb = 0; nb < cnt; nb += n) { + vd = dirBufGet(f->db); + if(vd == nil) { + if(!f->db->eof) + return -1; + break; + } + n = vdStat(vd, (uchar*)p, cnt-nb); + if(n <= BIT16SZ) { + dirBufUnget(f->db); + break; + } + vdCleanup(vd); + p += n; + } + return nb; +} + +Fid * +newfid(int fid) +{ + Fid *f, *ff; + + ff = 0; + for(f = fids; f; f = f->next) + if(f->fid == fid) + return f; + else if(!ff && !f->busy) + ff = f; + if(ff){ + ff->fid = fid; + return ff; + } + f = vtMemAllocZ(sizeof *f); + f->fid = fid; + f->next = fids; + fids = f; + return f; +} + +void +io(void) +{ + char *err; + int n; + + for(;;){ + /* + * reading from a pipe or a network device + * will give an error after a few eof reads + * however, we cannot tell the difference + * between a zero-length read and an interrupt + * on the processes writing to us, + * so we wait for the error + */ + n = read9pmsg(mfd[0], mdata, sizeof mdata); + if(n == 0) + continue; + if(n < 0) + break; + if(convM2S(mdata, n, &rhdr) != n) + sysfatal("convM2S conversion error"); + + if(dflag) + fprint(2, "vacfs:<-%F\n", &rhdr); + + thdr.data = (char*)mdata + IOHDRSZ; + if(!fcalls[rhdr.type]) + err = "bad fcall type"; + else + err = (*fcalls[rhdr.type])(newfid(rhdr.fid)); + if(err){ + thdr.type = Rerror; + thdr.ename = err; + }else{ + thdr.type = rhdr.type + 1; + thdr.fid = rhdr.fid; + } + thdr.tag = rhdr.tag; + if(dflag) + fprint(2, "vacfs:->%F\n", &thdr); + n = convS2M(&thdr, mdata, messagesize); + if(write(mfd[1], mdata, n) != n) + sysfatal("mount write: %r"); + } +} + +int +permf(VacFile *vf, char *user, int p) +{ + VacDir dir; + ulong perm; + + if(!vfGetDir(vf, &dir)) + return 0; + perm = dir.mode & 0777; + if(noperm) + goto Good; + if((p*Pother) & perm) + goto Good; + if(strcmp(user, dir.gid)==0 && ((p*Pgroup) & perm)) + goto Good; + if(strcmp(user, dir.uid)==0 && ((p*Powner) & perm)) + goto Good; + vdCleanup(&dir); + return 0; +Good: + vdCleanup(&dir); + return 1; +} + +int +perm(Fid *f, int p) +{ + return permf(f->file, f->user, p); +} + +void +init(char *file, char *host, long ncache, int readOnly) +{ + notify(notifyf); + user = getuser(); + + fmtinstall('V', vtScoreFmt); + fmtinstall('R', vtErrFmt); + + session = vtDial(host, 0); + if(session == nil) + vtFatal("could not connect to server: %s", vtGetError()); + + if(!vtConnect(session, 0)) + vtFatal("vtConnect: %s", vtGetError()); + + fs = vfsOpen(session, file, readOnly, ncache); + if(fs == nil) + vtFatal("vfsOpen: %s", vtGetError()); +} + +void +shutdown(void) +{ + Fid *f; + + for(f = fids; f; f = f->next) { + if(!f->busy) + continue; +fprint(2, "open fid: %d\n", f->fid); + rclunk(f); + } + + vfsClose(fs); + vtClose(session); +} + diff --git a/src/cmd/vac/vactest.c b/src/cmd/vac/vactest.c new file mode 100644 index 00000000..c456604f --- /dev/null +++ b/src/cmd/vac/vactest.c @@ -0,0 +1,182 @@ +#include "stdinc.h" +#include "vac.h" +#include "dat.h" +#include "fns.h" + +void usage(void); +int unvac(VacFS *fs); +int readScore(int fd, uchar score[VtScoreSize]); +static void warn(char *fmt, ...); +void dirlist(VacFS *fs, char *path); + +static int nwant; +static char **want; +static int dflag = 1; +static int cflag; +static int lower; +static int verbose; +static int settimes; + +void +main(int argc, char *argv[]) +{ + char *zfile; + int ok, table; + VtSession *z; + char *vsrv = nil; + char *host = nil; + char *p; + int ncache = 1000; + VacFS *fs; + + table = 0; + zfile = nil; + ARGBEGIN{ + case 'D': + dflag++; + break; + case 'c': + cflag++; + break; + case 'C': + p = ARGF(); + if(p == nil) + usage(); + ncache = atoi(p); + if(ncache < 10) + ncache = 10; + if(ncache > 1000000) + ncache = 1000000; + break; + case 'i': + lower++; + break; + case 'f': + zfile = ARGF(); + if(zfile == nil) + usage(); + break; + case 'h': + host = ARGF(); + break; + case 't': + table++; + break; + case 'T': + settimes++; + break; + case 's': + vsrv = ARGF(); + break; + case 'v': + verbose++; + break; + default: + usage(); + break; + }ARGEND + + nwant = argc; + want = argv; + + vtAttach(); + + if(zfile == nil) + usage(); + + if(vsrv != nil) + z = vtStdioServer(vsrv); + else + z = vtDial(host); + if(z == nil) + vtFatal("could not connect to server: %s", vtGetError()); + vtSetDebug(z, 0); + if(!vtConnect(z, 0)) + vtFatal("vtConnect: %s", vtGetError()); + fs = vfsOpen(z, zfile, 1, ncache); + if(fs == nil) + vtFatal("vfsOpen: %s", vtGetError()); + ok = unvac(fs); + vtClose(z); + vtDetach(); + + exits(ok? 0 : "error"); +} + +void +usage(void) +{ + fprint(2, "usage: %s [-tTcDv] -f zipfile [-s ventid] [-h host] [file ...]\n", argv0); + exits("usage"); +} + +void +suck(VacFile *f) +{ + USED(f); +} + + +void +vacfile(VacFS *fs, char *path, VacDir *vd) +{ + char *path2; + + path2 = vtMemAlloc(strlen(path) + 1 + strlen(vd->elem) + 1); + if(path[1] == 0) + sprintf(path2, "/%s", vd->elem); + else + sprintf(path2, "%s/%s", path, vd->elem); +fprint(2, "vac file: %s\n", path2); + if(vd->mode & ModeDir) + dirlist(fs, path2); + vtMemFree(path2); +} + +void +dirlist(VacFS *fs, char *path) +{ + VacDir vd[50]; + VacDirEnum *ds; + int i, n; + + ds = vdeOpen(fs, path); + if(ds == nil) { + fprint(2, "could not open: %s: %s\n", path, vtGetError()); + return; + } + for(;;) { + n = vdeRead(ds, vd, sizeof(vd)/sizeof(VacDir)); + if(n < 0) { + warn("vdRead failed: %s: %s", path, vtGetError()); + return; + } + if(n == 0) + break; + for(i=0; i<n; i++) { + vacfile(fs, path, &vd[i]); + vdCleanup(&vd[i]); + } + } + vdeFree(ds); +} + +int +unvac(VacFS *fs) +{ + dirlist(fs, "/"); + + return 1; +} + +static void +warn(char *fmt, ...) +{ + va_list arg; + + va_start(arg, fmt); + fprint(2, "%s: ", argv0); + vfprint(2, fmt, arg); + fprint(2, "\n"); + va_end(arg); +} diff --git a/src/cmd/vac/vtdump.c b/src/cmd/vac/vtdump.c new file mode 100644 index 00000000..963e4689 --- /dev/null +++ b/src/cmd/vac/vtdump.c @@ -0,0 +1,391 @@ +#include "stdinc.h" +#include <bio.h> + +typedef struct Source Source; + +struct Source +{ + ulong gen; + int psize; + int dsize; + int dir; + int active; + int depth; + uvlong size; + uchar score[VtScoreSize]; + int reserved; +}; + +int bsize; +Biobuf *bout; +VtRoot root; +int ver; +int cmp; +int all; +int find; +uchar fscore[VtScoreSize]; +VtSession *z; + +int vtGetUint16(uchar *p); +ulong vtGetUint32(uchar *p); +uvlong vtGetUint48(uchar *p); +void usage(void); +int parseScore(uchar *score, char *buf, int n); +void readRoot(VtRoot*, uchar *score, char *file); +int dumpDir(Source*, int indent); + +void +main(int argc, char *argv[]) +{ + char *host = nil; + uchar score[VtScoreSize]; + Source source; + uchar buf[VtMaxLumpSize]; + char *p; + int n; + + ARGBEGIN{ + case 'h': + host = ARGF(); + break; + case 'c': + cmp++; + break; + case 'f': + find++; + p = ARGF(); + if(p == nil || !parseScore(fscore, p, strlen(p))) + usage(); + break; + case 'a': + all = 1; + break; + }ARGEND + + vtAttach(); + + bout = vtMemAllocZ(sizeof(Biobuf)); + Binit(bout, 1, OWRITE); + + if(argc > 1) + usage(); + + vtAttach(); + + fmtinstall('V', vtScoreFmt); + fmtinstall('R', vtErrFmt); + + z = vtDial(host, 0); + if(z == nil) + vtFatal("could not connect to server: %s", vtGetError()); + + if(!vtConnect(z, 0)) + sysfatal("vtConnect: %r"); + + readRoot(&root, score, argv[0]); + ver = root.version; + bsize = root.blockSize; + if(!find) { + Bprint(bout, "score: %V\n", score); + Bprint(bout, "version: %d\n", ver); + Bprint(bout, "name: %s\n", root.name); + Bprint(bout, "type: %s\n", root.type); + Bprint(bout, "bsize: %d\n", bsize); + Bprint(bout, "prev: %V\n", root.prev); + } + + switch(ver) { + default: + sysfatal("unknown version"); + case VtRootVersion: + break; + } + + n = vtRead(z, root.score, VtDirType, buf, bsize); + if(n < 0) + sysfatal("could not read root dir"); + + /* fake up top level source */ + memset(&source, 0, sizeof(source)); + memmove(source.score, root.score, VtScoreSize); + source.psize = bsize; + source.dsize = bsize; + source.dir = 1; + source.active = 1; + source.depth = 0; + source.size = n; + + dumpDir(&source, 0); + + Bterm(bout); + + vtClose(z); + vtDetach(); + exits(0); +} + +void +sourcePrint(Source *s, int indent, int entry) +{ + int i; + uvlong size; + int ne; + + for(i=0; i<indent; i++) + Bprint(bout, " "); + Bprint(bout, "%4d", entry); + if(s->active) { + /* dir size in directory entries */ + if(s->dir) { + ne = s->dsize/VtEntrySize; + size = ne*(s->size/s->dsize) + (s->size%s->dsize)/VtEntrySize; + } else + size = s->size; + if(cmp) { + Bprint(bout, ": gen: %lud size: %llud", + s->gen, size); + if(!s->dir) + Bprint(bout, ": %V", s->score); + } else { + Bprint(bout, ": gen: %lud psize: %d dsize: %d", + s->gen, s->psize, s->dsize); + Bprint(bout, " depth: %d size: %llud: %V", + s->depth, size, s->score); + } + + if(s->reserved) + Bprint(bout, ": reserved not emtpy"); + } + Bprint(bout, "\n"); +} + +int +parse(Source *s, uchar *p) +{ + VtEntry dir; + + memset(s, 0, sizeof(*s)); + if(!vtEntryUnpack(&dir, p, 0)) + return 0; + + if(!(dir.flags & VtEntryActive)) + return 1; + + s->active = 1; + s->gen = dir.gen; + s->psize = dir.psize; + s->dsize = dir.size; + s->size = dir.size; + memmove(s->score, dir.score, VtScoreSize); + if(dir.flags & VtEntryDir) + s->dir = 1; + s->depth = dir.depth; + return 1; + +} + +int +sourceRead(Source *s, ulong block, uchar *p, int n) +{ + uchar buf[VtMaxLumpSize]; + uchar score[VtScoreSize]; + int i, nn, np, type; + int elem[VtPointerDepth]; + + memmove(score, s->score, VtScoreSize); + + np = s->psize/VtScoreSize; + for(i=0; i<s->depth; i++) { + elem[i] = block % np; + block /= np; + } + assert(block == 0); + + for(i=s->depth-1; i>=0; i--) { + nn = vtRead(z, score, VtPointerType0+i, buf, s->psize); + if(nn < 0) + return -1; + + if(!vtSha1Check(score, buf, nn)) { + vtSetError("vtSha1Check failed on root block"); + return -1; + } + + if((elem[i]+1)*VtScoreSize > nn) + return 0; + memmove(score, buf + elem[i]*VtScoreSize, VtScoreSize); + } + + if(s->dir) + type = VtDirType; + else + type = VtDataType; + + nn = vtRead(z, score, type, p, n); + if(nn < 0) + return -1; + + if(!vtSha1Check(score, p, nn)) { + vtSetError("vtSha1Check failed on root block"); + return -1; + } + + return nn; +} + +void +dumpFileContents(Source *s) +{ + int nb, lb, i, n; + uchar buf[VtMaxLumpSize]; + + nb = (s->size + s->dsize - 1)/s->dsize; + lb = s->size%s->dsize; + for(i=0; i<nb; i++) { + memset(buf, 0, s->dsize); + n = sourceRead(s, i, buf, s->dsize); + if(n < 0) { + fprint(2, "could not read block: %d: %s\n", i, vtGetError()); + continue; + } + if(i < nb-1) + Bwrite(bout, buf, s->dsize); + else + Bwrite(bout, buf, lb); + } +} + +void +dumpFile(Source *s, int indent) +{ + int nb, i, j, n; + uchar buf[VtMaxLumpSize]; + uchar score[VtScoreSize]; + + nb = (s->size + s->dsize - 1)/s->dsize; + for(i=0; i<nb; i++) { + memset(buf, 0, s->dsize); + n = sourceRead(s, i, buf, s->dsize); + if(n < 0) { + fprint(2, "could not read block: %d: %s\n", i, vtGetError()); + continue; + } + for(j=0; j<indent; j++) + Bprint(bout, " "); + vtSha1(score, buf, n); + Bprint(bout, "%4d: size: %ud: %V\n", i, n, score); + } +} + +int +dumpDir(Source *s, int indent) +{ + int pb, ne, nb, i, j, n, entry; + uchar buf[VtMaxLumpSize]; + Source ss; + + pb = s->dsize/VtEntrySize; + ne = pb*(s->size/s->dsize) + (s->size%s->dsize)/VtEntrySize; + nb = (s->size + s->dsize - 1)/s->dsize; + for(i=0; i<nb; i++) { + memset(buf, 0, s->dsize); + n = sourceRead(s, i, buf, s->dsize); + if(n < 0) { + fprint(2, "could not read block: %d: %s\n", i, vtGetError()); + continue; + } + for(j=0; j<pb; j++) { + entry = i*pb + j; + if(entry >= ne) + break; + parse(&ss, buf + j * VtEntrySize); + + if(!find) + sourcePrint(&ss, indent, entry); + else if(memcmp(ss.score, fscore, VtScoreSize) == 0) { + dumpFileContents(&ss); + return 0; + } + + if(ss.dir) { + if(!dumpDir(&ss, indent+1)) + return 0; + } else if(all) + dumpFile(&ss, indent+1); + } + } + return 1; +} + +void +usage(void) +{ + fprint(2, "%s: [file]\n", argv0); + exits("usage"); +} + +int +parseScore(uchar *score, char *buf, int n) +{ + int i, c; + + memset(score, 0, VtScoreSize); + + if(n < VtScoreSize*2) + return 0; + for(i=0; i<VtScoreSize*2; i++) { + if(buf[i] >= '0' && buf[i] <= '9') + c = buf[i] - '0'; + else if(buf[i] >= 'a' && buf[i] <= 'f') + c = buf[i] - 'a' + 10; + else if(buf[i] >= 'A' && buf[i] <= 'F') + c = buf[i] - 'A' + 10; + else { + return 0; + } + + if((i & 1) == 0) + c <<= 4; + + score[i>>1] |= c; + } + return 1; +} + +void +readRoot(VtRoot *root, uchar *score, char *file) +{ + int fd; + uchar buf[VtRootSize]; + int i, n, nn; + + if(file == 0) + fd = 0; + else { + fd = open(file, OREAD); + if(fd < 0) + sysfatal("could not open file: %s: %r\n", file); + } + n = readn(fd, buf, sizeof(buf)-1); + if(n < 0) + sysfatal("read failed: %r\n"); + buf[n] = 0; + close(fd); + + for(i=0; i<n; i++) { + if(!parseScore(score, (char*)(buf+i), n-i)) + continue; + nn = vtRead(z, score, VtRootType, buf, VtRootSize); + if(nn >= 0) { + if(nn != VtRootSize) + sysfatal("vtRead on root too short"); + if(!vtSha1Check(score, buf, VtRootSize)) + sysfatal("vtSha1Check failed on root block"); + if(!vtRootUnpack(root, buf)) + sysfatal("could not parse root: %r"); + return; + } + } + + sysfatal("could not find root"); +} diff --git a/src/cmd/vac/vtread.c b/src/cmd/vac/vtread.c new file mode 100644 index 00000000..8550279f --- /dev/null +++ b/src/cmd/vac/vtread.c @@ -0,0 +1,126 @@ +#include "stdinc.h" +#include <bio.h> + +typedef struct Source Source; + +struct Source +{ + ulong gen; + int psize; + int dsize; + int dir; + int active; + int depth; + uvlong size; + uchar score[VtScoreSize]; + int reserved; +}; + +int bsize; +Biobuf *bout; +VtRootLump root; +int ver; +int cmp; +int all; +int find; +uchar fscore[VtScoreSize]; +int dirSize; +void (*parse)(Source*, uchar*); +VtSession *z; + +int vtGetUint16(uchar *p); +ulong vtGetUint32(uchar *p); +uvlong vtGetUint48(uchar *p); +void usage(void); +int parseScore(uchar *score, char *buf, int n); +void readRoot(VtRootLump*, uchar *score, char *file); +void parse1(Source*, uchar*); +void parse2(Source*, uchar*); +int dumpDir(Source*, int indent); + +void +main(int argc, char *argv[]) +{ + char *host = nil; + uchar score[VtScoreSize]; + uchar buf[VtMaxLumpSize]; + int type; + int n; + + type = VtDataType; + + ARGBEGIN{ + case 't': + type = atoi(ARGF()); + break; + }ARGEND + + vtAttach(); + + bout = vtMemAllocZ(sizeof(Biobuf)); + Binit(bout, 1, OWRITE); + + if(argc != 1) + usage(); + + vtAttach(); + + fmtinstall('V', vtScoreFmt); + fmtinstall('R', vtErrFmt); + + z = vtDial(host); + if(z == nil) + vtFatal("could not connect to server: %s", vtGetError()); + + if(!vtConnect(z, 0)) + sysfatal("vtConnect: %r"); + + if(!parseScore(score, argv[0], strlen(argv[0]))) + vtFatal("could not parse score: %s", vtGetError()); + + n = vtRead(z, score, type, buf, VtMaxLumpSize); + if(n < 0) + vtFatal("could not read block: %s", vtGetError()); + Bwrite(bout, buf, n); + + Bterm(bout); + + vtClose(z); + vtDetach(); + exits(0); +} + +void +usage(void) +{ + fprint(2, "%s: -t type score\n", argv0); + exits("usage"); +} + +int +parseScore(uchar *score, char *buf, int n) +{ + int i, c; + + memset(score, 0, VtScoreSize); + + if(n < VtScoreSize*2) + return 0; + for(i=0; i<VtScoreSize*2; i++) { + if(buf[i] >= '0' && buf[i] <= '9') + c = buf[i] - '0'; + else if(buf[i] >= 'a' && buf[i] <= 'f') + c = buf[i] - 'a' + 10; + else if(buf[i] >= 'A' && buf[i] <= 'F') + c = buf[i] - 'A' + 10; + else { + return 0; + } + + if((i & 1) == 0) + c <<= 4; + + score[i>>1] |= c; + } + return 1; +} diff --git a/src/cmd/vac/wtest.c b/src/cmd/vac/wtest.c new file mode 100644 index 00000000..3cc15e9c --- /dev/null +++ b/src/cmd/vac/wtest.c @@ -0,0 +1,47 @@ +#include "stdinc.h" + +enum { + Nblock = 10000, + BlockSize = 8*1024, +}; + +uchar data[Nblock*BlockSize]; + +void +main(int argc, char *argv[]) +{ + VtSession *z; + int i; + uchar score[VtScoreSize]; + int start; + + ARGBEGIN{ + }ARGEND + + for(i=0; i<Nblock; i++) { + if(readn(0, data+i*BlockSize, BlockSize) < BlockSize) + sysfatal("read failed: %r"); + } + + vtAttach(); + + z = vtDial("iolaire2"); + if(z == nil) + sysfatal("cound not connect to venti"); + if(!vtConnect(z, 0)) + vtFatal("vtConnect: %s", vtGetError()); + + print("starting\n"); + + start = times(0); + + for(i=0; i<Nblock; i++) { + if(!vtWrite(z, score, VtDataType, data+i*BlockSize, BlockSize)) + vtFatal("vtWrite failed: %s", vtGetError()); + } + + print("time = %f\n", (times(0) - start)*0.001); + + vtClose(z); + vtDetach(); +} |