diff options
Diffstat (limited to 'src/cmd/vbackup/vbackup.c')
| -rw-r--r-- | src/cmd/vbackup/vbackup.c | 524 |
1 files changed, 524 insertions, 0 deletions
diff --git a/src/cmd/vbackup/vbackup.c b/src/cmd/vbackup/vbackup.c new file mode 100644 index 00000000..4fdaf3e4 --- /dev/null +++ b/src/cmd/vbackup/vbackup.c @@ -0,0 +1,524 @@ +/* + * vbackup [-Dnv] fspartition [score] + * + * Copy a file system to a disk image stored on Venti. + * Prints a vnfs config line for the copied image. + * + * -D print debugging + * -m set mount name + * -n nop -- don't actually write blocks + * -s print status updates + * -v print debugging trace + * -w write parallelism + * + * If score is given on the command line, it should be the + * score from a previous vbackup on this fspartition. + * In this mode, only the new blocks are stored to Venti. + * The result is still a complete image, but requires many + * fewer Venti writes in the common case. + * + * This program is structured as three processes connected + * by buffered queues: + * + * fsysproc | cmpproc | ventiproc + * + * Fsysproc reads the disk and queues the blocks. + * Cmpproc compares the blocks against the SHA1 hashes + * in the old image, if any. It discards the unchanged blocks + * and queues the changed ones. Ventiproc writes blocks to Venti. + * + * There is a fourth proc, statusproc, which prints status + * updates about how the various procs are progressing. + */ + +#include <u.h> +#include <libc.h> +#include <bio.h> +#include <thread.h> +#include <libsec.h> +#include <venti.h> +#include <diskfs.h> +#include "queue.h" + +enum +{ + STACK = 8192, +}; + +typedef struct WriteReq WriteReq; +struct WriteReq +{ + Packet *p; + uint type; +}; + +Biobuf bscores; /* biobuf filled with block scores */ +int debug; /* debugging flag (not used) */ +Disk* disk; /* disk being backed up */ +RWLock endlk; /* silly synchonization */ +int errors; /* are we exiting with an error status? */ +int fsscanblock; /* last block scanned */ +Fsys* fsys; /* file system being backed up */ +int nchange; /* number of changed blocks */ +int nop; /* don't actually send blocks to venti */ +int nwrite; /* number of write-behind threads */ +Queue* qcmp; /* queue fsys->cmp */ +Queue* qventi; /* queue cmp->venti */ +int statustime; /* print status every _ seconds */ +int verbose; /* print extra stuff */ +VtFile* vfile; /* venti file being written */ +Channel* writechan; /* chan(WriteReq) */ +VtConn* z; /* connection to venti */ +VtCache* zcache; /* cache of venti blocks */ +uchar* zero; /* blocksize zero bytes */ + +extern int ncopy, nread, nwrite; /* hidden in libventi */ + +void cmpproc(void*); +void fsysproc(void*); +void statusproc(void*); +void ventiproc(void*); +int timefmt(Fmt*); +char* mountplace(char *dev); + +void +usage(void) +{ + fprint(2, "usage: vbackup [-DVnv] [-m mtpt] [-s secs] [-w n] disk [score]\n"); + threadexitsall("usage"); +} + +void +threadmain(int argc, char **argv) +{ + char *pref, *mountname; + uchar score[VtScoreSize], prev[VtScoreSize]; + int i, fd, csize; + vlong bsize; + Tm tm; + VtEntry e; + VtBlock *b; + VtCache *c; + VtRoot root; + char *tmp, *tmpnam; + + fmtinstall('F', vtfcallfmt); + fmtinstall('H', encodefmt); + fmtinstall('T', timefmt); + fmtinstall('V', vtscorefmt); + + mountname = sysname(); + ARGBEGIN{ + default: + usage(); + break; + case 'D': + debug++; + break; + case 'V': + chattyventi = 1; + break; + case 'm': + mountname = EARGF(usage()); + break; + case 'n': + nop = 1; + break; + case 's': + statustime = atoi(EARGF(usage())); + break; + case 'v': + verbose = 1; + break; + case 'w': + nwrite = atoi(EARGF(usage())); + break; + }ARGEND + + if(argc != 1 && argc != 2) + usage(); + + if(statustime) + print("# %T vbackup %s %s\n", argv[0], argc>=2 ? argv[1] : ""); + /* + * open fs + */ + if((disk = diskopenfile(argv[0])) == nil) + sysfatal("diskopen: %r"); + if((disk = diskcache(disk, 16384, 2*MAXQ+16)) == nil) + sysfatal("diskcache: %r"); + if((fsys = fsysopen(disk)) == nil) + sysfatal("ffsopen: %r"); + + /* + * connect to venti + */ + if((z = vtdial(nil)) == nil) + sysfatal("vtdial: %r"); + if(vtconnect(z) < 0) + sysfatal("vtconnect: %r"); + + /* + * set up venti block cache + */ + zero = vtmallocz(fsys->blocksize); + bsize = fsys->blocksize; + csize = 50; /* plenty; could probably do with 5 */ + + if(verbose) + fprint(2, "cache %d blocks\n", csize); + c = vtcachealloc(z, bsize, csize, VtORDWR); + zcache = c; + + /* + * parse starting score + */ + memset(prev, 0, sizeof prev); + if(argc == 1){ + vfile = vtfilecreateroot(c, (fsys->blocksize/VtScoreSize)*VtScoreSize, + fsys->blocksize, VtDataType); + if(vfile == nil) + sysfatal("vtfilecreateroot: %r"); + vtfilelock(vfile, VtORDWR); + if(vtfilewrite(vfile, zero, 1, bsize*fsys->nblock-1) != 1) + sysfatal("vtfilewrite: %r"); + if(vtfileflush(vfile) < 0) + sysfatal("vtfileflush: %r"); + }else{ + if(vtparsescore(argv[1], &pref, score) < 0) + sysfatal("bad score: %r"); + if(pref!=nil && strcmp(pref, fsys->type) != 0) + sysfatal("score is %s but fsys is %s", pref, fsys->type); + b = vtcacheglobal(c, score, VtRootType); + if(b){ + if(vtrootunpack(&root, b->data) < 0) + sysfatal("bad root: %r"); + if(strcmp(root.type, fsys->type) != 0) + sysfatal("root is %s but fsys is %s", root.type, fsys->type); + memmove(prev, score, VtScoreSize); + memmove(score, root.score, VtScoreSize); + vtblockput(b); + } + b = vtcacheglobal(c, score, VtDirType); + if(b == nil) + sysfatal("vtcacheglobal %V: %r", score); + if(vtentryunpack(&e, b->data, 0) < 0) + sysfatal("%V: vtentryunpack failed", score); + if(verbose) + fprint(2, "entry: size %llud psize %d dsize %d\n", + e.size, e.psize, e.dsize); + vtblockput(b); + if((vfile = vtfileopenroot(c, &e)) == nil) + sysfatal("vtfileopenroot: %r"); + vtfilelock(vfile, VtORDWR); + if(e.dsize != bsize) + sysfatal("file system block sizes don't match %d %lld", e.dsize, bsize); + if(e.size != fsys->nblock*bsize) + sysfatal("file system block counts don't match %lld %lld", e.size, fsys->nblock*bsize); + } + + /* + * write scores of blocks into temporary file + */ + if((tmp = getenv("TMP")) != nil){ + /* okay, good */ + }else if(access("/var/tmp", 0) >= 0) + tmp = "/var/tmp"; + else + tmp = "/tmp"; + tmpnam = smprint("%s/vbackup.XXXXXX", tmp); + if(tmpnam == nil) + sysfatal("smprint: %r"); + + if((fd = opentemp(tmpnam)) < 0) + sysfatal("opentemp %s: %r", tmpnam); + if(statustime) + print("# %T reading scores into %s\n", tmpnam); + if(verbose) + fprint(2, "read scores into %s...\n", tmpnam); + + Binit(&bscores, fd, OWRITE); + for(i=0; i<fsys->nblock; i++){ + if(vtfileblockscore(vfile, i, score) < 0) + sysfatal("vtfileblockhash %d: %r", i); + if(Bwrite(&bscores, score, VtScoreSize) != VtScoreSize) + sysfatal("Bwrite: %r"); + } + Bterm(&bscores); + vtfileunlock(vfile); + + /* + * prep scores for rereading + */ + seek(fd, 0, 0); + Binit(&bscores, fd, OREAD); + + /* + * start the main processes + */ + if(statustime) + print("# %T starting procs\n"); + qcmp = qalloc(); + qventi = qalloc(); + + rlock(&endlk); + proccreate(fsysproc, nil, STACK); + rlock(&endlk); + proccreate(ventiproc, nil, STACK); + rlock(&endlk); + proccreate(cmpproc, nil, STACK); + if(statustime){ + rlock(&endlk); + proccreate(statusproc, nil, STACK); + } + + /* + * wait for processes to finish + */ + wlock(&endlk); + + if(statustime) + print("# %T procs exited: %d blocks changed, %d read, %d written, %d copied\n", + nchange, nread, nwrite, ncopy); + + /* + * prepare root block + */ + vtfilelock(vfile, -1); + if(vtfileflush(vfile) < 0) + sysfatal("vtfileflush: %r"); + if(vtfilegetentry(vfile, &e) < 0) + sysfatal("vtfilegetentry: %r"); + + b = vtcacheallocblock(c, VtDirType); + if(b == nil) + sysfatal("vtcacheallocblock: %r"); + vtentrypack(&e, b->data, 0); + if(vtblockwrite(b) < 0) + sysfatal("vtblockwrite: %r"); + + memset(&root, 0, sizeof root); + strecpy(root.name, root.name+sizeof root.name, argv[0]); + strecpy(root.type, root.type+sizeof root.type, fsys->type); + memmove(root.score, b->score, VtScoreSize); + root.blocksize = fsys->blocksize; + memmove(root.prev, prev, VtScoreSize); + vtblockput(b); + + b = vtcacheallocblock(c, VtRootType); + if(b == nil) + sysfatal("vtcacheallocblock: %r"); + vtrootpack(&root, b->data); + if(vtblockwrite(b) < 0) + sysfatal("vtblockwrite: %r"); + + tm = *localtime(time(0)); + tm.year += 1900; + tm.mon++; + print("mount /%s/%d/%02d%02d%s %s:%V %d/%02d%02d/%02d%02d\n", + mountname, tm.year, tm.mon, tm.mday, + mountplace(argv[0]), + root.type, b->score, + tm.year, tm.mon, tm.mday, tm.hour, tm.min); + print("# %T %s %s:%V\n", argv[0], root.type, b->score); + if(statustime) + print("# %T venti sync\n"); + vtblockput(b); + if(vtsync(z) < 0) + sysfatal("vtsync: %r"); + if(statustime) + print("# %T synced\n"); + threadexitsall(nil); +} + +void +fsysproc(void *dummy) +{ + u32int i; + Block *db; + + USED(dummy); + + for(i=0; i<fsys->nblock; i++){ + fsscanblock = i; + if((db = fsysreadblock(fsys, i)) != nil) + qwrite(qcmp, db, i); + } + fsscanblock = i; + qclose(qcmp); + + print("# %T fsys proc exiting\n"); + runlock(&endlk); +} + +void +cmpproc(void *dummy) +{ + uchar *data; + Block *db; + u32int bno, bsize; + uchar score[VtScoreSize]; + uchar score1[VtScoreSize]; + + USED(dummy); + + bsize = fsys->blocksize; + while((db = qread(qcmp, &bno)) != nil){ + data = db->data; + sha1(data, vtzerotruncate(VtDataType, data, bsize), score, nil); + if(Bseek(&bscores, (vlong)bno*VtScoreSize, 0) < 0) + sysfatal("cmpproc Bseek: %r"); + if(Bread(&bscores, score1, VtScoreSize) != VtScoreSize) + sysfatal("cmpproc Bread: %r"); + if(memcmp(score, score1, VtScoreSize) != 0){ + nchange++; + if(verbose) + print("# block %ud: old %V new %V\n", bno, score1, score); + qwrite(qventi, db, bno); + }else + blockput(db); + } + qclose(qventi); + runlock(&endlk); +} + +void +writethread(void *v) +{ + WriteReq wr; + uchar score[VtScoreSize]; + + USED(v); + + while(recv(writechan, &wr) == 1){ + if(wr.p == nil) + break; + if(vtwritepacket(z, score, wr.type, wr.p) < 0) + sysfatal("vtwritepacket: %r"); + } +} + +int +myvtwrite(VtConn *z, uchar score[VtScoreSize], uint type, uchar *buf, int n) +{ + WriteReq wr; + + if(nwrite == 0) + return vtwrite(z, score, type, buf, n); + + wr.p = packetalloc(); + packetappend(wr.p, buf, n); + packetsha1(wr.p, score); + wr.type = type; + send(writechan, &wr); + return 0; +} + +void +ventiproc(void *dummy) +{ + int i; + Block *db; + u32int bno; + u64int bsize; + + USED(dummy); + + proccreate(vtsendproc, z, STACK); + proccreate(vtrecvproc, z, STACK); + + writechan = chancreate(sizeof(WriteReq), 0); + for(i=0; i<nwrite; i++) + threadcreate(writethread, nil, STACK); + vtcachesetwrite(zcache, myvtwrite); + + bsize = fsys->blocksize; + vtfilelock(vfile, -1); + while((db = qread(qventi, &bno)) != nil){ + if(nop){ + blockput(db); + continue; + } + if(vtfilewrite(vfile, db->data, bsize, bno*bsize) != bsize) + sysfatal("ventiproc vtfilewrite: %r"); + if(vtfileflushbefore(vfile, (bno+1)*bsize) < 0) + sysfatal("ventiproc vtfileflushbefore: %r"); + blockput(db); + } + vtfileunlock(vfile); + vtcachesetwrite(zcache, nil); + for(i=0; i<nwrite; i++) + send(writechan, nil); + runlock(&endlk); +} + +static int +percent(u32int a, u32int b) +{ + return (vlong)a*100/b; +} + +void +statusproc(void *dummy) +{ + int n; + USED(dummy); + + for(n=0;;n++){ + sleep(1000); + if(qcmp->closed && qcmp->nel==0 && qventi->closed && qventi->nel==0) + break; + if(n < statustime) + continue; + n = 0; + print("# %T fsscan=%d%% cmpq=%d%% ventiq=%d%%\n", + percent(fsscanblock, fsys->nblock), + percent(qcmp->nel, MAXQ), + percent(qventi->nel, MAXQ)); + } + runlock(&endlk); +} + +int +timefmt(Fmt *fmt) +{ + vlong ns; + Tm tm; + ns = nsec(); + tm = *localtime(time(0)); + return fmtprint(fmt, "%04d/%02d%02d %02d:%02d:%02d.%03d", + tm.year+1900, tm.mon+1, tm.mday, tm.hour, tm.min, tm.sec, + (int)(ns%1000000000)/1000000); +} + +char* +mountplace(char *dev) +{ + char *cmd, *q; + int p[2], fd[3], n; + char buf[100]; + + if(pipe(p) < 0) + sysfatal("pipe: %r"); + + fd[0] = -1; + fd[1] = p[1]; + fd[2] = -1; + cmd = smprint("mount | awk '$1==\"%s\" && $2 == \"on\" {print $3}'", dev); + if(threadspawnl(fd, "sh", "sh", "-c", cmd, nil) < 0) + sysfatal("exec mount|awk (to find mtpt of %s): %r", dev); + /* threadspawnl closed p[1] */ + n = readn(p[0], buf, sizeof buf-1); + close(p[0]); + if(n <= 0) + return dev; + buf[n] = 0; + if((q = strchr(buf, '\n')) == nil) + return dev; + *q = 0; + q = buf+strlen(buf); + if(q>buf && *(q-1) == '/') + *--q = 0; + return strdup(buf); +} + |