vac: add -a and -x flags

Thanks to Michael Kaminsky for the suggestion.

10 files changed, 494 insertions, 81 deletions

diff --git a/src/cmd/vac/exc b/src/cmd/vac/exc
new file mode 100644
index 00000000..aa212068
--- /dev/null
+++ b/src/cmd/vac/exc
@@ -0,0 +1,8 @@
+exclude a/*
+exclude b/...
+exclude c/[~a]*
+exclude d/[a]*
+exclude e/[a-z]*
+exclude f/?a*
+exclude g/*/*/b
+exclude h/.../b
diff --git a/src/cmd/vac/exc.in b/src/cmd/vac/exc.in
new file mode 100644
index 00000000..fd2a98b3
--- /dev/null
+++ b/src/cmd/vac/exc.in
@@ -0,0 +1,26 @@
+a/abc
+a/foo
+a/.foo
+b/foo
+b/.foo
+c/abc
+c/def
+c/zab
+d/abc
+d/def
+d/zab
+e/abc
+e/.abc
+e/ABC
+f/a
+f/.abc
+f/az
+f/za
+f/zabc
+f/zza
+g/a/b
+g/a/c/b
+g/a/c/d/b
+h/a/b
+h/a/c/b
+h/a/c/d/b
diff --git a/src/cmd/vac/exc.out b/src/cmd/vac/exc.out
new file mode 100644
index 00000000..704b150e
--- /dev/null
+++ b/src/cmd/vac/exc.out
@@ -0,0 +1,26 @@
+0 a/abc
+0 a/foo
+1 a/.foo
+0 b/foo
+0 b/.foo
+1 c/abc
+0 c/def
+0 c/zab
+0 d/abc
+1 d/def
+1 d/zab
+0 e/abc
+1 e/.abc
+1 e/ABC
+1 f/a
+1 f/.abc
+1 f/az
+0 f/za
+0 f/zabc
+1 f/zza
+1 g/a/b
+0 g/a/c/b
+1 g/a/c/d/b
+0 h/a/b
+0 h/a/c/b
+0 h/a/c/d/b
diff --git a/src/cmd/vac/file.c b/src/cmd/vac/file.c
index f5f823d7..d7ad2056 100644
--- a/src/cmd/vac/file.c
+++ b/src/cmd/vac/file.c
@@ -974,6 +974,7 @@ filemetaalloc(VacFile *fp, VacDir *dir, u32int start)
 			vtblockput(b);
 			if((b = vtfileblock(ms, bo, VtORDWR)) == nil)
 				goto Err;
+			mbunpack(&mb, b->data, ms->dsize);
 			goto Found;
 		}
 		vtblockput(b);
@@ -1002,7 +1003,6 @@ Found:
 	me.p = p;
 	me.size = n;
 	vdpack(dir, &me, VacDirVersion);
-vdunpack(dir, &me);
 	mbinsert(&mb, i, &me);
 	mbpack(&mb);
 	vtblockput(b);
@@ -1166,6 +1166,7 @@ Err:
 /*
  * Flush all data associated with f out of the cache and onto venti.
  * If recursive is set, flush f's children too.
+ * Vacfiledecref knows how to flush source and msource too.
  */
 int
 vacfileflush(VacFile *f, int recursive)
@@ -1183,25 +1184,12 @@ vacfileflush(VacFile *f, int recursive)
 		ret = -1;
 	filemetaunlock(f);
 
-	/*
-	 * Vacfiledecref knows how to flush source and msource too.
-	 */
 	if(filelock(f) < 0)
 		return -1;
-	vtfilelock(f->source, -1);
-	if(vtfileflush(f->source) < 0)
-		ret = -1;
-	vtfileunlock(f->source);
-	if(f->msource){
-		vtfilelock(f->msource, -1);
-		if(vtfileflush(f->msource) < 0)
-			ret = -1;
-		vtfileunlock(f->msource);
-	}
-	
+
 	/*
 	 * Lock order prevents us from flushing kids while holding
-	 * lock, so make a list.
+	 * lock, so make a list and then flush without the lock.
 	 */
 	nkids = 0;
 	kids = nil;
@@ -1216,14 +1204,32 @@ vacfileflush(VacFile *f, int recursive)
 			p->ref++;
 		}
 	}
-	fileunlock(f);
-	
-	for(i=0; i<nkids; i++){
-		if(vacfileflush(kids[i], 1) < 0)
-			ret = -1;
-		vacfiledecref(kids[i]);
+	if(nkids > 0){
+		fileunlock(f);
+		for(i=0; i<nkids; i++){
+			if(vacfileflush(kids[i], 1) < 0)
+				ret = -1;
+			vacfiledecref(kids[i]);
+		}
+		filelock(f);
 	}
 	free(kids);
+
+	/*
+	 * Now we can flush our own data.
+	 */	
+	vtfilelock(f->source, -1);
+	if(vtfileflush(f->source) < 0)
+		ret = -1;
+	vtfileunlock(f->source);
+	if(f->msource){
+		vtfilelock(f->msource, -1);
+		if(vtfileflush(f->msource) < 0)
+			ret = -1;
+		vtfileunlock(f->msource);
+	}
+	fileunlock(f);
+
 	return ret;
 }
 		
@@ -1332,6 +1338,12 @@ vacfilecreate(VacFile *fp, char *elem, ulong mode)
 	vacfileincref(fp);
 
 	fileunlock(fp);
+	
+	filelock(ff);
+	vtfilelock(ff->source, -1);
+	vtfileunlock(ff->source);
+	fileunlock(ff);
+
 	return ff;
 
 Err:
@@ -2031,7 +2043,7 @@ vacfssync(VacFs *fs)
 		return -1;
 	}
 	vtfileclose(f);
-	
+
 	/* Build a root block. */
 	memset(&root, 0, sizeof root);
 	strcpy(root.type, "vac");
diff --git a/src/cmd/vac/fns.h b/src/cmd/vac/fns.h
index 3cf87840..0f691b1e 100644
--- a/src/cmd/vac/fns.h
+++ b/src/cmd/vac/fns.h
@@ -23,3 +23,8 @@ VacFile *_vacfileroot(VacFs *fs, VtFile *file);
 
 int	_vacfsnextqid(VacFs *fs, uvlong *qid);
 void	vacfsjumpqid(VacFs*, uvlong step);
+
+Reprog*	glob2regexp(char*);
+void	loadexcludefile(char*);
+int	includefile(char*);
+void	excludepattern(char*);
diff --git a/src/cmd/vac/glob.c b/src/cmd/vac/glob.c
new file mode 100644
index 00000000..863eb848
--- /dev/null
+++ b/src/cmd/vac/glob.c
@@ -0,0 +1,180 @@
+#include "stdinc.h"
+#include "vac.h"
+#include "dat.h"
+#include "fns.h"
+#include "error.h"
+
+// Convert globbish pattern to regular expression
+// The wildcards are
+//
+//	*	any non-slash characters
+//	...	any characters including /
+//	?	any single character except /
+//	[a-z]	character class
+//	[~a-z]	negated character class
+//
+
+Reprog*
+glob2regexp(char *glob)
+{
+	char *s, *p, *w;
+	Reprog *re;
+	int boe;	// beginning of path element
+
+	s = malloc(20*(strlen(glob)+1));
+	if(s == nil)
+		return nil;
+	w = s;
+	boe = 1;
+	*w++ = '^';
+	*w++ = '(';
+	for(p=glob; *p; p++){
+		if(p[0] == '.' && p[1] == '.' && p[2] == '.'){
+			strcpy(w, ".*");
+			w += strlen(w);
+			p += 3-1;
+			boe = 0;
+			continue;
+		}
+		if(p[0] == '*'){
+			if(boe)
+				strcpy(w, "([^./][^/]*)?");
+			else
+				strcpy(w, "[^/]*");
+			w += strlen(w);
+			boe = 0;
+			continue;
+		}
+		if(p[0] == '?'){
+			if(boe)
+				strcpy(w, "[^./]");
+			else
+				strcpy(w, "[^/]");
+			w += strlen(w);
+			boe = 0;
+			continue;
+		}
+		if(p[0] == '['){
+			*w++ = '[';
+			if(*++p == '~'){
+				*w++ = '^';
+				p++;
+			}
+			while(*p != ']'){
+				if(*p == '/')
+					goto syntax;
+				if(*p == '^' || *p == '\\')
+					*w++ = '\\';
+				*w++ = *p++;
+			}
+			*w++ = ']';
+			boe = 0;
+			continue;
+		}
+		if(strchr("()|^$[]*?+\\.", *p)){
+			*w++ = '\\';
+			*w++ = *p;
+			boe = 0;
+			continue;
+		}
+		if(*p == '/'){
+			*w++ = '/';
+			boe = 1;
+			continue;
+		}
+		*w++ = *p;
+		boe = 0;
+		continue;
+	}
+	*w++ = ')';
+	*w++ = '$';
+	*w = 0;
+	
+	re = regcomp(s);
+	if(re == nil){
+	syntax:
+		free(s);
+		werrstr("glob syntax error");
+		return nil;
+	}
+	free(s);
+	return re;
+}
+
+typedef struct Pattern Pattern;
+struct Pattern
+{
+	Reprog *re;
+	int include;
+};
+
+Pattern *pattern;
+int npattern;
+
+void
+loadexcludefile(char *file)
+{
+	Biobuf *b;
+	char *p, *q;
+	int n, inc;
+	Reprog *re;
+
+	if((b = Bopen(file, OREAD)) == nil)
+		sysfatal("open %s: %r", file);
+	for(n=1; (p=Brdstr(b, '\n', 1)) != nil; free(p), n++){
+		q = p+strlen(p);
+		while(q > p && isspace((uchar)*(q-1)))
+			*--q = 0;
+		switch(p[0]){
+		case '\0':
+		case '#':
+			continue;
+		}
+		
+		inc = 0;
+		if(strncmp(p, "include ", 8) == 0){
+			inc = 1;
+		}else if(strncmp(p, "exclude ", 8) == 0){
+			inc = 0;
+		}else
+			sysfatal("%s:%d: line does not begin with include or exclude", file, n);
+
+		if(strchr(p+8, ' '))
+			fprint(2, "%s:%d: warning: space in pattern\n", file, n);
+
+		if((re = glob2regexp(p+8)) == nil)
+			sysfatal("%s:%d: bad glob pattern", file, n);
+
+		pattern = vtrealloc(pattern, (npattern+1)*sizeof pattern[0]);
+		pattern[npattern].re = re;
+		pattern[npattern].include = inc;
+		npattern++;
+	}
+	Bterm(b);
+}
+
+void
+excludepattern(char *p)
+{
+	Reprog *re;
+	
+	if((re = glob2regexp(p)) == nil)
+		sysfatal("bad glob pattern %s", p);
+
+	pattern = vtrealloc(pattern, (npattern+1)*sizeof pattern[0]);
+	pattern[npattern].re = re;
+	pattern[npattern].include = 0;
+	npattern++;
+}
+
+int
+includefile(char *file)
+{
+	Pattern *p, *ep;
+	
+	for(p=pattern, ep=p+npattern; p<ep; p++)
+		if(regexec(p->re, file, nil, 0))
+			return p->include;
+	return 1;
+}
+
diff --git a/src/cmd/vac/mkfile b/src/cmd/vac/mkfile
index 98e122b8..6ee651cb 100644
--- a/src/cmd/vac/mkfile
+++ b/src/cmd/vac/mkfile
@@ -4,6 +4,7 @@ LIBFILES=\
 	error\
 	file\
 	pack\
+	glob\
 
 LIB=${LIBFILES:%=%.$O} $PLAN9/lib/libventi.a
 
@@ -20,3 +21,10 @@ TARG=vac vacfs unvac
 default:V: all
 
 <$PLAN9/src/mkmany
+
+testglob:V: $O.testinc
+	$O.testinc exc <exc.in >exc.test
+	diff exc.out exc.test
+	ls -l exc.out exc.test
+
+
diff --git a/src/cmd/vac/stdinc.h b/src/cmd/vac/stdinc.h
index bf72d3fd..aa3b5f7c 100644
--- a/src/cmd/vac/stdinc.h
+++ b/src/cmd/vac/stdinc.h
@@ -1,5 +1,7 @@
 #include <u.h>
 #include <libc.h>
+#include <bio.h>
 #include <thread.h>
 #include <venti.h>
 #include <libsec.h>
+#include <regexp.h>
diff --git a/src/cmd/vac/testinc.c b/src/cmd/vac/testinc.c
new file mode 100644
index 00000000..0cd43143
--- /dev/null
+++ b/src/cmd/vac/testinc.c
@@ -0,0 +1,31 @@
+#include "stdinc.h"
+#include "vac.h"
+#include "dat.h"
+#include "fns.h"
+#include "error.h"
+
+void
+threadmain(int argc, char **argv)
+{
+	Biobuf b;
+	char *p;
+
+	ARGBEGIN{
+	default:
+		goto usage;
+	}ARGEND
+	
+	if(argc != 1){
+	usage:
+		fprint(2, "usage: testinc includefile\n");
+		threadexitsall("usage");
+	}
+	
+	loadexcludefile(argv[0]);
+	Binit(&b, 0, OREAD);
+	while((p = Brdline(&b, '\n')) != nil){
+		p[Blinelen(&b)-1] = 0;
+		print("%d %s\n", includefile(p), p);
+	}
+	threadexitsall(0);
+}
diff --git a/src/cmd/vac/vac.c b/src/cmd/vac/vac.c
index 183fa380..99651981 100644
--- a/src/cmd/vac/vac.c
+++ b/src/cmd/vac/vac.c
@@ -8,14 +8,13 @@
 void
 usage(void)
 {
-	fprint(2, "vac [-imqsv] [-b bsize] [-d old.vac] [-e exclude]... [-f new.vac] [-h host] file...\n");
+	fprint(2, "vac [-imqsv] [-a archive.vac] [-b bsize] [-d old.vac] [-f new.vac] [-e exclude]... [-h host] file...\n");
 	threadexitsall("usage");
 }
 
 enum
 {
 	BlockSize = 8*1024,
-	MaxExclude = 1000
 };
 
 struct
@@ -33,17 +32,16 @@ int verbose;
 char *host;
 VtConn *z;
 VacFs *fs;
-char *exclude[MaxExclude];
-int nexclude;
+char *archivefile;
 char *vacfile;
 
 int vacmerge(VacFile*, char*);
 void vac(VacFile*, VacFile*, char*, Dir*);
 void vacstdin(VacFile*, char*);
+VacFile *recentarchive(VacFs*, char*);
 
 static u64int unittoull(char*);
 static void warn(char *fmt, ...);
-static int strpcmp(const void*, const void*);
 static void removevacfile(void);
 
 #ifdef PLAN9PORT
@@ -81,6 +79,7 @@ threadmain(int argc, char **argv)
 	_p9usepwlibrary = 1;
 #endif
 
+	fmtinstall('F', vtfcallfmt);
 	fmtinstall('H', encodefmt);
 	fmtinstall('V', vtscorefmt);
 
@@ -89,7 +88,14 @@ threadmain(int argc, char **argv)
 	printstats = 0;
 	fsdiff = nil;
 	diffvac = nil;
+
 	ARGBEGIN{
+	case 'V':
+		chattyventi++;
+		break;
+	case 'a':
+		archivefile = EARGF(usage());
+		break;
 	case 'b':
 		u = unittoull(EARGF(usage()));
 		if(u < 512)
@@ -102,12 +108,7 @@ threadmain(int argc, char **argv)
 		diffvac = EARGF(usage());
 		break;
 	case 'e':
-		if(nexclude >= MaxExclude)
-			sysfatal("too many exclusions\n");
-		exclude[nexclude] = ARGF();
-		if(exclude[nexclude] == nil)
-			usage();
-		nexclude++;
+		excludepattern(EARGF(usage()));
 		break;
 	case 'f':
 		vacfile = EARGF(usage());
@@ -130,40 +131,101 @@ threadmain(int argc, char **argv)
 	case 'v':
 		verbose++;
 		break;
+	case 'x':
+		loadexcludefile(EARGF(usage()));
+		break;
 	default:
 		usage();
 	}ARGEND
 	
 	if(argc == 0 && !stdinname)
 		usage();
-
-	if(vacfile == nil)
-		outfd = 1;
-	else if((outfd = create(vacfile, OWRITE, 0666)) < 0)
-		sysfatal("create %s: %r", vacfile);
-	atexit(removevacfile);
-
-	qsort(exclude, nexclude, sizeof(char*), strpcmp);
+	
+	if(archivefile && (vacfile || diffvac)){
+		fprint(2, "cannot use -a with -f, -d\n");
+		usage();
+	}
 
 	z = vtdial(host);
 	if(z == nil)
 		sysfatal("could not connect to server: %r");
 	if(vtconnect(z) < 0)
 		sysfatal("vtconnect: %r");
-	
-	if(diffvac){
-		if((fsdiff = vacfsopen(z, diffvac, VtOREAD, 128)) == nil)
-			warn("vacfsopen %s: %r", diffvac);
-	}
 
-	if((fs = vacfscreate(z, blocksize, 512)) == nil)
-		sysfatal("vacfscreate: %r");
+	// Setup:
+	//	fs is the output vac file system
+	//	f is directory in output vac to write new files
+	//	fdiff is corresponding directory in existing vac
+	if(archivefile){
+		VacFile *fp;
+		char yyyy[5];
+		char mmdd[10];
+		char oldpath[40];
+		Tm tm;
 
-	f = vacfsgetroot(fs);
-	if(fsdiff)
-		fdiff = vacfsgetroot(fsdiff);
-	else
 		fdiff = nil;
+		if((outfd = open(archivefile, ORDWR)) < 0){
+			if(access(archivefile, 0) >= 0)
+				sysfatal("open %s: %r", archivefile);
+			if((outfd = create(archivefile, OWRITE, 0666)) < 0)
+				sysfatal("create %s: %r", archivefile);
+			atexit(removevacfile);	// because it is new
+			if((fs = vacfscreate(z, blocksize, 512)) == nil)
+				sysfatal("vacfscreate: %r");
+		}else{
+			if((fs = vacfsopen(z, archivefile, VtORDWR, 512)) == nil)
+				sysfatal("vacfsopen %s: %r", archivefile);
+			if((fdiff = recentarchive(fs, oldpath)) != nil){
+				if(verbose)
+					fprint(2, "diff %s\n", oldpath);
+			}else
+				if(verbose)
+					fprint(2, "no recent archive to diff against\n");
+		}
+
+		// Create yyyy/mmdd.
+		tm = *localtime(time(0));
+		snprint(yyyy, sizeof yyyy, "%04d", tm.year+1900);
+		fp = vacfsgetroot(fs);
+		if((f = vacfilewalk(fp, yyyy)) == nil
+		&& (f = vacfilecreate(fp, yyyy, ModeDir|0555)) == nil)
+			sysfatal("vacfscreate %s: %r", yyyy);
+		vacfiledecref(fp);
+		fp = f;
+
+		snprint(mmdd, sizeof mmdd, "%02d%02d", tm.mon+1, tm.mday);
+		n = 0;
+		while((f = vacfilewalk(fp, mmdd)) != nil){
+			vacfiledecref(f);
+			n++;
+			snprint(mmdd+4, sizeof mmdd-4, ".%d", n);
+		}
+		f = vacfilecreate(fp, mmdd, ModeDir|0555);
+		if(f == nil)
+			sysfatal("vacfscreate %s/%s: %r", yyyy, mmdd);
+		vacfiledecref(fp);
+
+		if(verbose)
+			fprint(2, "archive %s/%s\n", yyyy, mmdd);
+	}else{
+		if(vacfile == nil)
+			outfd = 1;
+		else if((outfd = create(vacfile, OWRITE, 0666)) < 0)
+			sysfatal("create %s: %r", vacfile);
+		atexit(removevacfile);
+		if((fs = vacfscreate(z, blocksize, 512)) == nil)
+			sysfatal("vacfscreate: %r");
+		f = vacfsgetroot(fs);
+
+		fdiff = nil;
+		if(diffvac){
+			if((fsdiff = vacfsopen(z, diffvac, VtOREAD, 128)) == nil)
+				warn("vacfsopen %s: %r", diffvac);
+			else
+				fdiff = vacfsgetroot(fsdiff);
+		}
+	}
+
 	if(stdinname)
 		vacstdin(f, stdinname);
 	for(i=0; i<argc; i++){
@@ -228,8 +290,8 @@ threadmain(int argc, char **argv)
 		fprint(2, "vacfssync: %r\n");
 
 	fprint(outfd, "vac:%V\n", fs->score);
-	vacfsclose(fs);
 	atexitdont(removevacfile);
+	vacfsclose(fs);
 	vthangup(z);
 
 	if(printstats){
@@ -243,37 +305,90 @@ threadmain(int argc, char **argv)
 	threadexitsall(0);
 }
 
-static void
-removevacfile(void)
+VacFile*
+recentarchive(VacFs *fs, char *path)
 {
-	if(vacfile)
-		remove(vacfile);
-}
+	VacFile *fp, *f;
+	VacDirEnum *de;
+	VacDir vd;
+	char buf[10];
+	int year, mmdd, nn, n, n1;
+	char *p;
+	
+	fp = vacfsgetroot(fs);
+	de = vdeopen(fp);
+	year = 0;
+	if(de){
+		for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
+			if(strlen(vd.elem) != 4)
+				continue;
+			if((n = strtol(vd.elem, &p, 10)) < 1900 || *p != 0)
+				continue;
+			if(year < n)
+				year = n;
+		}
+	}
+	vdeclose(de);
+	if(year == 0){
+		vacfiledecref(fp);
+		return nil;
+	}
+	snprint(buf, sizeof buf, "%04d", year);
+	if((f = vacfilewalk(fp, buf)) == nil){
+		fprint(2, "warning: dirread %s but cannot walk", buf);
+		vacfiledecref(fp);
+		return nil;
+	}
+	fp = f;
+	
+	de = vdeopen(fp);
+	mmdd = 0;
+	nn = 0;
+	if(de){
+		for(; vderead(de, &vd) > 0; vdcleanup(&vd)){
+			if(strlen(vd.elem) < 4)
+				continue;
+			if((n = strtol(vd.elem, &p, 10)) < 100 || n > 1231 || p != vd.elem+4)
+				continue;
+			if(*p == '.'){
+				if(p[1] == '0' || (n1 = strtol(p+1, &p, 10)) == 0 || *p != 0)
+					continue;
+			}else{
+				if(*p != 0)
+					continue;
+				n1 = 0;
+			}
+			if(n < mmdd || (n == mmdd && n1 < nn))
+				continue;
+			mmdd = n;
+			nn = n1;
+		}
+	}
+	vdeclose(de);
+	if(mmdd == 0){
+		vacfiledecref(fp);
+		return nil;
+	}
+	if(nn == 0)
+		snprint(buf, sizeof buf, "%04d", mmdd);
+	else
+		snprint(buf, sizeof buf, "%04d.%d", mmdd, nn);
+	if((f = vacfilewalk(fp, buf)) == nil){
+		fprint(2, "warning: dirread %s but cannot walk", buf);
+		vacfiledecref(fp);
+		return nil;
+	}
+	vacfiledecref(fp);
 
-static int
-strpcmp(const void *p0, const void *p1)
-{
-	return strcmp(*(char**)p0, *(char**)p1);
+	sprint(path, "%04d/%s", year, buf);
+	return f;
 }
 
-static int
-isexcluded(char *name)
+static void
+removevacfile(void)
 {
-	int bot, top, i, x;
-
-	bot = 0;	
-	top = nexclude;
-	while(bot < top) {
-		i = (bot+top)>>1;
-		x = strcmp(exclude[i], name);
-		if(x == 0)
-			return 1;
-		if(x < 0)
-			bot = i + 1;
-		else /* x > 0 */
-			top = i;
-	}
-	return 0;
+	if(vacfile)
+		remove(vacfile);
 }
 
 void
@@ -361,7 +476,7 @@ vac(VacFile *fp, VacFile *diffp, char *name, Dir *d)
 	VacFile *f, *fdiff;
 	VtEntry e;
 
-	if(isexcluded(name)){
+	if(!includefile(name)){
 		warn("excluding %s%s", name, (d->mode&DMDIR) ? "/" : "");
 		return;
 	}