4 files changed, 120 insertions, 18 deletions

diff --git a/src/libthread/channel.c b/src/libthread/channel.c
index 53af86e6..9efb7a62 100644
--- a/src/libthread/channel.c
+++ b/src/libthread/channel.c
@@ -141,7 +141,7 @@ altdequeue(Alt *a)
 			delarray(ar, i);
 			return;
 		}
-	fprint(2, "cannot find self in altdq\n");
+	fprint(2, "cannot find self in altdequeue\n");
 	abort();
 }
 
diff --git a/src/libthread/pthread.c b/src/libthread/pthread.c
index 46bb396a..5e022f0b 100644
--- a/src/libthread/pthread.c
+++ b/src/libthread/pthread.c
@@ -99,6 +99,23 @@ startprocfn(void *v)
 	pthread_exit(0);
 }
 
+static void
+startpthreadfn(void *v)
+{
+	void **a;
+	Proc *p;
+	_Thread *t;
+
+	a = (void**)v;
+	p = a[0];
+	t = a[1];
+	free(a);
+	t->osprocid = pthread_self();
+	pthread_detach(t->osprocid);
+	_threadpthreadmain(p, t);
+	pthread_exit(0);
+}
+
 void
 _procstart(Proc *p, void (*fn)(Proc*))
 {
@@ -116,6 +133,22 @@ _procstart(Proc *p, void (*fn)(Proc*))
 	}
 }
 
+void
+_threadpthreadstart(Proc *p, _Thread *t)
+{
+	void **a;
+
+	a = malloc(3*sizeof a[0]);
+	if(a == nil)
+		sysfatal("_pthreadstart malloc: %r");
+	a[0] = p;
+	a[1] = t;
+	if(pthread_create(&t->osprocid, nil, (void*(*)(void*))startpthreadfn, (void*)a) < 0){
+		fprint(2, "pthread_create: %r\n");
+		abort();
+	}
+}
+
 static pthread_key_t prockey;
 
 Proc*
diff --git a/src/libthread/thread.c b/src/libthread/thread.c
index f657b5b2..f579b567 100644
--- a/src/libthread/thread.c
+++ b/src/libthread/thread.c
@@ -7,6 +7,7 @@ static	uint		threadnsysproc;
 static	Lock		threadnproclock;
 static	Ref		threadidref;
 static	Proc		*threadmainproc;
+static	int		pthreadperthread;
 
 static	void		addproc(Proc*);
 static	void		delproc(Proc*);
@@ -176,12 +177,16 @@ _threadcreate(Proc *p, void (*fn)(void*), void *arg, uint stack)
 	if(stack < (256<<10))
 		stack = 256<<10;
 
-	if(p->nthread == 0)
+	if(p->nthread == 0 || pthreadperthread)
 		stack = 0; // not using it
 	t = threadalloc(fn, arg, stack);
 	t->proc = p;
-	addthreadinproc(p, t);
+	if(p->nthread == 0)
+		p->thread0 = t;
+	else if(pthreadperthread)
+		_threadpthreadstart(p, t);
 	p->nthread++;
+	addthreadinproc(p, t);
 	_threadready(t);
 	return t;
 }
@@ -209,6 +214,29 @@ proccreate(void (*fn)(void*), void *arg, uint stack)
 	return id;
 }
 
+// For pthreadperthread mode, procswitch flips
+// between the threads.
+static void
+procswitch(Proc *p, _Thread *from, _Thread *to)
+{
+// fprint(2, "procswitch %p %d %d\n", p, from?from->id:-1, to?to->id:-1);
+	lock(&p->schedlock);
+	from->schedrend.l = &p->schedlock;
+	if(to) {
+		p->schedthread = to;
+		to->schedrend.l = &p->schedlock;
+		_procwakeup(&to->schedrend);
+	}
+	if(p->schedthread != from) {
+		if(from->exiting) {
+			unlock(&p->schedlock);
+			_threadpexit();
+		}
+		_procsleep(&from->schedrend);
+	}
+	unlock(&p->schedlock);
+}
+
 void
 _threadswitch(void)
 {
@@ -216,9 +244,13 @@ _threadswitch(void)
 
 	needstack(0);
 	p = proc();
+
 /*print("threadswtch %p\n", p); */
-	if(p->thread->stk == nil)
+
+	if(p->thread == p->thread0)
 		procscheduler(p);
+	else if(pthreadperthread)
+		procswitch(p, p->thread, p->thread0);
 	else
 		contextswitch(&p->thread->context, &p->schedcontext);
 }
@@ -346,6 +378,15 @@ procmain(Proc *p)
 		threadexits(nil);
 }
 
+void
+_threadpthreadmain(Proc *p, _Thread *t)
+{
+	_threadsetproc(p);
+	procswitch(p, t, nil);
+	t->startfn(t->startarg);
+	threadexits(nil);
+}
+
 static void
 procscheduler(Proc *p)
 {
@@ -401,9 +442,12 @@ Top:
 		p->nswitch++;
 		_threaddebug("run %d (%s)", t->id, t->name);
 //print("run %p %p %p %p\n", t, *(uintptr*)(t->context.uc.mc.sp), t->context.uc.mc.di, t->context.uc.mc.si);
-		if(t->stk == nil)
+		if(t == p->thread0)
 			return;
-		contextswitch(&p->schedcontext, &t->context);
+		if(pthreadperthread)
+			procswitch(p, p->thread0, t);
+		else
+			contextswitch(&p->schedcontext, &t->context);
 /*print("back in scheduler\n"); */
 		goto Top;
 	}
@@ -757,10 +801,24 @@ int
 main(int argc, char **argv)
 {
 	Proc *p;
+	char *opts;
 
 	argv0 = argv[0];
 
-	if(getenv("NOLIBTHREADDAEMONIZE") == nil)
+	opts = getenv("LIBTHREAD");
+	if(opts == nil)
+		opts = "";
+
+	pthreadperthread = (strstr(opts, "pthreadperthread") != nil);
+#ifdef PLAN9PORT_ASAN
+	// ASAN can't deal with the coroutine stack switches.
+	// In theory it has support for informing it about stack switches,
+	// but even with those calls added it can't deal with things
+	// like fork or exit from a coroutine stack.
+	// Easier to just run in pthread-per-thread mode.
+	pthreadperthread = 1;
+#endif
+	if(strstr(opts, "nodaemon") || getenv("NOLIBTHREADDAEMONIZE") == nil)
 		_threadsetupdaemonize();
 
 	threadargc = argc;
diff --git a/src/libthread/threadimpl.h b/src/libthread/threadimpl.h
index cceb1b8e..c7373843 100644
--- a/src/libthread/threadimpl.h
+++ b/src/libthread/threadimpl.h
@@ -102,6 +102,17 @@ struct Execjob
 	Channel *c;
 };
 
+struct _Procrendez
+{
+	Lock		*l;
+	int		asleep;
+#ifdef PLAN9PORT_USING_PTHREADS
+	pthread_cond_t	cond;
+#else
+	int		pid;
+#endif
+};
+
 struct _Thread
 {
 	_Thread	*next;
@@ -112,6 +123,11 @@ struct _Thread
 	void	(*startfn)(void*);
 	void	*startarg;
 	uint	id;
+#ifdef PLAN9PORT_USING_PTHREADS
+	pthread_t	osprocid;
+#else
+	int		osprocid;
+#endif
 	uchar	*stk;
 	uint	stksize;
 	int		exiting;
@@ -120,17 +136,7 @@ struct _Thread
 	char	state[256];
 	void *udata;
 	Alt	*alt;
-};
-
-struct _Procrendez
-{
-	Lock		*l;
-	int		asleep;
-#ifdef PLAN9PORT_USING_PTHREADS
-	pthread_cond_t	cond;
-#else
-	int		pid;
-#endif
+	_Procrendez schedrend;
 };
 
 extern	void	_procsleep(_Procrendez*);
@@ -149,6 +155,7 @@ struct Proc
 #endif
 	Lock		lock;
 	int			nswitch;
+	_Thread		*thread0;
 	_Thread		*thread;
 	_Thread		*pinthread;
 	_Threadlist	runqueue;
@@ -157,6 +164,8 @@ struct Proc
 	uint		nthread;
 	uint		sysproc;
 	_Procrendez	runrend;
+	Lock		schedlock;
+	_Thread	*schedthread;
 	Context	schedcontext;
 	void		*udata;
 	Jmp		sigjmp;
@@ -188,6 +197,8 @@ extern void _threadpexit(void);
 extern void _threaddaemonize(void);
 extern void *_threadstkalloc(int);
 extern void _threadstkfree(void*, int);
+extern void _threadpthreadmain(Proc*, _Thread*);
+extern void _threadpthreadstart(Proc*, _Thread*);
 
 #define USPALIGN(ucp, align) \
 	(void*)((((uintptr)(ucp)->uc_stack.ss_sp+(ucp)->uc_stack.ss_size)-(align))&~((align)-1))