From baef953da253314657be9adea8f371bfbf4ba09e Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sun, 17 May 2020 08:24:54 -0400 Subject: libthread: add pthreadperthread mode and use under ASAN ASAN can't deal with the coroutine stacks. In theory we can call into ASAN runtime to let it know about them, but ASAN still has problems with fork or exit happening from a non-system stack. Bypass all possible problems by just having a full OS thread for each libthread thread. The threads are still cooperatively scheduled within a proc (in thos mode, a group of OS threads). Setting the environment variable LIBTHREAD=pthreadperthread will enable the pthreadperthread mode, as will building with CC9FLAGS='-fsanitize=address' in $PLAN9/config. This solution is much more general than ASAN - for example if you are trying to find all the thread stacks in a reproducible crash you can use pthreadperthread mode with any debugger that knows only about OS threads. --- src/libthread/channel.c | 2 +- src/libthread/pthread.c | 33 ++++++++++++++++++++++ src/libthread/thread.c | 70 ++++++++++++++++++++++++++++++++++++++++++---- src/libthread/threadimpl.h | 33 ++++++++++++++-------- 4 files changed, 120 insertions(+), 18 deletions(-) (limited to 'src/libthread') diff --git a/src/libthread/channel.c b/src/libthread/channel.c index 53af86e6..9efb7a62 100644 --- a/src/libthread/channel.c +++ b/src/libthread/channel.c @@ -141,7 +141,7 @@ altdequeue(Alt *a) delarray(ar, i); return; } - fprint(2, "cannot find self in altdq\n"); + fprint(2, "cannot find self in altdequeue\n"); abort(); } diff --git a/src/libthread/pthread.c b/src/libthread/pthread.c index 46bb396a..5e022f0b 100644 --- a/src/libthread/pthread.c +++ b/src/libthread/pthread.c @@ -99,6 +99,23 @@ startprocfn(void *v) pthread_exit(0); } +static void +startpthreadfn(void *v) +{ + void **a; + Proc *p; + _Thread *t; + + a = (void**)v; + p = a[0]; + t = a[1]; + free(a); + t->osprocid = pthread_self(); + pthread_detach(t->osprocid); + _threadpthreadmain(p, t); + pthread_exit(0); +} + void _procstart(Proc *p, void (*fn)(Proc*)) { @@ -116,6 +133,22 @@ _procstart(Proc *p, void (*fn)(Proc*)) } } +void +_threadpthreadstart(Proc *p, _Thread *t) +{ + void **a; + + a = malloc(3*sizeof a[0]); + if(a == nil) + sysfatal("_pthreadstart malloc: %r"); + a[0] = p; + a[1] = t; + if(pthread_create(&t->osprocid, nil, (void*(*)(void*))startpthreadfn, (void*)a) < 0){ + fprint(2, "pthread_create: %r\n"); + abort(); + } +} + static pthread_key_t prockey; Proc* diff --git a/src/libthread/thread.c b/src/libthread/thread.c index f657b5b2..f579b567 100644 --- a/src/libthread/thread.c +++ b/src/libthread/thread.c @@ -7,6 +7,7 @@ static uint threadnsysproc; static Lock threadnproclock; static Ref threadidref; static Proc *threadmainproc; +static int pthreadperthread; static void addproc(Proc*); static void delproc(Proc*); @@ -176,12 +177,16 @@ _threadcreate(Proc *p, void (*fn)(void*), void *arg, uint stack) if(stack < (256<<10)) stack = 256<<10; - if(p->nthread == 0) + if(p->nthread == 0 || pthreadperthread) stack = 0; // not using it t = threadalloc(fn, arg, stack); t->proc = p; - addthreadinproc(p, t); + if(p->nthread == 0) + p->thread0 = t; + else if(pthreadperthread) + _threadpthreadstart(p, t); p->nthread++; + addthreadinproc(p, t); _threadready(t); return t; } @@ -209,6 +214,29 @@ proccreate(void (*fn)(void*), void *arg, uint stack) return id; } +// For pthreadperthread mode, procswitch flips +// between the threads. +static void +procswitch(Proc *p, _Thread *from, _Thread *to) +{ +// fprint(2, "procswitch %p %d %d\n", p, from?from->id:-1, to?to->id:-1); + lock(&p->schedlock); + from->schedrend.l = &p->schedlock; + if(to) { + p->schedthread = to; + to->schedrend.l = &p->schedlock; + _procwakeup(&to->schedrend); + } + if(p->schedthread != from) { + if(from->exiting) { + unlock(&p->schedlock); + _threadpexit(); + } + _procsleep(&from->schedrend); + } + unlock(&p->schedlock); +} + void _threadswitch(void) { @@ -216,9 +244,13 @@ _threadswitch(void) needstack(0); p = proc(); + /*print("threadswtch %p\n", p); */ - if(p->thread->stk == nil) + + if(p->thread == p->thread0) procscheduler(p); + else if(pthreadperthread) + procswitch(p, p->thread, p->thread0); else contextswitch(&p->thread->context, &p->schedcontext); } @@ -346,6 +378,15 @@ procmain(Proc *p) threadexits(nil); } +void +_threadpthreadmain(Proc *p, _Thread *t) +{ + _threadsetproc(p); + procswitch(p, t, nil); + t->startfn(t->startarg); + threadexits(nil); +} + static void procscheduler(Proc *p) { @@ -401,9 +442,12 @@ Top: p->nswitch++; _threaddebug("run %d (%s)", t->id, t->name); //print("run %p %p %p %p\n", t, *(uintptr*)(t->context.uc.mc.sp), t->context.uc.mc.di, t->context.uc.mc.si); - if(t->stk == nil) + if(t == p->thread0) return; - contextswitch(&p->schedcontext, &t->context); + if(pthreadperthread) + procswitch(p, p->thread0, t); + else + contextswitch(&p->schedcontext, &t->context); /*print("back in scheduler\n"); */ goto Top; } @@ -757,10 +801,24 @@ int main(int argc, char **argv) { Proc *p; + char *opts; argv0 = argv[0]; - if(getenv("NOLIBTHREADDAEMONIZE") == nil) + opts = getenv("LIBTHREAD"); + if(opts == nil) + opts = ""; + + pthreadperthread = (strstr(opts, "pthreadperthread") != nil); +#ifdef PLAN9PORT_ASAN + // ASAN can't deal with the coroutine stack switches. + // In theory it has support for informing it about stack switches, + // but even with those calls added it can't deal with things + // like fork or exit from a coroutine stack. + // Easier to just run in pthread-per-thread mode. + pthreadperthread = 1; +#endif + if(strstr(opts, "nodaemon") || getenv("NOLIBTHREADDAEMONIZE") == nil) _threadsetupdaemonize(); threadargc = argc; diff --git a/src/libthread/threadimpl.h b/src/libthread/threadimpl.h index cceb1b8e..c7373843 100644 --- a/src/libthread/threadimpl.h +++ b/src/libthread/threadimpl.h @@ -102,6 +102,17 @@ struct Execjob Channel *c; }; +struct _Procrendez +{ + Lock *l; + int asleep; +#ifdef PLAN9PORT_USING_PTHREADS + pthread_cond_t cond; +#else + int pid; +#endif +}; + struct _Thread { _Thread *next; @@ -112,6 +123,11 @@ struct _Thread void (*startfn)(void*); void *startarg; uint id; +#ifdef PLAN9PORT_USING_PTHREADS + pthread_t osprocid; +#else + int osprocid; +#endif uchar *stk; uint stksize; int exiting; @@ -120,17 +136,7 @@ struct _Thread char state[256]; void *udata; Alt *alt; -}; - -struct _Procrendez -{ - Lock *l; - int asleep; -#ifdef PLAN9PORT_USING_PTHREADS - pthread_cond_t cond; -#else - int pid; -#endif + _Procrendez schedrend; }; extern void _procsleep(_Procrendez*); @@ -149,6 +155,7 @@ struct Proc #endif Lock lock; int nswitch; + _Thread *thread0; _Thread *thread; _Thread *pinthread; _Threadlist runqueue; @@ -157,6 +164,8 @@ struct Proc uint nthread; uint sysproc; _Procrendez runrend; + Lock schedlock; + _Thread *schedthread; Context schedcontext; void *udata; Jmp sigjmp; @@ -188,6 +197,8 @@ extern void _threadpexit(void); extern void _threaddaemonize(void); extern void *_threadstkalloc(int); extern void _threadstkfree(void*, int); +extern void _threadpthreadmain(Proc*, _Thread*); +extern void _threadpthreadstart(Proc*, _Thread*); #define USPALIGN(ucp, align) \ (void*)((((uintptr)(ucp)->uc_stack.ss_sp+(ucp)->uc_stack.ss_size)-(align))&~((align)-1)) -- cgit v1.2.3