From baef953da253314657be9adea8f371bfbf4ba09e Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sun, 17 May 2020 08:24:54 -0400 Subject: libthread: add pthreadperthread mode and use under ASAN ASAN can't deal with the coroutine stacks. In theory we can call into ASAN runtime to let it know about them, but ASAN still has problems with fork or exit happening from a non-system stack. Bypass all possible problems by just having a full OS thread for each libthread thread. The threads are still cooperatively scheduled within a proc (in thos mode, a group of OS threads). Setting the environment variable LIBTHREAD=pthreadperthread will enable the pthreadperthread mode, as will building with CC9FLAGS='-fsanitize=address' in $PLAN9/config. This solution is much more general than ASAN - for example if you are trying to find all the thread stacks in a reproducible crash you can use pthreadperthread mode with any debugger that knows only about OS threads. --- src/libthread/channel.c | 2 +- src/libthread/pthread.c | 33 ++++++++++++++++++++++ src/libthread/thread.c | 70 ++++++++++++++++++++++++++++++++++++++++++---- src/libthread/threadimpl.h | 33 ++++++++++++++-------- 4 files changed, 120 insertions(+), 18 deletions(-) (limited to 'src/libthread') diff --git a/src/libthread/channel.c b/src/libthread/channel.c index 53af86e6..9efb7a62 100644 --- a/src/libthread/channel.c +++ b/src/libthread/channel.c @@ -141,7 +141,7 @@ altdequeue(Alt *a) delarray(ar, i); return; } - fprint(2, "cannot find self in altdq\n"); + fprint(2, "cannot find self in altdequeue\n"); abort(); } diff --git a/src/libthread/pthread.c b/src/libthread/pthread.c index 46bb396a..5e022f0b 100644 --- a/src/libthread/pthread.c +++ b/src/libthread/pthread.c @@ -99,6 +99,23 @@ startprocfn(void *v) pthread_exit(0); } +static void +startpthreadfn(void *v) +{ + void **a; + Proc *p; + _Thread *t; + + a = (void**)v; + p = a[0]; + t = a[1]; + free(a); + t->osprocid = pthread_self(); + pthread_detach(t->osprocid); + _threadpthreadmain(p, t); + pthread_exit(0); +} + void _procstart(Proc *p, void (*fn)(Proc*)) { @@ -116,6 +133,22 @@ _procstart(Proc *p, void (*fn)(Proc*)) } } +void +_threadpthreadstart(Proc *p, _Thread *t) +{ + void **a; + + a = malloc(3*sizeof a[0]); + if(a == nil) + sysfatal("_pthreadstart malloc: %r"); + a[0] = p; + a[1] = t; + if(pthread_create(&t->osprocid, nil, (void*(*)(void*))startpthreadfn, (void*)a) < 0){ + fprint(2, "pthread_create: %r\n"); + abort(); + } +} + static pthread_key_t prockey; Proc* diff --git a/src/libthread/thread.c b/src/libthread/thread.c index f657b5b2..f579b567 100644 --- a/src/libthread/thread.c +++ b/src/libthread/thread.c @@ -7,6 +7,7 @@ static uint threadnsysproc; static Lock threadnproclock; static Ref threadidref; static Proc *threadmainproc; +static int pthreadperthread; static void addproc(Proc*); static void delproc(Proc*); @@ -176,12 +177,16 @@ _threadcreate(Proc *p, void (*fn)(void*), void *arg, uint stack) if(stack < (256<<10)) stack = 256<<10; - if(p->nthread == 0) + if(p->nthread == 0 || pthreadperthread) stack = 0; // not using it t = threadalloc(fn, arg, stack); t->proc = p; - addthreadinproc(p, t); + if(p->nthread == 0) + p->thread0 = t; + else if(pthreadperthread) + _threadpthreadstart(p, t); p->nthread++; + addthreadinproc(p, t); _threadready(t); return t; } @@ -209,6 +214,29 @@ proccreate(void (*fn)(void*), void *arg, uint stack) return id; } +// For pthreadperthread mode, procswitch flips +// between the threads. +static void +procswitch(Proc *p, _Thread *from, _Thread *to) +{ +// fprint(2, "procswitch %p %d %d\n", p, from?from->id:-1, to?to->id:-1); + lock(&p->schedlock); + from->schedrend.l = &p->schedlock; + if(to) { + p->schedthread = to; + to->schedrend.l = &p->schedlock; + _procwakeup(&to->schedrend); + } + if(p->schedthread != from) { + if(from->exiting) { + unlock(&p->schedlock); + _threadpexit(); + } + _procsleep(&from->schedrend); + } + unlock(&p->schedlock); +} + void _threadswitch(void) { @@ -216,9 +244,13 @@ _threadswitch(void) needstack(0); p = proc(); + /*print("threadswtch %p\n", p); */ - if(p->thread->stk == nil) + + if(p->thread == p->thread0) procscheduler(p); + else if(pthreadperthread) + procswitch(p, p->thread, p->thread0); else contextswitch(&p->thread->context, &p->schedcontext); } @@ -346,6 +378,15 @@ procmain(Proc *p) threadexits(nil); } +void +_threadpthreadmain(Proc *p, _Thread *t) +{ + _threadsetproc(p); + procswitch(p, t, nil); + t->startfn(t->startarg); + threadexits(nil); +} + static void procscheduler(Proc *p) { @@ -401,9 +442,12 @@ Top: p->nswitch++; _threaddebug("run %d (%s)", t->id, t->name); //print("run %p %p %p %p\n", t, *(uintptr*)(t->context.uc.mc.sp), t->context.uc.mc.di, t->context.uc.mc.si); - if(t->stk == nil) + if(t == p->thread0) return; - contextswitch(&p->schedcontext, &t->context); + if(pthreadperthread) + procswitch(p, p->thread0, t); + else + contextswitch(&p->schedcontext, &t->context); /*print("back in scheduler\n"); */ goto Top; } @@ -757,10 +801,24 @@ int main(int argc, char **argv) { Proc *p; + char *opts; argv0 = argv[0]; - if(getenv("NOLIBTHREADDAEMONIZE") == nil) + opts = getenv("LIBTHREAD"); + if(opts == nil) + opts = ""; + + pthreadperthread = (strstr(opts, "pthreadperthread") != nil); +#ifdef PLAN9PORT_ASAN + // ASAN can't deal with the coroutine stack switches. + // In theory it has support for informing it about stack switches, + // but even with those calls added it can't deal with things + // like fork or exit from a coroutine stack. + // Easier to just run in pthread-per-thread mode. + pthreadperthread = 1; +#endif + if(strstr(opts, "nodaemon") || getenv("NOLIBTHREADDAEMONIZE") == nil) _threadsetupdaemonize(); threadargc = argc; diff --git a/src/libthread/threadimpl.h b/src/libthread/threadimpl.h index cceb1b8e..c7373843 100644 --- a/src/libthread/threadimpl.h +++ b/src/libthread/threadimpl.h @@ -102,6 +102,17 @@ struct Execjob Channel *c; }; +struct _Procrendez +{ + Lock *l; + int asleep; +#ifdef PLAN9PORT_USING_PTHREADS + pthread_cond_t cond; +#else + int pid; +#endif +}; + struct _Thread { _Thread *next; @@ -112,6 +123,11 @@ struct _Thread void (*startfn)(void*); void *startarg; uint id; +#ifdef PLAN9PORT_USING_PTHREADS + pthread_t osprocid; +#else + int osprocid; +#endif uchar *stk; uint stksize; int exiting; @@ -120,17 +136,7 @@ struct _Thread char state[256]; void *udata; Alt *alt; -}; - -struct _Procrendez -{ - Lock *l; - int asleep; -#ifdef PLAN9PORT_USING_PTHREADS - pthread_cond_t cond; -#else - int pid; -#endif + _Procrendez schedrend; }; extern void _procsleep(_Procrendez*); @@ -149,6 +155,7 @@ struct Proc #endif Lock lock; int nswitch; + _Thread *thread0; _Thread *thread; _Thread *pinthread; _Threadlist runqueue; @@ -157,6 +164,8 @@ struct Proc uint nthread; uint sysproc; _Procrendez runrend; + Lock schedlock; + _Thread *schedthread; Context schedcontext; void *udata; Jmp sigjmp; @@ -188,6 +197,8 @@ extern void _threadpexit(void); extern void _threaddaemonize(void); extern void *_threadstkalloc(int); extern void _threadstkfree(void*, int); +extern void _threadpthreadmain(Proc*, _Thread*); +extern void _threadpthreadstart(Proc*, _Thread*); #define USPALIGN(ucp, align) \ (void*)((((uintptr)(ucp)->uc_stack.ss_sp+(ucp)->uc_stack.ss_size)-(align))&~((align)-1)) -- cgit v1.2.3 From 162d0d5cd94fabab822ac66655be8957151cef99 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sun, 17 May 2020 22:31:38 -0400 Subject: libthread: handle spurious _procsleep wakeups, fix $LIBTHREAD handling --- src/libthread/pthread.c | 4 +++- src/libthread/thread.c | 28 +++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) (limited to 'src/libthread') diff --git a/src/libthread/pthread.c b/src/libthread/pthread.c index 5e022f0b..35f6ffe3 100644 --- a/src/libthread/pthread.c +++ b/src/libthread/pthread.c @@ -50,13 +50,15 @@ _threadunlock(Lock *lk, ulong pc) abort(); } +/* note: _procsleep can have spurious wakeups, like pthread_cond_wait */ void _procsleep(_Procrendez *r) { /* r is protected by r->l, which we hold */ pthread_cond_init(&r->cond, 0); r->asleep = 1; - pthread_cond_wait(&r->cond, &r->l->mutex); + if(pthread_cond_wait(&r->cond, &r->l->mutex) != 0) + sysfatal("pthread_cond_wait: %r"); pthread_cond_destroy(&r->cond); r->asleep = 0; } diff --git a/src/libthread/thread.c b/src/libthread/thread.c index f579b567..902942d9 100644 --- a/src/libthread/thread.c +++ b/src/libthread/thread.c @@ -54,9 +54,9 @@ _threaddebug(char *fmt, ...) va_end(arg); t = proc()->thread; if(t) - fprint(fd, "%d.%d: %s\n", getpid(), t->id, buf); + fprint(fd, "%p %d.%d: %s\n", proc(), getpid(), t->id, buf); else - fprint(fd, "%d._: %s\n", getpid(), buf); + fprint(fd, "%p %d._: %s\n", proc(), getpid(), buf); } static _Thread* @@ -219,20 +219,28 @@ proccreate(void (*fn)(void*), void *arg, uint stack) static void procswitch(Proc *p, _Thread *from, _Thread *to) { -// fprint(2, "procswitch %p %d %d\n", p, from?from->id:-1, to?to->id:-1); + _threaddebug("procswitch %p %d %d", p, from?from->id:-1, to?to->id:-1); lock(&p->schedlock); from->schedrend.l = &p->schedlock; if(to) { p->schedthread = to; to->schedrend.l = &p->schedlock; + _threaddebug("procswitch wakeup %p %d", p, to->id); _procwakeup(&to->schedrend); } if(p->schedthread != from) { if(from->exiting) { unlock(&p->schedlock); _threadpexit(); + _threaddebug("procswitch exit wakeup!!!\n"); } - _procsleep(&from->schedrend); + while(p->schedthread != from) { + _threaddebug("procswitch sleep %p %d", p, from->id); + _procsleep(&from->schedrend); + _threaddebug("procswitch awake %p %d", p, from->id); + } + if(p->schedthread != from) + sysfatal("_procswitch %p %p oops", p->schedthread, from); } unlock(&p->schedlock); } @@ -448,6 +456,7 @@ Top: procswitch(p, p->thread0, t); else contextswitch(&p->schedcontext, &t->context); + _threaddebug("back in scheduler"); /*print("back in scheduler\n"); */ goto Top; } @@ -589,6 +598,10 @@ threadqlock(QLock *l, int block, ulong pc) if(l->owner == nil){ l->owner = (*threadnow)(); /*print("qlock %p @%#x by %p\n", l, pc, l->owner); */ + if(l->owner == nil) { + fprint(2, "%s: qlock uncontended owner=nil oops\n", argv0); + abort(); + } unlock(&l->l); return 1; } @@ -613,6 +626,11 @@ threadqlock(QLock *l, int block, ulong pc) argv0, pc, l->owner, (*threadnow)()); abort(); } + if(l->owner == nil) { + fprint(2, "%s: qlock threadswitch owner=nil oops\n", argv0); + abort(); + } + /*print("qlock wakeup %p @%#x by %p\n", l, pc, (*threadnow)()); */ return 1; } @@ -818,7 +836,7 @@ main(int argc, char **argv) // Easier to just run in pthread-per-thread mode. pthreadperthread = 1; #endif - if(strstr(opts, "nodaemon") || getenv("NOLIBTHREADDAEMONIZE") == nil) + if(strstr(opts, "nodaemon") == nil && getenv("NOLIBTHREADDAEMONIZE") == nil) _threadsetupdaemonize(); threadargc = argc; -- cgit v1.2.3 From 329831171dd6ef81c113f101093c7b4947381003 Mon Sep 17 00:00:00 2001 From: Xiao-Yong Date: Mon, 15 Jun 2020 22:18:03 -0500 Subject: libthread: use libc functions in ucontext for macOS (#417) --- src/libthread/Darwin-x86_64-asm.s | 44 --------------------------------------- src/libthread/sysofiles.sh | 4 ++-- src/libthread/threadimpl.h | 36 +------------------------------- 3 files changed, 3 insertions(+), 81 deletions(-) delete mode 100644 src/libthread/Darwin-x86_64-asm.s (limited to 'src/libthread') diff --git a/src/libthread/Darwin-x86_64-asm.s b/src/libthread/Darwin-x86_64-asm.s deleted file mode 100644 index d50d3b6d..00000000 --- a/src/libthread/Darwin-x86_64-asm.s +++ /dev/null @@ -1,44 +0,0 @@ -.text -.align 8 - -.globl _libthread_getmcontext -_libthread_getmcontext: - movq $1, 0*8(%rdi) // rax - movq %rbx, 1*8(%rdi) - movq %rcx, 2*8(%rdi) - movq %rdx, 3*8(%rdi) - movq %rsi, 4*8(%rdi) - movq %rdi, 5*8(%rdi) - movq %rbp, 6*8(%rdi) - movq %rsp, 7*8(%rdi) - movq %r8, 8*8(%rdi) - movq %r9, 9*8(%rdi) - movq %r10, 10*8(%rdi) - movq %r11, 11*8(%rdi) - movq %r12, 12*8(%rdi) - movq %r13, 13*8(%rdi) - movq %r14, 14*8(%rdi) - movq %r15, 15*8(%rdi) - movq $0, %rax - ret - -.globl _libthread_setmcontext -_libthread_setmcontext: - movq 0*8(%rdi), %rax - movq 1*8(%rdi), %rbx - movq 2*8(%rdi), %rcx - movq 3*8(%rdi), %rdx - movq 4*8(%rdi), %rsi - // %rdi later - movq 6*8(%rdi), %rbp - movq 7*8(%rdi), %rsp - movq 8*8(%rdi), %r8 - movq 9*8(%rdi), %r9 - movq 10*8(%rdi), %r10 - movq 11*8(%rdi), %r11 - movq 12*8(%rdi), %r12 - movq 13*8(%rdi), %r13 - movq 14*8(%rdi), %r14 - movq 15*8(%rdi), %r15 - movq 5*8(%rdi), %rdi - ret diff --git a/src/libthread/sysofiles.sh b/src/libthread/sysofiles.sh index 20811cdf..833afbe0 100644 --- a/src/libthread/sysofiles.sh +++ b/src/libthread/sysofiles.sh @@ -15,14 +15,14 @@ esac # Various libc don't supply swapcontext, makecontext, so we do. case "$SYSNAME-$OBJTYPE" in -Darwin-x86_64 | Linux-arm | Linux-sparc64 | NetBSD-arm | OpenBSD-386 | OpenBSD-power | OpenBSD-x86_64) +Linux-arm | Linux-sparc64 | NetBSD-arm | OpenBSD-386 | OpenBSD-power | OpenBSD-x86_64) echo $OBJTYPE-ucontext.o ;; esac # A few libc don't supply setcontext, getcontext, so we do. case "$SYSNAME-$OBJTYPE" in -Darwin-x86_64 | Linux-arm | Linux-sparc64 | OpenBSD-386 | OpenBSD-power | OpenBSD-x86_64) +Linux-arm | Linux-sparc64 | OpenBSD-386 | OpenBSD-power | OpenBSD-x86_64) echo $SYSNAME-$OBJTYPE-asm.o ;; esac diff --git a/src/libthread/threadimpl.h b/src/libthread/threadimpl.h index c7373843..8d22a161 100644 --- a/src/libthread/threadimpl.h +++ b/src/libthread/threadimpl.h @@ -7,7 +7,7 @@ #include #if !defined(__OpenBSD__) # if defined(__APPLE__) -# define _XOPEN_SOURCE /* for Snow Leopard */ +# define _XOPEN_SOURCE /* for Snow Leopard */ # endif # include #endif @@ -15,31 +15,6 @@ #include "libc.h" #include "thread.h" -#if defined(__APPLE__) - /* - * OS X before 10.5 (Leopard) does not provide - * swapcontext nor makecontext, so we have to use our own. - * In theory, Leopard does provide them, but when we use - * them, they seg fault. Maybe we're using them wrong. - * So just use our own versions, even on Leopard. - */ -# define mcontext libthread_mcontext -# define mcontext_t libthread_mcontext_t -# define ucontext libthread_ucontext -# define ucontext_t libthread_ucontext_t -# define swapcontext libthread_swapcontext -# define makecontext libthread_makecontext -# if defined(__i386__) -# include "386-ucontext.h" -# elif defined(__x86_64__) -# include "x86_64-ucontext.h" -# elif defined(__ppc__) || defined(__power__) -# include "power-ucontext.h" -# else -# error "unknown architecture" -# endif -#endif - #if defined(__OpenBSD__) # define mcontext libthread_mcontext # define mcontext_t libthread_mcontext_t @@ -82,15 +57,6 @@ enum struct Context { ucontext_t uc; -#ifdef __APPLE__ - /* - * On Snow Leopard, etc., the context routines exist, - * so we use them, but apparently they write past the - * end of the ucontext_t. Sigh. We put some extra - * scratch space here for them. - */ - uchar buf[1024]; -#endif }; struct Execjob -- cgit v1.2.3