/*-*- Mode: C; c-basic-offset: 8 -*-*/
/***
This file is part of mutrace.
Copyright 2009 Lennart Poettering
mutrace is free software: you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
mutrace is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with mutrace. If not, see .
***/
#include "config.h"
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#if !defined (__linux__) || !defined(__GLIBC__)
#error "This stuff only works on Linux!"
#endif
#ifndef SCHED_RESET_ON_FORK
/* "Your libc lacks the definition of SCHED_RESET_ON_FORK. We'll now
* define it ourselves, however make sure your kernel is new
* enough! */
#define SCHED_RESET_ON_FORK 0x40000000
#endif
#if defined(__i386__) || defined(__x86_64__)
#define DEBUG_TRAP __asm__("int $3")
#else
#include
#define DEBUG_TRAP raise(SIGTRAP)
#endif
#define LIKELY(x) (__builtin_expect(!!(x),1))
#define UNLIKELY(x) (__builtin_expect(!!(x),0))
struct stacktrace_info {
void **frames;
int nb_frame;
};
struct mutex_info {
pthread_mutex_t *mutex;
pthread_rwlock_t *rwlock;
int type, protocol, kind;
bool broken:1;
bool realtime:1;
bool dead:1;
unsigned n_lock_level;
pid_t last_owner;
unsigned n_locked;
unsigned n_owner_changed;
unsigned n_contended;
uint64_t nsec_locked_total;
uint64_t nsec_locked_max;
uint64_t nsec_timestamp;
struct stacktrace_info stacktrace;
unsigned id;
struct mutex_info *next;
};
static unsigned hash_size = 3371; /* probably a good idea to pick a prime here */
static unsigned frames_max = 16;
static volatile unsigned n_broken = 0;
static volatile unsigned n_collisions = 0;
static volatile unsigned n_self_contended = 0;
static unsigned show_n_locked_min = 1;
static unsigned show_n_owner_changed_min = 2;
static unsigned show_n_contended_min = 0;
static unsigned show_n_max = 10;
static bool raise_trap = false;
static bool track_rt = false;
static int (*real_pthread_mutex_init)(pthread_mutex_t *mutex, const pthread_mutexattr_t *mutexattr) = NULL;
static int (*real_pthread_mutex_destroy)(pthread_mutex_t *mutex) = NULL;
static int (*real_pthread_mutex_lock)(pthread_mutex_t *mutex) = NULL;
static int (*real_pthread_mutex_trylock)(pthread_mutex_t *mutex) = NULL;
static int (*real_pthread_mutex_timedlock)(pthread_mutex_t *mutex, const struct timespec *abstime) = NULL;
static int (*real_pthread_mutex_unlock)(pthread_mutex_t *mutex) = NULL;
static int (*real_pthread_cond_wait)(pthread_cond_t *cond, pthread_mutex_t *mutex) = NULL;
static int (*real_pthread_cond_timedwait)(pthread_cond_t *cond, pthread_mutex_t *mutex, const struct timespec *abstime) = NULL;
static int (*real_pthread_create)(pthread_t *newthread, const pthread_attr_t *attr, void *(*start_routine) (void *), void *arg) = NULL;
static int (*real_pthread_rwlock_init)(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) = NULL;
static int (*real_pthread_rwlock_destroy)(pthread_rwlock_t *rwlock) = NULL;
static int (*real_pthread_rwlock_rdlock)(pthread_rwlock_t *rwlock) = NULL;
static int (*real_pthread_rwlock_tryrdlock)(pthread_rwlock_t *rwlock) = NULL;
static int (*real_pthread_rwlock_timedrdlock)(pthread_rwlock_t *rwlock, const struct timespec *abstime) = NULL;
static int (*real_pthread_rwlock_wrlock)(pthread_rwlock_t *rwlock) = NULL;
static int (*real_pthread_rwlock_trywrlock)(pthread_rwlock_t *rwlock) = NULL;
static int (*real_pthread_rwlock_timedwrlock)(pthread_rwlock_t *rwlock, const struct timespec *abstime) = NULL;
static int (*real_pthread_rwlock_unlock)(pthread_rwlock_t *rwlock);
static void (*real_exit)(int status) __attribute__((noreturn)) = NULL;
static void (*real__exit)(int status) __attribute__((noreturn)) = NULL;
static void (*real__Exit)(int status) __attribute__((noreturn)) = NULL;
static int (*real_backtrace)(void **array, int size) = NULL;
static char **(*real_backtrace_symbols)(void *const *array, int size) = NULL;
static void (*real_backtrace_symbols_fd)(void *const *array, int size, int fd) = NULL;
static struct mutex_info **alive_mutexes = NULL, **dead_mutexes = NULL;
static pthread_mutex_t *mutexes_lock = NULL;
static __thread bool recursive = false;
static volatile bool initialized = false;
static volatile bool threads_existing = false;
static uint64_t nsec_timestamp_setup;
static void setup(void) __attribute ((constructor));
static void shutdown(void) __attribute ((destructor));
static char *stacktrace_to_string(struct stacktrace_info stacktrace);
static pid_t _gettid(void) {
return (pid_t) syscall(SYS_gettid);
}
static uint64_t nsec_now(void) {
struct timespec ts;
int r;
r = clock_gettime(CLOCK_MONOTONIC, &ts);
assert(r == 0);
return
(uint64_t) ts.tv_sec * 1000000000ULL +
(uint64_t) ts.tv_nsec;
}
static const char *get_prname(void) {
static char prname[17];
int r;
r = prctl(PR_GET_NAME, prname);
assert(r == 0);
prname[16] = 0;
return prname;
}
static int parse_env(const char *n, unsigned *t) {
const char *e;
char *x = NULL;
unsigned long ul;
if (!(e = getenv(n)))
return 0;
errno = 0;
ul = strtoul(e, &x, 0);
if (!x || *x || errno != 0)
return -1;
*t = (unsigned) ul;
if ((unsigned long) *t != ul)
return -1;
return 0;
}
#define LOAD_FUNC(name) \
do { \
*(void**) (&real_##name) = dlsym(RTLD_NEXT, #name); \
assert(real_##name); \
} while (false)
#define LOAD_FUNC_VERSIONED(name, version) \
do { \
*(void**) (&real_##name) = dlvsym(RTLD_NEXT, #name, version); \
assert(real_##name); \
} while (false)
static void load_functions(void) {
static volatile bool loaded = false;
if (LIKELY(loaded))
return;
recursive = true;
/* If someone uses a shared library constructor that is called
* before ours we might not be initialized yet when the first
* lock related operation is executed. To deal with this we'll
* simply call the original implementation and do nothing
* else, but for that we do need the original function
* pointers. */
LOAD_FUNC(pthread_mutex_init);
LOAD_FUNC(pthread_mutex_destroy);
LOAD_FUNC(pthread_mutex_lock);
LOAD_FUNC(pthread_mutex_trylock);
LOAD_FUNC(pthread_mutex_timedlock);
LOAD_FUNC(pthread_mutex_unlock);
LOAD_FUNC(pthread_create);
LOAD_FUNC(pthread_rwlock_init);
LOAD_FUNC(pthread_rwlock_destroy);
LOAD_FUNC(pthread_rwlock_rdlock);
LOAD_FUNC(pthread_rwlock_tryrdlock);
LOAD_FUNC(pthread_rwlock_timedrdlock);
LOAD_FUNC(pthread_rwlock_wrlock);
LOAD_FUNC(pthread_rwlock_trywrlock);
LOAD_FUNC(pthread_rwlock_timedwrlock);
LOAD_FUNC(pthread_rwlock_unlock);
/* There's some kind of weird incompatibility problem causing
* pthread_cond_timedwait() to freeze if we don't ask for this
* explicit version of these functions */
LOAD_FUNC_VERSIONED(pthread_cond_wait, "GLIBC_2.3.2");
LOAD_FUNC_VERSIONED(pthread_cond_timedwait, "GLIBC_2.3.2");
LOAD_FUNC(exit);
LOAD_FUNC(_exit);
LOAD_FUNC(_Exit);
LOAD_FUNC(backtrace);
LOAD_FUNC(backtrace_symbols);
LOAD_FUNC(backtrace_symbols_fd);
loaded = true;
recursive = false;
}
static void setup(void) {
pthread_mutex_t *m, *last;
int r;
unsigned t;
load_functions();
if (LIKELY(initialized))
return;
if (!dlsym(NULL, "main"))
fprintf(stderr,
"mutrace: Application appears to be compiled without -rdynamic. It might be a\n"
"mutrace: good idea to recompile with -rdynamic enabled since this produces more\n"
"mutrace: useful stack traces.\n\n");
if (__malloc_hook) {
fprintf(stderr,
"mutrace: Detected non-glibc memory allocator. Your program uses some\n"
"mutrace: alternative memory allocator (jemalloc?) which is not compatible with\n"
"mutrace: mutrace. Please rebuild your program with the standard memory\n"
"mutrace: allocator or fix mutrace to handle yours correctly.\n");
/* The reason for this is that jemalloc and other
* allocators tend to call pthread_mutex_xxx() from
* the allocator. However, we need to call malloc()
* ourselves from some mutex operations so this might
* create an endless loop eventually overflowing the
* stack. glibc's malloc() does locking too but uses
* lock routines that do not end up calling
* pthread_mutex_xxx(). */
real_exit(1);
}
t = hash_size;
if (parse_env("MUTRACE_HASH_SIZE", &t) < 0 || t <= 0)
fprintf(stderr, "mutrace: WARNING: Failed to parse $MUTRACE_HASH_SIZE.\n");
else
hash_size = t;
t = frames_max;
if (parse_env("MUTRACE_FRAMES", &t) < 0 || t <= 0)
fprintf(stderr, "mutrace: WARNING: Failed to parse $MUTRACE_FRAMES.\n");
else
frames_max = t;
t = show_n_locked_min;
if (parse_env("MUTRACE_LOCKED_MIN", &t) < 0)
fprintf(stderr, "mutrace: WARNING: Failed to parse $MUTRACE_LOCKED_MIN.\n");
else
show_n_locked_min = t;
t = show_n_owner_changed_min;
if (parse_env("MUTRACE_OWNER_CHANGED_MIN", &t) < 0)
fprintf(stderr, "mutrace: WARNING: Failed to parse $MUTRACE_OWNER_CHANGED_MIN.\n");
else
show_n_owner_changed_min = t;
t = show_n_contended_min;
if (parse_env("MUTRACE_CONTENDED_MIN", &t) < 0)
fprintf(stderr, "mutrace: WARNING: Failed to parse $MUTRACE_CONTENDED_MIN.\n");
else
show_n_contended_min = t;
t = show_n_max;
if (parse_env("MUTRACE_MAX", &t) < 0)
fprintf(stderr, "mutrace: WARNING: Failed to parse $MUTRACE_MAX.\n");
else
show_n_max = t;
if (getenv("MUTRACE_TRAP"))
raise_trap = true;
if (getenv("MUTRACE_TRACK_RT"))
track_rt = true;
alive_mutexes = calloc(hash_size, sizeof(struct mutex_info*));
assert(alive_mutexes);
dead_mutexes = calloc(hash_size, sizeof(struct mutex_info*));
assert(dead_mutexes);
mutexes_lock = malloc(hash_size * sizeof(pthread_mutex_t));
assert(mutexes_lock);
for (m = mutexes_lock, last = mutexes_lock+hash_size; m < last; m++) {
r = real_pthread_mutex_init(m, NULL);
assert(r == 0);
}
nsec_timestamp_setup = nsec_now();
initialized = true;
fprintf(stderr, "mutrace: "PACKAGE_VERSION" sucessfully initialized for process %s (pid %lu).\n",
get_prname(), (unsigned long) getpid());
}
static unsigned long mutex_hash(pthread_mutex_t *mutex) {
unsigned long u;
u = (unsigned long) mutex;
u /= sizeof(void*);
return u % hash_size;
}
static unsigned long rwlock_hash(pthread_rwlock_t *rwlock) {
unsigned long u;
u = (unsigned long) rwlock;
u /= sizeof(void*);
return u % hash_size;
}
static void lock_hash_mutex(unsigned u) {
int r;
r = real_pthread_mutex_trylock(mutexes_lock + u);
if (UNLIKELY(r == EBUSY)) {
__sync_fetch_and_add(&n_self_contended, 1);
r = real_pthread_mutex_lock(mutexes_lock + u);
}
assert(r == 0);
}
static void unlock_hash_mutex(unsigned u) {
int r;
r = real_pthread_mutex_unlock(mutexes_lock + u);
assert(r == 0);
}
static int mutex_info_compare(const void *_a, const void *_b) {
const struct mutex_info
*a = *(const struct mutex_info**) _a,
*b = *(const struct mutex_info**) _b;
if (a->n_contended > b->n_contended)
return -1;
else if (a->n_contended < b->n_contended)
return 1;
if (a->n_owner_changed > b->n_owner_changed)
return -1;
else if (a->n_owner_changed < b->n_owner_changed)
return 1;
if (a->n_locked > b->n_locked)
return -1;
else if (a->n_locked < b->n_locked)
return 1;
if (a->nsec_locked_max > b->nsec_locked_max)
return -1;
else if (a->nsec_locked_max < b->nsec_locked_max)
return 1;
/* Let's make the output deterministic */
if (a > b)
return -1;
else if (a < b)
return 1;
return 0;
}
static bool mutex_info_show(struct mutex_info *mi) {
/* Mutexes used by real-time code are always noteworthy */
if (mi->realtime)
return true;
if (mi->n_locked < show_n_locked_min)
return false;
if (mi->n_owner_changed < show_n_owner_changed_min)
return false;
if (mi->n_contended < show_n_contended_min)
return false;
return true;
}
static bool mutex_info_dump(struct mutex_info *mi) {
char *stacktrace_str;
if (!mutex_info_show(mi))
return false;
stacktrace_str = stacktrace_to_string(mi->stacktrace);
fprintf(stderr,
"\nMutex #%u (0x%p) first referenced by:\n"
"%s", mi->id, mi->mutex ? (void*) mi->mutex : (void*) mi->rwlock, stacktrace_str);
free(stacktrace_str);
return true;
}
static char mutex_type_name(int type) {
switch (type) {
case PTHREAD_MUTEX_NORMAL:
return '-';
case PTHREAD_MUTEX_RECURSIVE:
return 'r';
case PTHREAD_MUTEX_ERRORCHECK:
return 'e';
case PTHREAD_MUTEX_ADAPTIVE_NP:
return 'a';
default:
return '?';
}
}
static char mutex_protocol_name(int protocol) {
switch (protocol) {
case PTHREAD_PRIO_NONE:
return '-';
case PTHREAD_PRIO_INHERIT:
return 'i';
case PTHREAD_PRIO_PROTECT:
return 'p';
default:
return '?';
}
}
static char rwlock_kind_name(int kind) {
switch (kind) {
case PTHREAD_RWLOCK_PREFER_READER_NP:
return 'r';
case PTHREAD_RWLOCK_PREFER_WRITER_NP:
return 'w';
case PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP:
return 'W';
default:
return '?';
}
}
static bool mutex_info_stat(struct mutex_info *mi) {
if (!mutex_info_show(mi))
return false;
fprintf(stderr,
"%8u %8u %8u %8u %12.3f %12.3f %12.3f %c%c%c%c%c%c\n",
mi->id,
mi->n_locked,
mi->n_owner_changed,
mi->n_contended,
(double) mi->nsec_locked_total / 1000000.0,
(double) mi->nsec_locked_total / mi->n_locked / 1000000.0,
(double) mi->nsec_locked_max / 1000000.0,
mi->mutex ? 'M' : 'W',
mi->broken ? '!' : (mi->dead ? 'x' : '-'),
track_rt ? (mi->realtime ? 'R' : '-') : '.',
mi->mutex ? mutex_type_name(mi->type) : '.',
mi->mutex ? mutex_protocol_name(mi->protocol) : '.',
mi->rwlock ? rwlock_kind_name(mi->kind) : '.');
return true;
}
static void show_summary(void) {
static pthread_mutex_t summary_mutex = PTHREAD_MUTEX_INITIALIZER;
static bool shown_summary = false;
struct mutex_info *mi, **table;
unsigned n, u, i, m;
uint64_t t;
long n_cpus;
real_pthread_mutex_lock(&summary_mutex);
if (shown_summary)
goto finish;
t = nsec_now() - nsec_timestamp_setup;
fprintf(stderr,
"\n"
"mutrace: Showing statistics for process %s (pid %lu).\n", get_prname(), (unsigned long) getpid());
n = 0;
for (u = 0; u < hash_size; u++) {
lock_hash_mutex(u);
for (mi = alive_mutexes[u]; mi; mi = mi->next)
n++;
for (mi = dead_mutexes[u]; mi; mi = mi->next)
n++;
}
if (n <= 0) {
fprintf(stderr,
"mutrace: No mutexes used.\n");
goto finish;
}
fprintf(stderr,
"mutrace: %u mutexes used.\n", n);
table = malloc(sizeof(struct mutex_info*) * n);
i = 0;
for (u = 0; u < hash_size; u++) {
for (mi = alive_mutexes[u]; mi; mi = mi->next) {
mi->id = i;
table[i++] = mi;
}
for (mi = dead_mutexes[u]; mi; mi = mi->next) {
mi->id = i;
table[i++] = mi;
}
}
assert(i == n);
qsort(table, n, sizeof(table[0]), mutex_info_compare);
for (i = 0, m = 0; i < n && (show_n_max <= 0 || m < show_n_max); i++)
m += mutex_info_dump(table[i]) ? 1 : 0;
if (m > 0) {
fprintf(stderr,
"\n"
"mutrace: Showing %u most contended mutexes:\n"
"\n"
" Mutex # Locked Changed Cont. tot.Time[ms] avg.Time[ms] max.Time[ms] Flags\n",
m);
for (i = 0, m = 0; i < n && (show_n_max <= 0 || m < show_n_max); i++)
m += mutex_info_stat(table[i]) ? 1 : 0;
if (i < n)
fprintf(stderr,
" ... ... ... ... ... ... ... ||||||\n");
else
fprintf(stderr,
" ||||||\n");
fprintf(stderr,
" /|||||\n"
" Object: M = Mutex, W = RWLock /||||\n"
" State: x = dead, ! = inconsistent /|||\n"
" Use: R = used in realtime thread /||\n"
" Mutex Type: r = RECURSIVE, e = ERRRORCHECK, a = ADAPTIVE /|\n"
" Mutex Protocol: i = INHERIT, p = PROTECT /\n"
" RWLock Kind: r = PREFER_READER, w = PREFER_WRITER, W = PREFER_WRITER_NONREC \n");
if (!track_rt)
fprintf(stderr,
"\n"
"mutrace: Note that the flags column R is only valid in --track-rt mode!\n");
} else
fprintf(stderr,
"\n"
"mutrace: No mutex contended according to filtering parameters.\n");
free(table);
for (u = 0; u < hash_size; u++)
unlock_hash_mutex(u);
fprintf(stderr,
"\n"
"mutrace: Total runtime is %0.3f ms.\n", (double) t / 1000000.0);
n_cpus = sysconf(_SC_NPROCESSORS_ONLN);
assert(n_cpus >= 1);
if (n_cpus <= 1)
fprintf(stderr,
"\n"
"mutrace: WARNING: Results for uniprocessor machine. Results might be more interesting\n"
" when run on an SMP machine!\n");
else
fprintf(stderr,
"\n"
"mutrace: Results for SMP with %li processors.\n", n_cpus);
if (n_broken > 0)
fprintf(stderr,
"\n"
"mutrace: WARNING: %u inconsistent mutex uses detected. Results might not be reliable.\n"
"mutrace: Fix your program first!\n", n_broken);
if (n_collisions > 0)
fprintf(stderr,
"\n"
"mutrace: WARNING: %u internal hash collisions detected. Results might not be as reliable as they could be.\n"
"mutrace: Try to increase --hash-size=, which is currently at %u.\n", n_collisions, hash_size);
if (n_self_contended > 0)
fprintf(stderr,
"\n"
"mutrace: WARNING: %u internal mutex contention detected. Results might not be reliable as they could be.\n"
"mutrace: Try to increase --hash-size=, which is currently at %u.\n", n_self_contended, hash_size);
finish:
shown_summary = true;
real_pthread_mutex_unlock(&summary_mutex);
}
static void shutdown(void) {
show_summary();
}
void exit(int status) {
show_summary();
real_exit(status);
}
void _exit(int status) {
show_summary();
real_exit(status);
}
void _Exit(int status) {
show_summary();
real__Exit(status);
}
static bool is_realtime(void) {
int policy;
policy = sched_getscheduler(_gettid());
assert(policy >= 0);
policy &= ~SCHED_RESET_ON_FORK;
return
policy == SCHED_FIFO ||
policy == SCHED_RR;
}
static bool verify_frame(const char *s) {
/* Generated by glibc's native backtrace_symbols() on Fedora */
if (strstr(s, "/" SONAME "("))
return false;
/* Generated by glibc's native backtrace_symbols() on Debian */
if (strstr(s, "/" SONAME " ["))
return false;
/* Generated by backtrace-symbols.c */
if (strstr(s, __FILE__":"))
return false;
return true;
}
static int light_backtrace(void **buffer, int size) {
#if defined(__i386__) || defined(__x86_64__)
int osize = 0;
void *stackaddr;
size_t stacksize;
void *frame;
pthread_attr_t attr;
pthread_getattr_np(pthread_self(), &attr);
pthread_attr_getstack(&attr, &stackaddr, &stacksize);
pthread_attr_destroy(&attr);
#if defined(__i386__)
__asm__("mov %%ebp, %[frame]": [frame] "=r" (frame));
#elif defined(__x86_64__)
__asm__("mov %%rbp, %[frame]": [frame] "=r" (frame));
#endif
while (osize < size &&
frame >= stackaddr &&
frame < (void *)((char *)stackaddr + stacksize)) {
buffer[osize++] = *((void **)frame + 1);
frame = *(void **)frame;
}
return osize;
#else
return real_backtrace(buffer, size);
#endif
}
static struct stacktrace_info generate_stacktrace(void) {
struct stacktrace_info stacktrace;
stacktrace.frames = malloc(sizeof(void*) * frames_max);
assert(stacktrace.frames);
stacktrace.nb_frame = light_backtrace(stacktrace.frames, frames_max);
assert(stacktrace.nb_frame >= 0);
return stacktrace;
}
static char *stacktrace_to_string(struct stacktrace_info stacktrace) {
char **strings, *ret, *p;
int i;
size_t k;
bool b;
strings = real_backtrace_symbols(stacktrace.frames, stacktrace.nb_frame);
assert(strings);
k = 0;
for (i = 0; i < stacktrace.nb_frame; i++)
k += strlen(strings[i]) + 2;
ret = malloc(k + 1);
assert(ret);
b = false;
for (i = 0, p = ret; i < stacktrace.nb_frame; i++) {
if (!b && !verify_frame(strings[i]))
continue;
if (!b && i > 0) {
/* Skip all but the first stack frame of ours */
*(p++) = '\t';
strcpy(p, strings[i-1]);
p += strlen(strings[i-1]);
*(p++) = '\n';
}
b = true;
*(p++) = '\t';
strcpy(p, strings[i]);
p += strlen(strings[i]);
*(p++) = '\n';
}
*p = 0;
free(strings);
return ret;
}
static struct mutex_info *mutex_info_add(unsigned long u, pthread_mutex_t *mutex, int type, int protocol) {
struct mutex_info *mi;
/* Needs external locking */
if (alive_mutexes[u])
__sync_fetch_and_add(&n_collisions, 1);
mi = calloc(1, sizeof(struct mutex_info));
assert(mi);
mi->mutex = mutex;
mi->type = type;
mi->protocol = protocol;
mi->stacktrace = generate_stacktrace();
mi->next = alive_mutexes[u];
alive_mutexes[u] = mi;
return mi;
}
static void mutex_info_remove(unsigned u, pthread_mutex_t *mutex) {
struct mutex_info *mi, *p;
/* Needs external locking */
for (mi = alive_mutexes[u], p = NULL; mi; p = mi, mi = mi->next)
if (mi->mutex == mutex)
break;
if (!mi)
return;
if (p)
p->next = mi->next;
else
alive_mutexes[u] = mi->next;
mi->dead = true;
mi->next = dead_mutexes[u];
dead_mutexes[u] = mi;
}
static struct mutex_info *mutex_info_acquire(pthread_mutex_t *mutex) {
unsigned long u;
struct mutex_info *mi;
u = mutex_hash(mutex);
lock_hash_mutex(u);
for (mi = alive_mutexes[u]; mi; mi = mi->next)
if (mi->mutex == mutex)
return mi;
/* FIXME: We assume that static mutexes are NORMAL, which
* might not actually be correct */
return mutex_info_add(u, mutex, PTHREAD_MUTEX_NORMAL, PTHREAD_PRIO_NONE);
}
static void mutex_info_release(pthread_mutex_t *mutex) {
unsigned long u;
u = mutex_hash(mutex);
unlock_hash_mutex(u);
}
int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *mutexattr) {
int r;
unsigned long u;
if (UNLIKELY(!initialized && recursive)) {
static const pthread_mutex_t template = PTHREAD_MUTEX_INITIALIZER;
/* Now this is incredibly ugly. */
memcpy(mutex, &template, sizeof(pthread_mutex_t));
return 0;
}
load_functions();
r = real_pthread_mutex_init(mutex, mutexattr);
if (r != 0)
return r;
if (LIKELY(initialized && !recursive)) {
int type = PTHREAD_MUTEX_NORMAL;
int protocol = PTHREAD_PRIO_NONE;
recursive = true;
u = mutex_hash(mutex);
lock_hash_mutex(u);
mutex_info_remove(u, mutex);
if (mutexattr) {
int k;
k = pthread_mutexattr_gettype(mutexattr, &type);
assert(k == 0);
k = pthread_mutexattr_getprotocol(mutexattr, &protocol);
assert(k == 0);
}
mutex_info_add(u, mutex, type, protocol);
unlock_hash_mutex(u);
recursive = false;
}
return r;
}
int pthread_mutex_destroy(pthread_mutex_t *mutex) {
unsigned long u;
assert(initialized || !recursive);
load_functions();
if (LIKELY(initialized && !recursive)) {
recursive = true;
u = mutex_hash(mutex);
lock_hash_mutex(u);
mutex_info_remove(u, mutex);
unlock_hash_mutex(u);
recursive = false;
}
return real_pthread_mutex_destroy(mutex);
}
static void mutex_lock(pthread_mutex_t *mutex, bool busy) {
struct mutex_info *mi;
pid_t tid;
if (UNLIKELY(!initialized || recursive))
return;
recursive = true;
mi = mutex_info_acquire(mutex);
if (mi->n_lock_level > 0 && mi->type != PTHREAD_MUTEX_RECURSIVE) {
__sync_fetch_and_add(&n_broken, 1);
mi->broken = true;
if (raise_trap)
DEBUG_TRAP;
}
mi->n_lock_level++;
mi->n_locked++;
if (busy)
mi->n_contended++;
tid = _gettid();
if (mi->last_owner != tid) {
if (mi->last_owner != 0)
mi->n_owner_changed++;
mi->last_owner = tid;
}
if (track_rt && !mi->realtime && is_realtime())
mi->realtime = true;
mi->nsec_timestamp = nsec_now();
mutex_info_release(mutex);
recursive = false;
}
int pthread_mutex_lock(pthread_mutex_t *mutex) {
int r;
bool busy;
if (UNLIKELY(!initialized && recursive)) {
/* During the initialization phase we might be called
* inside of dlsym(). Since we'd enter an endless loop
* if we tried to resolved the real
* pthread_mutex_lock() here then we simply fake the
* lock which should be safe since no thread can be
* running yet. */
assert(!threads_existing);
return 0;
}
load_functions();
r = real_pthread_mutex_trylock(mutex);
if (UNLIKELY(r != EBUSY && r != 0))
return r;
if (UNLIKELY((busy = (r == EBUSY)))) {
r = real_pthread_mutex_lock(mutex);
if (UNLIKELY(r != 0))
return r;
}
mutex_lock(mutex, busy);
return r;
}
int pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec *abstime) {
int r;
bool busy;
if (UNLIKELY(!initialized && recursive)) {
assert(!threads_existing);
return 0;
}
load_functions();
r = real_pthread_mutex_trylock(mutex);
if (UNLIKELY(r != EBUSY && r != 0))
return r;
if (UNLIKELY((busy = (r == EBUSY)))) {
r = real_pthread_mutex_timedlock(mutex, abstime);
if (UNLIKELY(r == ETIMEDOUT))
busy = true;
else if (UNLIKELY(r != 0))
return r;
}
mutex_lock(mutex, busy);
return r;
}
int pthread_mutex_trylock(pthread_mutex_t *mutex) {
int r;
if (UNLIKELY(!initialized && recursive)) {
assert(!threads_existing);
return 0;
}
load_functions();
r = real_pthread_mutex_trylock(mutex);
if (UNLIKELY(r != 0))
return r;
mutex_lock(mutex, false);
return r;
}
static void mutex_unlock(pthread_mutex_t *mutex) {
struct mutex_info *mi;
uint64_t t;
if (UNLIKELY(!initialized || recursive))
return;
recursive = true;
mi = mutex_info_acquire(mutex);
if (mi->n_lock_level <= 0) {
__sync_fetch_and_add(&n_broken, 1);
mi->broken = true;
if (raise_trap)
DEBUG_TRAP;
}
mi->n_lock_level--;
t = nsec_now() - mi->nsec_timestamp;
mi->nsec_locked_total += t;
if (t > mi->nsec_locked_max)
mi->nsec_locked_max = t;
mutex_info_release(mutex);
recursive = false;
}
int pthread_mutex_unlock(pthread_mutex_t *mutex) {
if (UNLIKELY(!initialized && recursive)) {
assert(!threads_existing);
return 0;
}
load_functions();
mutex_unlock(mutex);
return real_pthread_mutex_unlock(mutex);
}
int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) {
int r;
assert(initialized || !recursive);
load_functions();
mutex_unlock(mutex);
r = real_pthread_cond_wait(cond, mutex);
/* Unfortunately we cannot distuingish mutex contention and
* the condition not being signalled here. */
mutex_lock(mutex, false);
return r;
}
int pthread_cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex, const struct timespec *abstime) {
int r;
assert(initialized || !recursive);
load_functions();
mutex_unlock(mutex);
r = real_pthread_cond_timedwait(cond, mutex, abstime);
mutex_lock(mutex, false);
return r;
}
int pthread_create(pthread_t *newthread,
const pthread_attr_t *attr,
void *(*start_routine) (void *),
void *arg) {
load_functions();
if (UNLIKELY(!threads_existing)) {
threads_existing = true;
setup();
}
return real_pthread_create(newthread, attr, start_routine, arg);
}
int backtrace(void **array, int size) {
int r;
load_functions();
/* backtrace() internally uses a mutex. To avoid an endless
* loop we need to disable ourselves so that we don't try to
* call backtrace() ourselves when looking at that lock. */
recursive = true;
r = real_backtrace(array, size);
recursive = false;
return r;
}
char **backtrace_symbols(void *const *array, int size) {
char **r;
load_functions();
recursive = true;
r = real_backtrace_symbols(array, size);
recursive = false;
return r;
}
void backtrace_symbols_fd(void *const *array, int size, int fd) {
load_functions();
recursive = true;
real_backtrace_symbols_fd(array, size, fd);
recursive = false;
}
static struct mutex_info *rwlock_info_add(unsigned long u, pthread_rwlock_t *rwlock, int kind) {
struct mutex_info *mi;
/* Needs external locking */
if (alive_mutexes[u])
__sync_fetch_and_add(&n_collisions, 1);
mi = calloc(1, sizeof(struct mutex_info));
assert(mi);
mi->rwlock = rwlock;
mi->kind = kind;
mi->stacktrace = generate_stacktrace();
mi->next = alive_mutexes[u];
alive_mutexes[u] = mi;
return mi;
}
static void rwlock_info_remove(unsigned u, pthread_rwlock_t *rwlock) {
struct mutex_info *mi, *p;
/* Needs external locking */
for (mi = alive_mutexes[u], p = NULL; mi; p = mi, mi = mi->next)
if (mi->rwlock == rwlock)
break;
if (!mi)
return;
if (p)
p->next = mi->next;
else
alive_mutexes[u] = mi->next;
mi->dead = true;
mi->next = dead_mutexes[u];
dead_mutexes[u] = mi;
}
static struct mutex_info *rwlock_info_acquire(pthread_rwlock_t *rwlock) {
unsigned long u;
struct mutex_info *mi;
u = rwlock_hash(rwlock);
lock_hash_mutex(u);
for (mi = alive_mutexes[u]; mi; mi = mi->next)
if (mi->rwlock == rwlock)
return mi;
/* FIXME: We assume that static mutexes are RWLOCK_DEFAULT,
* which might not actually be correct */
return rwlock_info_add(u, rwlock, PTHREAD_RWLOCK_DEFAULT_NP);
}
static void rwlock_info_release(pthread_rwlock_t *rwlock) {
unsigned long u;
u = rwlock_hash(rwlock);
unlock_hash_mutex(u);
}
int pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr) {
int r;
unsigned long u;
if (UNLIKELY(!initialized && recursive)) {
static const pthread_rwlock_t template = PTHREAD_RWLOCK_INITIALIZER;
/* Now this is incredibly ugly. */
memcpy(rwlock, &template, sizeof(pthread_rwlock_t));
return 0;
}
load_functions();
r = real_pthread_rwlock_init(rwlock, attr);
if (r != 0)
return r;
if (LIKELY(initialized && !recursive)) {
int kind = PTHREAD_RWLOCK_DEFAULT_NP;
recursive = true;
u = rwlock_hash(rwlock);
lock_hash_mutex(u);
rwlock_info_remove(u, rwlock);
if (attr) {
int k;
k = pthread_rwlockattr_getkind_np(attr, &kind);
assert(k == 0);
}
rwlock_info_add(u, rwlock, kind);
unlock_hash_mutex(u);
recursive = false;
}
return r;
}
int pthread_rwlock_destroy(pthread_rwlock_t *rwlock) {
unsigned long u;
assert(initialized || !recursive);
load_functions();
if (LIKELY(initialized && !recursive)) {
recursive = true;
u = rwlock_hash(rwlock);
lock_hash_mutex(u);
rwlock_info_remove(u, rwlock);
unlock_hash_mutex(u);
recursive = false;
}
return real_pthread_rwlock_destroy(rwlock);
}
static void rwlock_lock(pthread_rwlock_t *rwlock, bool for_write, bool busy) {
struct mutex_info *mi;
pid_t tid;
if (UNLIKELY(!initialized || recursive))
return;
recursive = true;
mi = rwlock_info_acquire(rwlock);
if (mi->n_lock_level > 0 && for_write) {
__sync_fetch_and_add(&n_broken, 1);
mi->broken = true;
if (raise_trap)
DEBUG_TRAP;
}
mi->n_lock_level++;
mi->n_locked++;
if (busy)
mi->n_contended++;
tid = _gettid();
if (mi->last_owner != tid) {
if (mi->last_owner != 0)
mi->n_owner_changed++;
mi->last_owner = tid;
}
if (track_rt && !mi->realtime && is_realtime())
mi->realtime = true;
mi->nsec_timestamp = nsec_now();
rwlock_info_release(rwlock);
recursive = false;
}
int pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) {
int r;
bool busy;
if (UNLIKELY(!initialized && recursive)) {
assert(!threads_existing);
return 0;
}
load_functions();
r = real_pthread_rwlock_tryrdlock(rwlock);
if (UNLIKELY(r != EBUSY && r != 0))
return r;
if (UNLIKELY((busy = (r == EBUSY)))) {
r = real_pthread_rwlock_rdlock(rwlock);
if (UNLIKELY(r == ETIMEDOUT))
busy = true;
else if (UNLIKELY(r != 0))
return r;
}
rwlock_lock(rwlock, false, busy);
return r;
}
int pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock) {
int r;
if (UNLIKELY(!initialized && recursive)) {
assert(!threads_existing);
return 0;
}
load_functions();
r = real_pthread_rwlock_tryrdlock(rwlock);
if (UNLIKELY(r != EBUSY && r != 0))
return r;
rwlock_lock(rwlock, false, false);
return r;
}
int pthread_rwlock_timedrdlock(pthread_rwlock_t *rwlock, const struct timespec *abstime) {
int r;
bool busy;
if (UNLIKELY(!initialized && recursive)) {
assert(!threads_existing);
return 0;
}
load_functions();
r = real_pthread_rwlock_tryrdlock(rwlock);
if (UNLIKELY(r != EBUSY && r != 0))
return r;
if (UNLIKELY((busy = (r == EBUSY)))) {
r = real_pthread_rwlock_timedrdlock(rwlock, abstime);
if (UNLIKELY(r == ETIMEDOUT))
busy = true;
else if (UNLIKELY(r != 0))
return r;
}
rwlock_lock(rwlock, false, busy);
return r;
}
int pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) {
int r;
bool busy;
if (UNLIKELY(!initialized && recursive)) {
assert(!threads_existing);
return 0;
}
load_functions();
r = real_pthread_rwlock_trywrlock(rwlock);
if (UNLIKELY(r != EBUSY && r != 0))
return r;
if (UNLIKELY((busy = (r == EBUSY)))) {
r = real_pthread_rwlock_wrlock(rwlock);
if (UNLIKELY(r == ETIMEDOUT))
busy = true;
else if (UNLIKELY(r != 0))
return r;
}
rwlock_lock(rwlock, true, busy);
return r;
}
int pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock) {
int r;
if (UNLIKELY(!initialized && recursive)) {
assert(!threads_existing);
return 0;
}
load_functions();
r = real_pthread_rwlock_trywrlock(rwlock);
if (UNLIKELY(r != EBUSY && r != 0))
return r;
rwlock_lock(rwlock, true, false);
return r;
}
int pthread_rwlock_timedwrlock(pthread_rwlock_t *rwlock, const struct timespec *abstime) {
int r;
bool busy;
if (UNLIKELY(!initialized && recursive)) {
assert(!threads_existing);
return 0;
}
load_functions();
r = real_pthread_rwlock_trywrlock(rwlock);
if (UNLIKELY(r != EBUSY && r != 0))
return r;
if (UNLIKELY((busy = (r == EBUSY)))) {
r = real_pthread_rwlock_timedwrlock(rwlock, abstime);
if (UNLIKELY(r == ETIMEDOUT))
busy = true;
else if (UNLIKELY(r != 0))
return r;
}
rwlock_lock(rwlock, true, busy);
return r;
}
static void rwlock_unlock(pthread_rwlock_t *rwlock) {
struct mutex_info *mi;
uint64_t t;
if (UNLIKELY(!initialized || recursive))
return;
recursive = true;
mi = rwlock_info_acquire(rwlock);
if (mi->n_lock_level <= 0) {
__sync_fetch_and_add(&n_broken, 1);
mi->broken = true;
if (raise_trap)
DEBUG_TRAP;
}
mi->n_lock_level--;
t = nsec_now() - mi->nsec_timestamp;
mi->nsec_locked_total += t;
if (t > mi->nsec_locked_max)
mi->nsec_locked_max = t;
rwlock_info_release(rwlock);
recursive = false;
}
int pthread_rwlock_unlock(pthread_rwlock_t *rwlock) {
if (UNLIKELY(!initialized && recursive)) {
assert(!threads_existing);
return 0;
}
load_functions();
rwlock_unlock(rwlock);
return real_pthread_rwlock_unlock(rwlock);
}