Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ build/
# `make tsan` builds here so that build/ -- which the gate reads -- is
# never clobbered. It is scratch, exactly as build/ is.
build-tsan/
# Any other BUILD= the Makefile is handed, which is how a sanitiser or a
# bisecting build is kept out of build/. build-asan/ arrived this way and was
# untracked but NOT ignored, which is one `git add -A` away from committing a
# few hundred object files.
build-*/
dist/
*.o
*.d
Expand Down
51 changes: 42 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -172,16 +172,43 @@ selftest: $(BUILD)/tests/harness/selftest_outcomes
# accesses, which survive -O2, and an instrumented build that tests different
# code from the one that ships answers a question nobody asked.
#
# Two suites are excluded, and must stay excluded: test_model and
# test_expected_counts_accounting link with -Wl,--wrap=malloc, while TSan
# replaces the allocator itself, so the wrapper and the sanitiser contend for
# one symbol. Both test allocation failure rather than parallelism, so nothing
# about threads is lost with them out.
LLVM_VER ?= 19
# FOUR suites are excluded, and must stay excluded. All four are about the
# allocator, which is the one thing TSan replaces wholesale:
#
# test_model and test_expected_counts_accounting link with
# -Wl,--wrap=malloc, so the wrapper and the sanitiser contend for one
# symbol. Both test allocation failure rather than parallelism.
#
# test_allocator_linkage and test_allocator_policy assert on glibc's mmap
# threshold and on resident memory. TSan does not honour mallopt and maps
# shadow memory several times the heap, so under instrumentation they
# measure the sanitiser: run anyway, they failed at "the allocator policy
# did not reach this binary" and "resident memory is many times the model's
# own size". A suite that cannot pass instrumented has to be excluded
# rather than left to fail, or the run grows two failures everyone learns
# to scroll past. verification/tests/test_tsan.py derives this set from the
# sources, so a new suite of either kind is caught rather than assumed.
#
# -mavx IS NOT AN OPTIMISATION. src/native/math/vtrig.c passes a 32-byte
# vector by value to a static helper, which clang reports as changing the ABI
# when AVX is off. With target_clones in play that was a hard ERROR -- the
# default clone cannot satisfy it, and __attribute__((always_inline)) does not
# help because the check is in the frontend, before inlining -- and it was the
# reason this flag went in. BEAM_TSAN now turns the clones off, which demotes
# it to a -Wpsabi warning, so the flag is no longer what makes the build
# compile. It stays for two smaller reasons: it keeps that call's vector ABI
# the same as the shipping build's, and it keeps two warnings per build out of
# a log where a new warning should stand out. AVX, not AVX2: the minimum the
# ABI asks for, and on every x86-64 part since 2011.
#
# The version defaults to the one verified here. Archer ships in libomp-19-dev
# as well; `make tsan LLVM_VER=19` uses it.
LLVM_VER ?= 21
LLVM_DIR ?= /usr/lib/llvm-$(LLVM_VER)
TSAN_CC ?= $(LLVM_DIR)/bin/clang
TSAN_ARCHER ?= $(LLVM_DIR)/lib/libarcher.so
TSAN_EXCLUDE := test_model test_expected_counts_accounting
TSAN_EXCLUDE := test_model test_expected_counts_accounting \
test_allocator_linkage test_allocator_policy
TSAN_SUITES ?= $(filter-out $(TSAN_EXCLUDE),\
$(patsubst tests/native/%.c,%,$(TEST_SRCS)))

Expand All @@ -197,7 +224,7 @@ tsan:
@test -x "$(TSAN_CC)" || { \
echo "make tsan: no clang at $(TSAN_CC)."; \
echo " Install it: sudo apt install clang-$(LLVM_VER) libomp-$(LLVM_VER)-dev"; \
echo " Or point at another one: make tsan LLVM_VER=21"; \
echo " Or point at another one: make tsan LLVM_VER=19"; \
echo " GCC is NOT a fallback -- see the comment above this target."; \
exit 1; }
@test -f "$(TSAN_ARCHER)" || { \
Expand All @@ -206,9 +233,15 @@ tsan:
echo " Without Archer every barrier reads as a race."; \
exit 1; }
@$(MAKE) --no-print-directory BUILD=build-tsan CC="$(TSAN_CC)" \
SANFLAGS="-fsanitize=thread -g -fno-omit-frame-pointer" \
SANFLAGS="-fsanitize=thread -g -fno-omit-frame-pointer -mavx" \
OMPFLAG=-fopenmp=libomp \
$(addprefix build-tsan/tests/native/,$(TSAN_SUITES))
@echo
@echo "Built $(words $(TSAN_SUITES)) instrumented suite(s) in build-tsan/."
@echo "Run them: tools/qa/run c --tsan"
@echo " (which loads Archer and sets the TSAN_OPTIONS the run needs;"
@echo " running the binaries bare reports hundreds of races that are"
@echo " not there -- see verification/tests/test_tsan.py)"

clean:
rm -rf $(BUILD)
Expand Down
33 changes: 33 additions & 0 deletions src/native/core/threads.c
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,39 @@ int beam_plan_enter_default(void) {

int beam_threads(void) {
#ifdef _OPENMP
/*
* Already inside an active team: whatever region this thread opens next
* is a NESTED one, and one thread is the only answer that cannot
* multiply. It comes first because the alternative is not that the plan
* gets multiplied -- it is that the plan is not consulted at all. The
* inner pragmas are encountered by the outer team's workers, and a worker
* holds no plan (it is thread-local, deliberately: see
* beam_plan_enter's neighbours and test_a_plan_does_not_reach_another_thread),
* so it fell through to threads_cap() -- omp_get_max_threads() with no
* configured cap. An 8-thread plan opened eight teams of sixteen.
* Measured on one entropy() call with the plan pinned at 8: 117 threads
* against 9 with nesting off.
*
* Nesting is off by default, so this changes nothing until a caller sets
* OMP_MAX_ACTIVE_LEVELS, which nothing here can prevent: pinning it would
* mean omp_set_max_active_levels, process-global state that two BEAM calls
* on two Python threads would fight over. The thread's own level is the
* one thing that is always true locally.
*
* ACTIVE level, not level. A team of one is an inactive region; it
* multiplies nothing, and treating it as nested would throw away the
* parallelism inside a single-length sweep for no gain.
* test_an_inactive_region_does_not_lose_the_plan pins that.
*
* Safe for the scratch sizing too, and in the direction that matters.
* A resolved count both sizes buffers and sets num_threads, so the
* dangerous disagreement is a buffer smaller than its team. This can only
* shrink the answer as the nesting deepens, and a region entered at this
* level or deeper is sized by a read at this level or deeper, never by a
* shallower one. math/spectrum.c's per-thread scratch, allocated inside a
* nested call, was being sized for omp_get_max_threads() and used by one.
*/
if (omp_get_active_level() > 0) return 1;
if (tl_plan > 0) return tl_plan;
return threads_cap();
#else
Expand Down
2 changes: 1 addition & 1 deletion src/native/dp/jet.c
Original file line number Diff line number Diff line change
Expand Up @@ -1042,7 +1042,7 @@ static inline void line_store_body(uint32_t T, const double *restrict are,
* agree to the bit. On AVX2 without AVX-512 the tile runs line_edge_body4,
* the same arithmetic per lane with fewer lanes in flight
* (`line_narrow_pick`). */
#if defined(__x86_64__) && defined(__ELF__) && defined(__GNUC__)
#if defined(__x86_64__) && defined(__ELF__) && defined(__GNUC__) && !BEAM_TSAN
#define BEAM_EDGE_CLONES __attribute__((target_clones("avx512f", "avx2", "default")))
#else
#define BEAM_EDGE_CLONES
Expand Down
34 changes: 34 additions & 0 deletions src/native/internal/beam/internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,40 @@
#include <omp.h>
#endif

/*
* BEAM_TSAN -- 1 when this translation unit is being compiled with
* ThreadSanitizer instrumentation, 0 otherwise. Zero in every shipped build,
* so nothing downstream of it changes a byte of what users get; `make tsan`
* is the only consumer.
*
* It exists because function multiversioning and TSan cannot coexist.
* `__attribute__((target_clones(...)))` makes the compiler emit an IFUNC
* resolver, and the dynamic loader calls that resolver during startup
* relocation -- before the sanitiser runtime has initialised. Clang
* instruments the resolver like any other function, so `__tsan_func_entry`
* runs against uninitialised state and the process dies inside ld.so, before
* main, having printed nothing. Nine of the 35 instrumented suites died that
* way: exit 139, empty log, no message to search for.
*
* So the three multiversioned functions compile single-versioned under
* instrumentation. The cost is that the instrumented build runs the baseline
* copy rather than the one the CPU would have picked, which is the right
* trade for a race detector: it changes how fast the arithmetic runs, not
* which threads touch which bytes. The copies are bit-identical by
* construction anyway -- each lane is its own sum with no fused
* multiply-add -- so nothing about a result depends on which one ran.
*/
#if defined(__SANITIZE_THREAD__)
#define BEAM_TSAN 1
#elif defined(__has_feature)
#if __has_feature(thread_sanitizer)
#define BEAM_TSAN 1
#endif
#endif
#ifndef BEAM_TSAN
#define BEAM_TSAN 0
#endif

/*
* The former contents of this header, split by responsibility. They are
* included here in their original order so that every translation unit still
Expand Down
12 changes: 9 additions & 3 deletions src/native/internal/beam/plan.h
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,15 @@ int beam_plan_enter_default(void);
void beam_plan_leave(int token);

/**
* Threads the next parallel region should use: the plan in force on this
* thread if there is one, otherwise the cap from `beam_model_set_threads`,
* otherwise the OpenMP runtime's own maximum. Always 1 without OpenMP.
* Threads the next parallel region should use: ONE if this thread is already
* inside an active team, since the region it opens next is a nested one and
* any other answer multiplies; otherwise the plan in force on this thread if
* there is one; otherwise the cap from `beam_model_set_threads`; otherwise the
* OpenMP runtime's own maximum. Always 1 without OpenMP.
*
* The nested case is first, and `threads.c` records what it is holding back:
* a worker holds no plan, so before this it fell through to the cap, and an
* 8-thread plan opened eight teams of sixteen under OMP_MAX_ACTIVE_LEVELS=2.
*
* Every `omp parallel` in the implementation carries `num_threads(...)`,
* naming either this directly (64 regions) or a count already resolved from
Expand Down
2 changes: 1 addition & 1 deletion src/native/math/spectrum.c
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ static inline void spec_move(double *dst, const double *src, int64_t k,
/* The CPU picks, at load time, a copy compiled for AVX-512, for AVX2 or for
* the baseline; each lane is its own sum, with no fused multiply-add
* (contraction is off under -std=c11), so the copies agree to the bit. */
#if defined(__x86_64__) && defined(__ELF__) && defined(__GNUC__)
#if defined(__x86_64__) && defined(__ELF__) && defined(__GNUC__) && !BEAM_TSAN
#define SPEC_CLONES __attribute__((target_clones("avx512f", "avx2", "default")))
#else
#define SPEC_CLONES
Expand Down
2 changes: 1 addition & 1 deletion src/native/math/vtrig.c
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ static inline void vt_sincos4(vt_v4d x, vt_v4d *s_out, vt_v4d *c_out) {
*c_out = (vt_v4d)(ci ^ (cneg & sign));
}

#if defined(__x86_64__) && defined(__ELF__) && defined(__GNUC__)
#if defined(__x86_64__) && defined(__ELF__) && defined(__GNUC__) && !BEAM_TSAN
#define VT_CLONES __attribute__((target_clones("avx2", "default")))
#else
#define VT_CLONES
Expand Down
114 changes: 112 additions & 2 deletions tests/native/test_plan.c
Original file line number Diff line number Diff line change
Expand Up @@ -630,8 +630,116 @@ static void test_a_plan_does_not_reach_another_thread(void) {
ASSERT_MSG(want != 7, "the plan must differ from the cap for this to test");
ASSERT_MSG(master_saw == want,
"the thread that entered the scope must see the plan");
ASSERT_MSG(worker_saw == 7,
"a worker must fall back to the cap, 7, not inherit the plan");
/*
* This asserted `worker_saw == 7` -- the cap -- until the nesting hazard
* below was measured. The property being pinned is unchanged and is in
* the name: the plan does not reach another thread, so 4 must not appear
* here. What changed is the fallback. A worker is by definition inside an
* active team, so the region it would open next is a nested one, and
* beam_threads() now answers 1 rather than the cap. The cap was how an
* 8-thread plan opened eight teams of sixteen.
*
* The cap of 7 is still set above, and still has to differ from the plan,
* because this test would otherwise pass on a day when they coincided.
*/
ASSERT_MSG(worker_saw != want,
"a worker must not inherit the plan");
ASSERT_MSG(worker_saw == 1,
"a worker is inside an active team, so its next region would "
"be nested: it must ask for one thread, not the cap");
PASS();
#else
BEAM_SKIP("no OpenMP in this build");
#endif
}

/* ── Nested regions ──────────────────────────────────────────── */

/*
* The hazard these two pin, measured before they were written.
*
* BEAM enters nested regions at depth 2 -- `entropy` is one, math/diversity.c
* line 306 parallel-for into expected_raw, which has regions of its own at
* :207 and :232. Nesting is off by default, so an inner region's request is
* ignored and its team is 1, and none of this shows. Nothing in the library
* pins that default: no omp_set_max_active_levels, no omp_set_nested, because
* both are process-global state and two BEAM calls on two Python threads
* would fight over them. So OMP_MAX_ACTIVE_LEVELS=2 in the caller's
* environment turns nesting on.
*
* What happened then was not that the plan multiplied. It was ignored. The
* inner pragma is encountered by the OUTER TEAM'S WORKERS, and
* test_a_plan_does_not_reach_another_thread above pins that a worker has no
* plan -- it fell back to the cap, which with no configured cap is
* omp_get_max_threads(), 16 on the box this was measured on. So an 8-thread
* plan opened eight teams of sixteen. Measured on one entropy() call with the
* plan pinned at 8: 117 threads in the process against 9 with nesting off,
* and the call went from under a millisecond to 240 ms. Phase 0 measured that
* class of oversubscription at 301x to 2275x worse than serial on a loaded
* cluster node.
*
* Hence: inside an ACTIVE region, the next region this thread opens is a
* nested one, and one thread is the only answer that cannot multiply. The
* check is on the active level and not the level, which the second test
* pins: a team of one is an inactive region, it multiplies nothing, and
* serialising the work inside it would cost parallelism for no gain.
*/

static void test_a_nested_region_is_planned_at_one_thread(void) {
#ifdef _OPENMP
if (omp_get_max_threads() < 2) BEAM_SKIP("needs at least two OpenMP threads");
BeamPlanIn in = base();
in.available = 16;
const int token = beam_plan_enter(&in);
const int outside = beam_threads();
int master_inside = -1, worker_inside = -1, team = -1;
#pragma omp parallel num_threads(2)
{
if (omp_get_thread_num() == 0) {
team = omp_get_num_threads();
master_inside = beam_threads();
} else {
worker_inside = beam_threads();
}
}
beam_plan_leave(token);
ASSERT_MSG(outside > 1,
"the fixture must plan more than one thread, or this test "
"cannot tell the plan from the nested answer");
ASSERT_MSG(team == 2,
"the runtime gave no team of two, so nothing was nested");
ASSERT_MSG(master_inside == 1,
"the thread holding the plan must still ask for one thread "
"inside an active region: its next region would be nested");
ASSERT_MSG(worker_inside == 1,
"a worker must ask for one thread, not the cap -- the cap is "
"how an 8-thread plan opened eight teams of sixteen");
PASS();
#else
BEAM_SKIP("no OpenMP in this build");
#endif
}

static void test_an_inactive_region_does_not_lose_the_plan(void) {
#ifdef _OPENMP
BeamPlanIn in = base();
in.available = 16;
const int want = beam_plan_threads(&in);
const int token = beam_plan_enter(&in);
int inside = -1, team = -1;
#pragma omp parallel num_threads(1)
{
team = omp_get_num_threads();
inside = beam_threads();
}
beam_plan_leave(token);
ASSERT_MSG(want > 1, "the fixture must plan more than one thread");
ASSERT_MSG(team == 1, "num_threads(1) must give a team of one");
ASSERT_MSG(inside == want,
"a team of one is an INACTIVE region: it multiplies nothing, "
"so the plan still applies. Testing omp_get_level() here "
"instead of omp_get_active_level() would read 1 and throw "
"away the parallelism inside a single-length sweep");
PASS();
#else
BEAM_SKIP("no OpenMP in this build");
Expand Down Expand Up @@ -857,6 +965,8 @@ int main(int argc, char **argv) {
RUN_TEST(test_effective_threads_reports_the_cap_not_the_plan);
RUN_TEST(test_budget_concurrency_follows_the_active_plan);
RUN_TEST(test_a_plan_does_not_reach_another_thread);
RUN_TEST(test_a_nested_region_is_planned_at_one_thread);
RUN_TEST(test_an_inactive_region_does_not_lose_the_plan);

RUN_TEST(test_the_default_scope_plans_from_the_pinned_availability);
RUN_TEST(test_the_default_scope_stops_at_the_ceiling);
Expand Down
Loading
Loading