Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@

# Build artifacts
build/
# `make tsan` builds here so that build/ -- which the gate reads -- is
# never clobbered. It is scratch, exactly as build/ is.
build-tsan/
dist/
*.o
*.d
Expand Down
123 changes: 107 additions & 16 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,34 @@
# wildcards. See build_support/native_sources.txt for why.

CC ?= gcc

# Where objects and binaries land. A variable, not a constant: an instrumented
# build must not overwrite the ordinary one. $(BUILD)/libbeam.a and the suites
# beside it are what verification/check.py reads, rebuilding them costs about
# an hour on a loaded box, and a ThreadSanitizer archive left at that path
# would be handed to the gate as though it were the shipping library. The
# `tsan` target sets this to build-tsan for that reason alone.
BUILD ?= build

# Appended to both the compile and the link, and empty for an ordinary build.
# A sanitiser needs the same flag in both places. It is a separate variable
# rather than an override of CFLAGS so that an instrumented build ADDS to the
# shipping flags instead of restating them -- restating is how -mpopcnt or
# -Isrc/native/internal goes missing from one build, and that failure arrives
# as a wrong answer rather than as a compile error.
SANFLAGS ?=

# gcc spells it -fopenmp; clang against libomp needs -fopenmp=libomp.
OMPFLAG ?= -fopenmp
CFLAGS = -O2 -Wall -Wextra -std=c11 -mpopcnt -Iinclude -Isrc/native/internal
CFLAGS += -fopenmp
LDLIBS = -lm -fopenmp
CFLAGS += $(OMPFLAG) $(SANFLAGS)
LDLIBS = -lm $(OMPFLAG) $(SANFLAGS)

# Strip comments and blank lines from the manifests.
read_manifest = $(shell sed -e 's/#.*//' -e '/^[[:space:]]*$$/d' $(1))

SRCS := $(call read_manifest,build_support/native_sources.txt)
OBJS := $(SRCS:%.c=build/%.o)
OBJS := $(SRCS:%.c=$(BUILD)/%.o)
DEPS := $(OBJS:.o=.d)

# Fail loudly if the manifest names a file that is not there, rather than
Expand All @@ -28,9 +47,9 @@ ifneq ($(MISSING),)
$(error source manifest lists missing file(s): $(MISSING))
endif

all: build/libbeam.a
all: $(BUILD)/libbeam.a

build/libbeam.a: $(OBJS)
$(BUILD)/libbeam.a: $(OBJS)
@mkdir -p $(@D)
# Deleted before it is created, because `ar rcs` ADDS to an existing archive
# rather than replacing it. A source removed from the manifest leaves its
Expand All @@ -44,13 +63,13 @@ build/libbeam.a: $(OBJS)

# The target directory is created per object, so nested source layouts need no
# hard-coded directory list here.
build/%.o: %.c
$(BUILD)/%.o: %.c
@mkdir -p $(@D)
$(CC) $(CFLAGS) -MMD -MP -c $< -o $@

# ── C unit tests ────────────────────────────────────────────────────────────
TEST_SRCS := $(wildcard tests/native/test_*.c)
TEST_BINS := $(TEST_SRCS:tests/native/%.c=build/tests/native/%)
TEST_BINS := $(TEST_SRCS:tests/native/%.c=$(BUILD)/tests/native/%)

# tools/qa/run reports every suite and continues past a failure. The loop
# this replaces was `for t in $(TEST_BINS); do ./$$t || exit 1; done`, which
Expand All @@ -59,7 +78,10 @@ TEST_BINS := $(TEST_SRCS:tests/native/%.c=build/tests/native/%)
# parallelism. The runner caps threads per suite and runs several suites at
# once instead.
test: $(TEST_BINS)
@tools/qa/run c
# --bin-dir, because BUILD is a variable now: without it `make test
# BUILD=elsewhere` linked into elsewhere/ and then ran build/, reporting
# a pass for binaries it had not just built.
@tools/qa/run c --bin-dir $(BUILD)/tests/native

# verification/check.py reads the C test inventory by asking each binary what
# it runs, rather than by grepping RUN_TEST out of these sources -- see
Expand All @@ -74,13 +96,13 @@ list-tests: $(TEST_BINS)
# target-specific and must stay that way: --wrap redirects EVERY malloc in the
# linked objects to __wrap_malloc, so applying it to a binary that does not
# define that symbol fails to link.
build/tests/native/test_model: LDEXTRA = -Wl,--wrap=malloc
$(BUILD)/tests/native/test_model: LDEXTRA = -Wl,--wrap=malloc

# The same mechanism used to MEASURE rather than to fail: this test sums what
# the allocator actually handed out during one analytic and checks the figure
# that analytic reports against it. Nothing else can catch an allocation that
# is simply never charged, because the account is otherwise the only witness.
build/tests/native/test_expected_counts_accounting: \
$(BUILD)/tests/native/test_expected_counts_accounting: \
LDEXTRA = -Wl,--wrap=malloc -Wl,--wrap=calloc -Wl,--wrap=realloc \
-Wl,--wrap=free

Expand All @@ -105,22 +127,91 @@ HARNESS_SRC := tests/harness/beam_test.c
TEST_HDRS := $(wildcard tests/native/*.h tests/harness/*.h \
tests/harness/oracles/*.h)

build/tests/native/%: tests/native/%.c $(HARNESS_SRC) $(TEST_HDRS) build/libbeam.a
$(BUILD)/tests/native/%: tests/native/%.c $(HARNESS_SRC) $(TEST_HDRS) $(BUILD)/libbeam.a
@mkdir -p $(@D)
$(CC) $(CFLAGS) -Itests/native -Itests/harness $< $(HARNESS_SRC) \
build/libbeam.a $(LDLIBS) $(LDEXTRA) -o $@
$(BUILD)/libbeam.a $(LDLIBS) $(LDEXTRA) -o $@

# The harness's own self-test. It fails on purpose, so it is deliberately not
# a dependency of `test`, and it lives outside tests/native so that neither
# TEST_SRCS nor the gate's inventory can see it.
build/tests/harness/%: tests/harness/%.c $(HARNESS_SRC) $(TEST_HDRS)
$(BUILD)/tests/harness/%: tests/harness/%.c $(HARNESS_SRC) $(TEST_HDRS)
@mkdir -p $(@D)
$(CC) $(CFLAGS) -Itests/harness $< $(HARNESS_SRC) $(LDLIBS) -o $@

selftest: build/tests/harness/selftest_outcomes
selftest: $(BUILD)/tests/harness/selftest_outcomes


# ── ThreadSanitizer ─────────────────────────────────────────────────────────
#
# `make tsan` builds the library and the suites instrumented, into build-tsan/,
# leaving build/ untouched -- build/libbeam.a is what verification/check.py
# reads, and an instrumented archive left at that path would be handed to the
# gate as though it were the shipping library.
#
# IT MUST BE CLANG, AND libomp, AND ARCHER. This is not a preference, and the
# GCC path was tried first and measured, so that nobody has to try it again:
#
# libgomp carries no sanitiser annotations (`nm -D libgomp.so.1` finds no
# __tsan_* and no Annotate* symbol). TSan therefore sees NO happens-before
# edge between any two OpenMP threads, so every implicit barrier is
# invisible and every region that writes a result someone later reads is
# reported as a race. Measured on test_distance alone, at 8 threads: 1186
# reports, of which 1184 were main-thread-against-worker and 2 were
# worker-against-worker -- and both of those two were ALSO barrier
# artifacts, between pool threads reused across different regions
# (advance._omp_fn.0 against advance._omp_fn.2). There is no filter that
# separates a real intra-region race from a barrier-separated one when the
# tool has no edges at all, which is why no such filter is shipped here.
#
# Archer is an OMPT tool that inserts exactly those edges. It is in Ubuntu's
# libomp-N-dev (/usr/lib/llvm-N/lib/libarcher.so), so this needs
# `apt install clang-N libomp-N-dev` and nothing built by hand.
#
# The optimisation level is deliberately NOT lowered. TSan instruments memory
# accesses, which survive -O2, and an instrumented build that tests different
# code from the one that ships answers a question nobody asked.
#
# Two suites are excluded, and must stay excluded: test_model and
# test_expected_counts_accounting link with -Wl,--wrap=malloc, while TSan
# replaces the allocator itself, so the wrapper and the sanitiser contend for
# one symbol. Both test allocation failure rather than parallelism, so nothing
# about threads is lost with them out.
LLVM_VER ?= 19
LLVM_DIR ?= /usr/lib/llvm-$(LLVM_VER)
TSAN_CC ?= $(LLVM_DIR)/bin/clang
TSAN_ARCHER ?= $(LLVM_DIR)/lib/libarcher.so
TSAN_EXCLUDE := test_model test_expected_counts_accounting
TSAN_SUITES ?= $(filter-out $(TSAN_EXCLUDE),\
$(patsubst tests/native/%.c,%,$(TEST_SRCS)))

# Builds every suite without running one, which `test` and `list-tests` both
# do. The tsan target needs it, and so does anyone bisecting a compile error.
tests-only: $(TEST_BINS)

# Refuses rather than producing a log that cannot be read. A GCC-built
# instrumented binary runs perfectly well and reports a thousand races that
# are not there; failing here is the difference between no answer and a wrong
# one.
tsan:
@test -x "$(TSAN_CC)" || { \
echo "make tsan: no clang at $(TSAN_CC)."; \
echo " Install it: sudo apt install clang-$(LLVM_VER) libomp-$(LLVM_VER)-dev"; \
echo " Or point at another one: make tsan LLVM_VER=21"; \
echo " GCC is NOT a fallback -- see the comment above this target."; \
exit 1; }
@test -f "$(TSAN_ARCHER)" || { \
echo "make tsan: clang is present but Archer is not ($(TSAN_ARCHER))."; \
echo " Install it: sudo apt install libomp-$(LLVM_VER)-dev"; \
echo " Without Archer every barrier reads as a race."; \
exit 1; }
@$(MAKE) --no-print-directory BUILD=build-tsan CC="$(TSAN_CC)" \
SANFLAGS="-fsanitize=thread -g -fno-omit-frame-pointer" \
OMPFLAG=-fopenmp=libomp \
$(addprefix build-tsan/tests/native/,$(TSAN_SUITES))

clean:
rm -rf build
rm -rf $(BUILD)

.PHONY: all clean test list-tests selftest
.PHONY: all clean test list-tests selftest tests-only tsan
-include $(DEPS)
17 changes: 15 additions & 2 deletions docs/src/content/docs/reference/runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,21 @@ models in parallel and combine them from one thread.

Start worker processes with **spawn**, for example
`multiprocessing.get_context("spawn").Pool(...)`, and call `BEAM.set_threads(T)` in each worker
so that workers × `T` fits the machine. Models are not pickled: pass each worker the model's
file path and load it there with `BeamModel.from_file`.
so that workers × `T` fits the machine.

Spawn rather than fork, because an OpenMP thread pool does not survive `fork`: only the thread
that called it exists in the child, so the child waits at its first parallel computation instead
of raising. BEAM warns in the child when it is forked from a process that had worker threads
running, which is the signal to switch start methods — but spawn avoids it altogether.

The check is whether *any* native thread was running at the fork, not whether BEAM itself had
started one, so another library's threads can trigger it even when BEAM has not run. That is
deliberate — forking a multithreaded process is the hazard whoever started the threads — and the
warning is a `RuntimeWarning`, so it can be filtered by category. A fork from a single-threaded
process is not warned about.

Models are not pickled: pass each worker the model's file path and load it there with
`BeamModel.from_file`.

```python
import multiprocessing as mp
Expand Down
99 changes: 98 additions & 1 deletion src/BEAM/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
"""
from __future__ import annotations


import os
import warnings

from . import _cbeam as _c
from ._simulation_result import SimulationResult
Expand Down Expand Up @@ -123,3 +124,99 @@ def file_provenance_span() -> tuple[int, int]:
"""
off, nbytes = _c.file_provenance_span()
return int(off), int(nbytes)


# ── Forking ────────────────────────────────────────────────────────────────
#
# BEAM's OpenMP runtime does not survive `fork`. Once any parallel region has
# run, the runtime holds a pool of worker threads, and `fork` copies only the
# thread that called it: the child's first parallel computation then waits on
# workers that do not exist there and hangs -- no exception, no traceback, no
# exit. That is the worst failure mode a library has, and the documentation
# that says to use `spawn` gets read after the hang rather than before it.
#
# So the child says so itself, on the way past. The condition is "this process
# had more than one native thread when it forked", read from /proc rather than
# from `threading.active_count()`, which cannot see an OpenMP worker because
# it is not a Python thread.
#
# That condition is deliberately broader than "BEAM had a pool": another
# library's threads satisfy it too, and this will occasionally warn a child
# that would have been fine. Forking a multithreaded process is the hazard
# whoever started the threads, the warning is suppressible by category, and a
# guard that stays quiet through the case it exists for is worth less than one
# that is sometimes early.
_FORK_THREADS = 1

_FORK_WARNING = (
"this process forked while {n} native threads were running. An OpenMP "
"thread pool does not survive fork, so if this child uses BEAM its first "
"parallel computation may hang, with no error raised. Start workers with "
'multiprocessing.get_context("spawn") and call BEAM.set_threads(T) in '
"each one."
)

#: The same thing, pre-encoded, for the one path that cannot use `warnings`.
#: Built once at import: the fallback runs in a child forked from a threaded
#: process, where anything that allocates or takes a lock may not come back.
_FORK_WARNING_BYTES = (
b"RuntimeWarning: BEAM: this process forked while native threads were "
b"running. An OpenMP thread pool does not survive fork, so if this child "
b"uses BEAM its first parallel computation may hang. Start workers with "
b'multiprocessing.get_context("spawn").\n'
)


def _native_thread_count() -> int:
"""Threads in this process, including the ones Python cannot see.

1 when the count cannot be read, which makes the fork warning silent on a
platform without /proc rather than wrong about it.
"""
try:
with open("/proc/self/status", encoding="utf-8") as fh:
for line in fh:
if line.startswith("Threads:"):
return int(line.split()[1])
except (OSError, ValueError, IndexError):
pass
return 1


def _before_fork() -> None:
# Read in the parent: by the time the child runs it has one thread, so the
# count that matters is gone.
global _FORK_THREADS
_FORK_THREADS = _native_thread_count()


def _after_fork_in_child() -> None:
if _FORK_THREADS <= 1:
return
# Wrapped, because a child that cannot warn must still run: the point is
# to explain a hang, never to become a different failure.
#
# The except branch is NOT a silent pass, and that was a real defect once.
# Under `-W error` -- or a pytest `filterwarnings = error` -- `warn` RAISES
# the warning, so a bare `except: pass` turned the guard off in exactly
# the configuration that asked to be told about warnings most loudly. It
# was measured: the child hung and printed nothing. The fallback is a
# single `write` of a constant built at import, which is the most that can
# safely be done in a child forked from a threaded process.
#
# It degrades to a stderr line rather than an exception on purpose: an
# exception raised from an at-fork handler has nowhere to go.
try:
warnings.warn(_FORK_WARNING.format(n=_FORK_THREADS),
RuntimeWarning, stacklevel=2)
except Exception:
try:
os.write(2, _FORK_WARNING_BYTES)
except OSError: # pragma: no cover
pass


# Unix only: Windows has no fork, and no register_at_fork to hang it on.
if hasattr(os, "register_at_fork"):
os.register_at_fork(before=_before_fork,
after_in_child=_after_fork_in_child)
8 changes: 4 additions & 4 deletions src/native/bindings/python/algorithm_bindings.c
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ PyObject *py_sample(PyObject *self, PyObject *args) {
BeamRng rng;
beam_rng_seed(&rng, (uint64_t)seed);
BeamError err;
Py_BEGIN_ALLOW_THREADS
BEAM_PY_PLANNED_BEGIN
err = beam_model_sample(m, n, length, &rng, buf, NULL);
Py_END_ALLOW_THREADS
BEAM_PY_PLANNED_END
if (err != BEAM_OK) { PyMem_Free(buf); return set_beam_error(err); }

PyObject *list = PyList_New(n);
Expand Down Expand Up @@ -89,9 +89,9 @@ PyObject *py_top_k(PyObject *self, PyObject *args) {
+ (uint64_t)k * sizeof(double)); }
uint32_t n = 0;
BeamError err;
Py_BEGIN_ALLOW_THREADS
BEAM_PY_PLANNED_BEGIN
err = beam_model_top_k(m, k, length, buf, lp, &n);
Py_END_ALLOW_THREADS
BEAM_PY_PLANNED_END
if (err != BEAM_OK) { PyMem_Free(buf); PyMem_Free(lp); return set_beam_error(err); }

PyObject *seqs = PyList_New(n), *probs = PyList_New(n);
Expand Down
4 changes: 2 additions & 2 deletions src/native/bindings/python/distance_bindings.c
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ PyObject *py_local_distance(PyObject *self, PyObject *args) {
if (!p || !q) return NULL;
BeamLocalDistanceResult *r=NULL;
BeamError err;
Py_BEGIN_ALLOW_THREADS
BEAM_PY_PLANNED_BEGIN
err=beam_local_distance(p,q,&config,&r);
Py_END_ALLOW_THREADS
BEAM_PY_PLANNED_END
if (err!=BEAM_OK) return set_beam_error(err);
/* Native scratch has been freed. Reserve retained native output plus the
* raw lists/tuples/scalars AND the Python facade's result objects, attribute
Expand Down
4 changes: 2 additions & 2 deletions src/native/bindings/python/fingerprint_bindings.c
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ PyObject *py_fingerprint_search(PyObject *self, PyObject *args) {
if (!p || !q) return NULL;
BeamFingerprintResult *r = NULL;
BeamError err;
Py_BEGIN_ALLOW_THREADS
BEAM_PY_PLANNED_BEGIN
err = beam_fingerprint_search(p,q,L,&cfg,&r);
Py_END_ALLOW_THREADS
BEAM_PY_PLANNED_END
if (err != BEAM_OK) return set_beam_error(err);
PyObject *indices = PyBytes_FromStringAndSize(NULL,(Py_ssize_t)r->n_nodes*6*sizeof(int32_t));
PyObject *masses = PyBytes_FromStringAndSize(NULL,(Py_ssize_t)r->n_nodes*2*sizeof(double));
Expand Down
4 changes: 2 additions & 2 deletions src/native/bindings/python/insights_bindings.c
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,9 @@ PyObject *py_insights_logos(PyObject *self, PyObject *args) {

BeamLogoSet set;
BeamError err;
Py_BEGIN_ALLOW_THREADS
BEAM_PY_PLANNED_BEGIN
err = beam_insights_logos(m, &req, &set);
Py_END_ALLOW_THREADS
BEAM_PY_PLANNED_END
if (err != BEAM_OK) return set_beam_error(err);

PyObject *logos = PyList_New((Py_ssize_t)set.n_logos);
Expand Down
Loading
Loading