Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ build/tellc-macos-arm64/tellc/tellc TellC/examples/factorial.tell -o factorial
./factorial
```

The example prints `120`. The standalone compiler embeds LLVM code generation and the Mach-O linker; it and its output target macOS 11 or later. A Linux build uses a compatible LLVM installation, `llc`, and Clang. Build details, dependencies, and tests are in the [TellC guide](TellC/README.md), [build guide](TellC/docs/BUILDING.md), and [test guide](TellC/docs/TESTING.md). `tellc source.tell` defaults to `a.out`; use `-o` to choose the output and `tellc --llvm-version` to inspect its LLVM version.
The example prints `120`. The standalone compiler embeds LLVM code generation and the Mach-O linker; it and its output target macOS 11 or later. The [standalone Linux build](TellC/docs/BUILDING.md#standalone-linux) produces static x86-64 and ARM64 compilers that can each emit executables for either architecture. Build details, dependencies, and tests are in the [TellC guide](TellC/README.md), [build guide](TellC/docs/BUILDING.md), and [test guide](TellC/docs/TESTING.md). `tellc source.tell` defaults to `a.out`; use `-o` to choose the output and `tellc --llvm-version` to inspect its LLVM version.

## Legacy Python runtime (deprecated)

Expand Down
37 changes: 34 additions & 3 deletions TellC/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,20 @@ endif()
option(TELL_STANDALONE_MACOS "Embed LLVM code generation and LLD for Apple silicon macOS" ${_tell_standalone_default})
unset(_tell_standalone_default)

if(TELL_STANDALONE_MACOS)
option(TELL_STANDALONE_LINUX "Embed LLVM, LLD and static runtimes for Linux" OFF)
if(TELL_STANDALONE_LINUX)
if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux" OR
NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|aarch64)$" OR
TELL_STANDALONE_MACOS)
message(FATAL_ERROR "Standalone Linux TellC requires an x86_64 or aarch64 Linux target")
endif()
if(NOT TELL_ZIG OR NOT TELL_ZIG_CACHE_DIR)
message(FATAL_ERROR "Use scripts/build-tellc-linux.sh to configure the Zig cross-toolchain")
endif()
find_package(LLD REQUIRED CONFIG)
find_package(Python3 REQUIRED COMPONENTS Interpreter)
set(TELL_LLVM_LIBS lldELF)
elseif(TELL_STANDALONE_MACOS)
if(NOT APPLE)
message(FATAL_ERROR "TELL_STANDALONE_MACOS requires macOS")
endif()
Expand Down Expand Up @@ -50,8 +63,26 @@ target_compile_features(tellc PRIVATE cxx_std_17)
target_include_directories(tellc SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
target_compile_definitions(tellc PRIVATE ${LLVM_DEFINITIONS} TELL_LLC_PATH="${TELL_LLC_PATH}")

if(TELL_STANDALONE_MACOS)
target_compile_definitions(tellc PRIVATE TELL_STANDALONE_MACOS)
if(TELL_STANDALONE_LINUX)
enable_language(ASM)
set(runtime_dir "${CMAKE_CURRENT_BINARY_DIR}/linux-runtime")
add_custom_command(
OUTPUT "${runtime_dir}/runtime.S" "${runtime_dir}/linux_runtime.hpp"
COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/embed-linux-runtime.py"
"${TELL_ZIG}" "${TELL_ZIG_CACHE_DIR}" "${runtime_dir}"
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/cmake/embed-linux-runtime.py" "${TELL_ZIG}"
VERBATIM
)
target_sources(tellc PRIVATE src/linux_linker.cpp
"${runtime_dir}/runtime.S" "${runtime_dir}/linux_runtime.hpp")
target_compile_definitions(tellc PRIVATE TELL_STANDALONE_LINUX TELL_EMBEDDED_CODEGEN
TELL_LINUX_ARCH="${CMAKE_SYSTEM_PROCESSOR}")
target_include_directories(tellc SYSTEM PRIVATE ${LLD_INCLUDE_DIRS})
target_include_directories(tellc PRIVATE "${runtime_dir}")
target_link_options(tellc PRIVATE -s)
target_link_libraries(tellc PRIVATE tell_parser ${TELL_LLVM_LIBS})
elseif(TELL_STANDALONE_MACOS)
target_compile_definitions(tellc PRIVATE TELL_STANDALONE_MACOS TELL_EMBEDDED_CODEGEN)
target_include_directories(tellc SYSTEM PRIVATE "${LLD_SOURCE_DIR}/include")

# Keep an escape hatch for custom LLVM builds that still expose shared zstd.
Expand Down
6 changes: 5 additions & 1 deletion TellC/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,11 @@ be needed to avoid Gatekeeper warnings. A build made with newer LLVM libraries
or a newer deployment target may not run on macOS 11; use the script above to
build the dependencies and compiler together.

On Linux, build with CMake and a compatible LLVM installation. Set `LLVM_DIR` to
The [standalone Linux build](docs/BUILDING.md#standalone-linux) embeds LLVM,
LLD, and the static runtimes for x86-64 and ARM64. Each compiler can generate
Linux executables for both architectures.

To use an installed LLVM and Clang on Linux, build with CMake. Set `LLVM_DIR` to
the directory containing `LLVMConfig.cmake`. TellC uses `llc` from that LLVM
build and Clang on `PATH` to link. For cross-built compilers, set
`-DTELL_LLC_PATH=llc-21` (or the matching tool) to find it on the target's
Expand Down
51 changes: 51 additions & 0 deletions TellC/cmake/embed-linux-runtime.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
"""Embed the startup object and static libraries from Zig's Linux link."""
import os
from pathlib import Path
import re
import subprocess
import sys

zig, cache, directory = sys.argv[1:]
directory = Path(directory)
directory.mkdir(parents=True, exist_ok=True)
env = dict(os.environ, ZIG_VERBOSE_LINK="1", ZIG_GLOBAL_CACHE_DIR=cache,
ZIG_LOCAL_CACHE_DIR=str(directory / "cache"))
probe = directory / "probe.cpp"
probe.write_text("int main() { return 0; }\n")
cpp_libraries = ("libc++abi.a", "libc++.a", "libunwind.a")
runtime_files = ("crt1.o", *cpp_libraries, "libc.a", "libzigc.a", "libcompiler_rt.a")
assembly = ['.section .rodata']
header = ['#pragma once']

for arch in ("x86_64", "aarch64"):
link = subprocess.run(
[zig, "c++", "-target", f"{arch}-linux-musl", "-mcpu=baseline",
"-O2", "-static", "-no-pie", str(probe), "-o", str(directory / f"probe-{arch}")],
env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
)
if link.returncode:
sys.exit(link.stdout)
trace = next((line for line in reversed(link.stdout.splitlines())
if line.startswith("ld.lld ") and " --entry _start " in line), "")
entries = []
for name in runtime_files:
pattern = r"/o/([0-9a-f]+)/" + re.escape(name) + r'(?=[\s"]|$)'
match = re.search(pattern, trace)
if not match:
sys.exit(f"Unable to find {arch} {name} in Zig's link trace:\n{trace}")
source = Path(cache) / "o" / match[1] / name
file_id = name.replace("+", "x").replace(".", "_")
symbol = f"tell_{arch}_{file_id}"
path = str(source).replace("\\", "\\\\").replace('"', '\\"')
assembly += ['.balign 16', f'.globl {symbol}', f'.hidden {symbol}',
f'{symbol}:', f'.incbin "{path}"']
header.append(f'extern "C" const char {symbol}[{source.stat().st_size}];')
optional = "true" if name in cpp_libraries else "false"
entries.append(f' {{"{name}", {symbol}, sizeof({symbol}), {optional}}},')
header.append(f'constexpr EmbeddedFile {arch}_runtime[] = {{')
header.extend(entries)
header.append('};')

assembly.append('.section .note.GNU-stack,""')
(directory / "runtime.S").write_text("\n".join(assembly) + "\n")
(directory / "linux_runtime.hpp").write_text("\n".join(header) + "\n")
14 changes: 14 additions & 0 deletions TellC/cmake/linux-zig.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR "${TELL_LINUX_ARCH}")
set(CMAKE_TRY_COMPILE_PLATFORM_VARIABLES TELL_ZIG TELL_LINUX_ARCH)
set(zig_target "${TELL_LINUX_ARCH}-linux-musl")
set(CMAKE_C_COMPILER "${TELL_ZIG};cc;-target;${zig_target};-mcpu=baseline")
set(CMAKE_CXX_COMPILER "${TELL_ZIG};c++;-target;${zig_target};-mcpu=baseline")
set(CMAKE_ASM_COMPILER "${CMAKE_C_COMPILER}")
set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -no-pie")
set(CMAKE_SKIP_RPATH TRUE)
# Zig 0.16.0 crashes on CMake's linker depfile flag for aarch64.
set(CMAKE_LINK_DEPENDS_USE_LINKER FALSE)
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
7 changes: 5 additions & 2 deletions TellC/docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,15 @@ the parser is not automatically guaranteed to have backend support.
4. The selected platform path turns that module into a native executable:
- Apple silicon standalone mode emits an AArch64 Mach-O object in memory
and invokes the embedded LLD Mach-O driver with in-memory inputs.
- Linux and Intel macOS write temporary LLVM IR and object files, invoke
- Standalone Linux emits x86-64 or AArch64 ELF object code and uses the
embedded LLD ELF driver. It passes the object and embedded static runtime
inputs via anonymous memory-backed files.
- External-toolchain builds write temporary LLVM IR and object files, invoke
the matching llc, and use Clang for the final link.

## Platform boundaries

The standalone linker is currently limited to Apple silicon and macOS 11+.
The standalone macOS linker targets Apple silicon and macOS 11+.
Its libSystem text stub is deliberately minimal and mirrors the symbols the
current generated code imports. New runtime calls must update that stub and
the no-toolchain integration test together.
Expand Down
52 changes: 48 additions & 4 deletions TellC/docs/BUILDING.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# Building TellC

TellC has two build modes. The Apple silicon standalone build embeds LLVM code
generation and LLD. Linux and Intel macOS use an installed LLVM toolchain plus
Clang. Standalone mode is enabled by default only on Apple silicon.
The standalone macOS and Linux builds embed LLVM code generation and LLD.
The external-toolchain build uses an installed LLVM and Clang. Standalone mode
is enabled by default only on Apple silicon macOS.

## Requirements

Expand All @@ -24,7 +24,7 @@ revision. The build script disables optional compression and documentation
dependencies to keep the build smaller and avoid runtime shared-library
dependencies.

## Linux and Intel macOS
## External toolchain on Linux and macOS

Install LLVM and Clang for your platform, then configure and build:

Expand All @@ -40,6 +40,50 @@ On Intel macOS, TELL_STANDALONE_MACOS defaults to OFF and the compiler uses the
external-toolchain path. Set -DTELL_STANDALONE_MACOS=OFF explicitly when
configuring an Intel Mac if the option was previously cached as ON.

## Standalone Linux

Build from macOS or Linux with [Zig 0.16.0](https://ziglang.org/download/),
CMake 3.31 or later, Ninja, Python 3, a native C++ compiler, and an LLVM 24
source tree. The builds here used LLVM revision
`7090701baa47d03b1e9877eff3b4e8f032b99946`. The script builds TableGen for
the build host and cross-compiles LLVM, LLD, and TellC directly for Linux.

```sh
LLVM_SOURCE=/path/to/llvm-project ./scripts/build-tellc-linux.sh
```

This writes `build/tellc-linux/tellc-linux-x86_64` and
`build/tellc-linux/tellc-linux-aarch64`. Pass `x86_64` or `aarch64` to build
one binary. Set `ZIG` to the Zig executable, `TELL_BUILD_ROOT` to change the
output directory, or `JOBS` to control parallelism. If you already have
`llvm-tblgen` and `llvm-min-tblgen` built on the host from the same LLVM
revision, set `LLVM_NATIVE_TOOL_DIR` to their directory.

Copy the compiler for the machine's architecture to Linux and run it directly.
Both compilers embed LLVM, LLD, and static musl and C++ runtime inputs for both
targets. The compiler defaults to its own architecture. Use `--target` to
select the other architecture:

```sh
./tellc-linux-x86_64 TellC/examples/factorial.tell -o factorial
./tellc-linux-x86_64 TellC/examples/factorial.tell --target aarch64 -o factorial-arm64
```

The compiler and its outputs are static ELF executables. Compiling uses
anonymous files through `memfd_create` (available since Linux 3.17) and
`/proc/self/fd`, which requires procfs mounted. Compilation requires no installed
LLVM, Clang, system libraries, SDK, or writable temporary directory.
With `--link`, supply ELF objects or static archives built for the selected
architecture and musl ABI. C++ archives should use Zig's libc++ ABI.
When redistributing the compiler, include Tell's license, LLVM's
`LICENSE.TXT`, and the license notices for the Zig and musl runtimes.

Run the standalone smoke test on matching Linux hardware:

```sh
python3 TellC/tests/standalone_linux.py ./tellc-linux-x86_64 x86_64
```

## Apple silicon standalone compiler

From the repository root, point LLVM_SOURCE to a matching llvm-project
Expand Down
23 changes: 15 additions & 8 deletions TellC/src/codegen.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#include <llvm/IR/Module.h>
#include <llvm/IR/Verifier.h>
#include <llvm/Support/raw_ostream.h>
#ifdef TELL_STANDALONE_MACOS
#ifdef TELL_EMBEDDED_CODEGEN
#include <llvm/MC/TargetRegistry.h>
#include <llvm/Target/TargetMachine.h>
#include <llvm/TargetParser/Host.h>
Expand Down Expand Up @@ -883,8 +883,8 @@ class Generator {
}

void generate(const Program& program, const std::string& path, bool check_only = false
#ifdef TELL_STANDALONE_MACOS
, llvm::SmallVectorImpl<char>* object = nullptr
#ifdef TELL_EMBEDDED_CODEGEN
, llvm::SmallVectorImpl<char>* object = nullptr, const std::string& target_triple = {}
#endif
) {
declare(program);
Expand All @@ -906,11 +906,18 @@ class Generator {
ir.CreateRet(ir.getInt32(0));
if(llvm::verifyModule(module,&llvm::errs())) throw std::runtime_error("generated invalid LLVM IR");
if (check_only) return;
#ifdef TELL_STANDALONE_MACOS
#ifdef TELL_EMBEDDED_CODEGEN
if (object) {
#ifdef TELL_STANDALONE_LINUX
llvm::InitializeAllTargetInfos();
llvm::InitializeAllTargets();
llvm::InitializeAllTargetMCs();
llvm::InitializeAllAsmPrinters();
#else
llvm::InitializeNativeTarget();
llvm::InitializeNativeTargetAsmPrinter();
llvm::Triple triple("arm64-apple-macosx11.0.0");
#endif
llvm::Triple triple(target_triple);
std::string error;
const auto* target = llvm::TargetRegistry::lookupTarget(triple, error);
if (!target) throw std::runtime_error("cannot select LLVM target: " + error);
Expand All @@ -933,9 +940,9 @@ class Generator {
};
} // namespace

#ifdef TELL_STANDALONE_MACOS
void generate_object(const Program& program, llvm::SmallVectorImpl<char>& object) {
Generator{}.generate(program, "", false, &object);
#ifdef TELL_EMBEDDED_CODEGEN
void generate_object(const Program& program, llvm::SmallVectorImpl<char>& object, const std::string& target_triple) {
Generator{}.generate(program, "", false, &object, target_triple);
}
#endif
void check_program(const Program& program) {
Expand Down
6 changes: 3 additions & 3 deletions TellC/src/codegen.hpp
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
#pragma once
#include "tell/parser.hpp"
#ifdef TELL_STANDALONE_MACOS
#ifdef TELL_EMBEDDED_CODEGEN
#include <llvm/ADT/SmallVector.h>
#endif

namespace tell {
#ifdef TELL_STANDALONE_MACOS
void generate_object(const Program& program, llvm::SmallVectorImpl<char>& object);
#ifdef TELL_EMBEDDED_CODEGEN
void generate_object(const Program& program, llvm::SmallVectorImpl<char>& object, const std::string& target_triple);
#endif
void check_program(const Program& program);
void generate_ir(const Program& program, const std::string& path);
Expand Down
85 changes: 85 additions & 0 deletions TellC/src/linux_linker.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
#include "linux_linker.hpp"

#include <lld/Common/Driver.h>
#include <llvm/Support/raw_ostream.h>
#include <cerrno>
#include <filesystem>
#include <iterator>
#include <memory>
#include <stdexcept>
#include <system_error>
#include <sys/mman.h>
#include <unistd.h>

LLD_HAS_DRIVER(elf)

namespace {
struct EmbeddedFile {
const char* name;
const char* bytes;
size_t size;
bool for_foreign_libraries;
};
#include "linux_runtime.hpp"

// ELF LLD takes paths. Anonymous files let it read the embedded inputs
// without extracting a toolchain to disk or depending on TMPDIR.
class LinkerInput {
int fd;
public:
LinkerInput(const char* name, llvm::StringRef data)
: fd(memfd_create(name, MFD_CLOEXEC)) {
if (fd < 0)
throw std::system_error(errno, std::generic_category(), "creating linker input");
while (!data.empty()) {
auto count = write(fd, data.data(), data.size());
if (count < 0 && errno == EINTR) continue;
if (count <= 0) {
int error = count < 0 ? errno : EIO;
close(fd);
throw std::system_error(error, std::generic_category(), "writing linker input");
}
data = data.drop_front(count);
}
}

~LinkerInput() { close(fd); }
LinkerInput(const LinkerInput&) = delete;
LinkerInput& operator=(const LinkerInput&) = delete;

std::string path() const { return "/proc/self/fd/" + std::to_string(fd); }
};
}

void tell::link_linux(llvm::StringRef object, const std::string& target,
const std::string& output, const std::vector<std::string>& libraries) {
bool x86 = target == "x86_64";
const auto& runtime = x86 ? x86_64_runtime : aarch64_runtime;
std::vector<std::string> args = {
"ld.lld", "-m", x86 ? "elf_x86_64" : "aarch64linux",
"-static", "--gc-sections", "--eh-frame-hdr", "--strip-debug", "-o", output
};
LinkerInput generated("tell.o", object);
std::vector<std::unique_ptr<LinkerInput>> inputs;
auto add_runtime_file = [&](const EmbeddedFile& file) {
inputs.push_back(std::make_unique<LinkerInput>(file.name, llvm::StringRef(file.bytes, file.size)));
args.push_back(inputs.back()->path());
};

add_runtime_file(runtime[0]);
args.push_back(generated.path());
for (const auto& library : libraries)
args.push_back(std::filesystem::absolute(library).string());
// LLD resolves archive references across its inputs and rescans as needed.
for (size_t i = 1; i < std::size(runtime); ++i) {
const auto& file = runtime[i];
if (libraries.empty() && file.for_foreign_libraries) continue;
add_runtime_file(file);
}

std::vector<const char*> argv;
for (const auto& arg : args) argv.push_back(arg.c_str());
const lld::DriverDef drivers[] = {{lld::Gnu, &lld::elf::link}};
if (lld::lldMain(argv, llvm::outs(), llvm::errs(), drivers).retCode)
throw std::runtime_error("linking failed");
}
10 changes: 10 additions & 0 deletions TellC/src/linux_linker.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#pragma once

#include <llvm/ADT/StringRef.h>
#include <string>
#include <vector>

namespace tell {
void link_linux(llvm::StringRef object, const std::string& target,
const std::string& output, const std::vector<std::string>& libraries);
}
Loading
Loading