diff --git a/.github/workflows/idric-core.yml b/.github/workflows/idric-core.yml index 99acfb0..06faa3f 100644 --- a/.github/workflows/idric-core.yml +++ b/.github/workflows/idric-core.yml @@ -7,6 +7,7 @@ on: - 'docs/idric-implementation.md' - 'docs/developer-workbench.md' - 'docs/storage-model.md' + - 'docs/vector-index.md' - '.github/workflows/idric-core.yml' permissions: @@ -107,3 +108,8 @@ jobs: grep -Fx 'copy-label=Copy' /tmp/ib-smoke.txt grep -Fx 'copy-target=code-1' /tmp/ib-smoke.txt grep -Fx 'copy-payload=git status' /tmp/ib-smoke.txt + grep -Fx 'vector-backend=flat-f32-exact' /tmp/ib-smoke.txt + grep -Fx 'vector-scalar=f32' /tmp/ib-smoke.txt + grep -Fx 'vector-directory=indexes/vectors/pages/test-model' /tmp/ib-smoke.txt + grep -Fx 'vector-format-readable=True' /tmp/ib-smoke.txt + grep -Fx 'vector-bytes-readable=False' /tmp/ib-smoke.txt diff --git a/.github/workflows/vector-index.yml b/.github/workflows/vector-index.yml new file mode 100644 index 0000000..ab89ff6 --- /dev/null +++ b/.github/workflows/vector-index.yml @@ -0,0 +1,84 @@ +name: Filesystem vector index + +on: + pull_request: + paths: + - 'native/vector-index/**' + - 'tests/vector-index-smoke.sh' + - 'src/IB/VectorIndex.idric' + - 'src/IB/Storage.idric' + - 'docs/vector-index.md' + - '.github/workflows/vector-index.yml' + +permissions: + contents: read + +jobs: + host: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Compile warning-clean C99 backend + run: make -C native/vector-index + + - name: Exercise build, persistence, validation, and exact query + run: make -C native/vector-index test + + - name: Exercise the 10,000 URL scale with Float32 storage + run: | + index_root="$(mktemp -d)/state/indexes/vectors/pages/scale-smoke" + awk 'BEGIN { + for (row = 0; row < 10000; row++) { + printf "url-%d\t", row + for (column = 0; column < 384; column++) + printf "%s0.01", column == 0 ? "" : " " + printf "\n" + } + }' | native/vector-index/ib-vector-index build "$index_root" 384 cosine | tee /tmp/vector-scale-build.txt + grep -Fx 'count=10000' /tmp/vector-scale-build.txt + vector_file="$(sed -n 's/^vectors //p' "$index_root/format.txt")" + test "$(stat -c %s "$index_root/$vector_file")" = 15360000 + awk 'BEGIN { + for (column = 0; column < 384; column++) + printf "%s0.01", column == 0 ? "" : " " + printf "\n" + }' | native/vector-index/ib-vector-index query "$index_root" 3 > /tmp/vector-scale-query.txt + test "$(wc -l < /tmp/vector-scale-query.txt)" = 3 + sed -n '1s/\t.*//p' /tmp/vector-scale-query.txt | grep -Fx 'url-0' + + android: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - abi: arm64-v8a + compiler: aarch64-linux-android24-clang + file_architecture: ARM aarch64 + - abi: armeabi-v7a + compiler: armv7a-linux-androideabi24-clang + file_architecture: ARM + steps: + - uses: actions/checkout@v4 + + - uses: android-actions/setup-android@v3 + + - name: Install pinned Android NDK + run: sdkmanager 'ndk;27.2.12479018' + + - name: Cross-compile small Android executable + run: | + toolchain="$ANDROID_SDK_ROOT/ndk/27.2.12479018/toolchains/llvm/prebuilt/linux-x86_64/bin" + mkdir -p build/android + "$toolchain/${{ matrix.compiler }}" \ + -O2 -Wall -Wextra -Werror -Wpedantic -std=c99 \ + native/vector-index/ib_vector_index.c -lm \ + -o build/android/ib-vector-index + "$toolchain/llvm-strip" build/android/ib-vector-index + file build/android/ib-vector-index | grep -F '${{ matrix.file_architecture }}' + test "$(stat -c %s build/android/ib-vector-index)" -lt 100000 + + - uses: actions/upload-artifact@v4 + with: + name: ib-vector-index-android-${{ matrix.abi }} + path: build/android/ib-vector-index diff --git a/.gitignore b/.gitignore index 84c048a..545e50e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /build/ +/native/vector-index/ib-vector-index diff --git a/docs/idric-implementation.md b/docs/idric-implementation.md index a211c13..3240784 100644 --- a/docs/idric-implementation.md +++ b/docs/idric-implementation.md @@ -10,6 +10,7 @@ The initial source modules deliberately keep the executable boundary small: - `IB.Index` builds transparent rebuildable list indices without collapsing duplicate visits. - `IB.Storage` classifies schema-shaped paths and defines which records may be generically inspected. - `IB.FileStore` performs the first real browser-owned file I/O: it creates the store/tab directories, reads and writes tab manifests, appends history records, rejects paths outside the canonical schema, and reports filesystem failures. +- `IB.VectorIndex` owns the replaceable vector-backend specification and lowers it to the versioned text-stream process contract. The first backend is the filesystem-native exact `float32` tool in `native/vector-index`. - `IB.Inspect` summarizes physical rows without following or interpreting renderer state. The first slice does not embed Python, Ithon, WebView UI, or a renderer. Chrome/Firefox SQLite import and Android filesystem walking are platform adapters to add around this core, not reasons to move the core out of Idriç. diff --git a/docs/storage-model.md b/docs/storage-model.md index 1719721..93c5c68 100644 --- a/docs/storage-model.md +++ b/docs/storage-model.md @@ -98,6 +98,8 @@ or For tens of thousands of tabs, the first implementation can remain intentionally simple. If scans become expensive, an SQLite or custom index can be introduced without changing the canonical tab model. +The first vector implementation follows the same rule without SQLite: inspectable `format.txt` and ID files point to row-major `float32` vector bytes under `indexes/vectors/`. It uses exact scanning at the 10,000-URL scale and exposes a backend-neutral streaming command contract, so an approximate implementation can replace it without changing canonical records. See `vector-index.md`. + ## Sync Sync should operate on the browser-owned records and snapshots, not on a renderer profile directory. This allows multiple browser front ends or machines to share the same durable browsing corpus while maintaining separate live renderer processes and caches. diff --git a/docs/vector-index.md b/docs/vector-index.md new file mode 100644 index 0000000..a35979d --- /dev/null +++ b/docs/vector-index.md @@ -0,0 +1,61 @@ +# Filesystem vector index + +IB's first vector backend is a flat exact scan over 32-bit floats. It is deliberately a file tool, not a database. + +At the initial 10,000-URL workbench size, 384-dimensional vectors occupy 15.4 MB and one exact query performs 3.84 million multiply-adds. A graph index would add persistent graph state, tuning, and approximate results before this corpus needs them. + +## Boundary + +Canonical URLs, visits, extracted text, and document identities remain ordinary inspectable browser records. Embeddings and their index are derived state and can be deleted and rebuilt. + +Idriç owns the index specification: collection, embedding model, dimensions, metric, and selected backend. `IB.VectorIndex` lowers that specification to a versioned process contract. Grease or thin platform glue may run the selected program. The initial program is `ib-vector-index`, implemented in C99 so the same source builds for Linux and the Android NDK. + +A replacement backend must implement the same standard-input/standard-output contract: + +```text +BACKEND build INDEX_DIRECTORY DIMENSIONS cosine|dot < rows.tsv +BACKEND query INDEX_DIRECTORY RESULT_COUNT < vector.txt +BACKEND check INDEX_DIRECTORY +BACKEND inspect INDEX_DIRECTORY +``` + +Build input has one row per line: + +```text +document-id0.1 -0.2 0.3 ... +``` + +Query input is one space-separated vector. Query output is score-descending text: + +```text +document-id0.8125 +``` + +This interface does not expose the flat backend's private files. A later USearch or HNSW program can occupy the same boundary without changing canonical browser state or the callers that stream rows and queries. + +## Files + +An index lives below the existing derived namespace: + +```text +state/indexes/vectors/// + format.txt + ids-.txt + vectors-.f32 +``` + +`format.txt` is the atomic pointer to one immutable generation. It records the contract version, backend, scalar, byte order, metric, dimensions, row count, and current data filenames. IDs remain text. Vectors are row-major little-endian IEEE 754 `float32` values. + +New builds write new generation files and replace `format.txt` last. A failed build therefore cannot make a partial generation current. Old generation files may be removed during serialized index maintenance after active queries finish. + +The generic inspector may read `format.txt` and generated ID text. It classifies the vector bytes as derived but does not treat them as generic readable text. + +## Metric and precision + +`cosine` normalizes stored and query vectors once and then uses a dot product. Zero, NaN, and infinite vectors are rejected. `dot` stores the supplied values without normalization. + +Storage and accumulation both use 32-bit float. This is the natural precision of common embedding outputs, halves the vector bytes relative to doubles, and is sufficient for similarity ranking here. The rebuildable boundary lets a later backend use another representation without migrating canonical data. + +## Growth path + +The flat scan is also the correctness reference for any approximate replacement. Add an ANN backend only after measurements on the phone show that exact query latency or corpus size is actually a problem. USearch is the leading replacement candidate because it has a C API, Android support, `f32`, and disk-backed index viewing, but it is not a dependency of this first backend. diff --git a/native/vector-index/Makefile b/native/vector-index/Makefile new file mode 100644 index 0000000..7355f7e --- /dev/null +++ b/native/vector-index/Makefile @@ -0,0 +1,17 @@ +CC ?= cc +CFLAGS ?= -O2 +CPPFLAGS ?= +WARNINGS = -Wall -Wextra -Werror -Wpedantic + +.PHONY: all clean test + +all: ib-vector-index + +ib-vector-index: ib_vector_index.c + $(CC) $(CPPFLAGS) $(CFLAGS) $(WARNINGS) -std=c99 $< -lm -o $@ + +test: ib-vector-index + sh ../../tests/vector-index-smoke.sh ./ib-vector-index + +clean: + rm -f ib-vector-index diff --git a/native/vector-index/ib_vector_index.c b/native/vector-index/ib_vector_index.c new file mode 100644 index 0000000..ca842e5 --- /dev/null +++ b/native/vector-index/ib_vector_index.c @@ -0,0 +1,771 @@ +#define _POSIX_C_SOURCE 200809L + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FORMAT_HEADER "ib-vector-index 1" +#define BACKEND_NAME "flat-f32-exact" +#define PATH_BUFFER 4096 + +enum metric_kind { + METRIC_DOT, + METRIC_COSINE +}; + +struct manifest { + enum metric_kind metric; + size_t dimensions; + size_t count; + char ids_file[256]; + char vectors_file[256]; +}; + +struct match { + float score; + size_t row; + char *id; +}; + +static int fail(const char *message) { + fprintf(stderr, "ib-vector-index: %s\n", message); + return 1; +} + +static int fail_path(const char *action, const char *path) { + fprintf(stderr, "ib-vector-index: %s %s: %s\n", action, path, + strerror(errno)); + return 1; +} + +static int path_join(char *out, size_t capacity, const char *directory, + const char *name) { + int written = snprintf(out, capacity, "%s/%s", directory, name); + if (written < 0 || (size_t)written >= capacity) { + fprintf(stderr, "ib-vector-index: path is too long: %s/%s\n", directory, + name); + return 0; + } + return 1; +} + +static int plain_filename(const char *name) { + return name[0] != '\0' && strcmp(name, ".") != 0 && + strcmp(name, "..") != 0 && strchr(name, '/') == NULL; +} + +static int make_directories(const char *path) { + char copy[PATH_BUFFER]; + size_t length = strlen(path); + size_t start = 1; + + if (length == 0 || length >= sizeof(copy)) { + return fail("index directory is empty or too long"); + } + + memcpy(copy, path, length + 1); + if (copy[length - 1] == '/' && length > 1) { + copy[length - 1] = '\0'; + } + if (copy[0] != '/') { + start = 0; + } + + for (size_t i = start; copy[i] != '\0'; ++i) { + if (copy[i] != '/') { + continue; + } + copy[i] = '\0'; + if (copy[0] != '\0' && mkdir(copy, 0700) != 0 && errno != EEXIST) { + return fail_path("cannot create", copy); + } + copy[i] = '/'; + } + + if (mkdir(copy, 0700) != 0 && errno != EEXIST) { + return fail_path("cannot create", copy); + } + return 0; +} + +static int flush_file(FILE *file, const char *path) { + if (fflush(file) != 0) { + return fail_path("cannot flush", path); + } + if (fsync(fileno(file)) != 0) { + return fail_path("cannot synchronize", path); + } + return 0; +} + +static FILE *create_exclusive(const char *path) { + int descriptor = open(path, O_WRONLY | O_CREAT | O_EXCL, 0600); + if (descriptor < 0) { + return NULL; + } + FILE *file = fdopen(descriptor, "wb"); + if (file == NULL) { + int saved_errno = errno; + close(descriptor); + unlink(path); + errno = saved_errno; + } + return file; +} + +static int parse_size(const char *text, size_t *value) { + char *end = NULL; + unsigned long long parsed; + + errno = 0; + parsed = strtoull(text, &end, 10); + if (errno != 0 || end == text || *end != '\0' || parsed == 0 || + parsed > SIZE_MAX) { + return 0; + } + *value = (size_t)parsed; + return 1; +} + +static const char *metric_text(enum metric_kind metric) { + return metric == METRIC_COSINE ? "cosine" : "dot"; +} + +static int parse_metric(const char *text, enum metric_kind *metric) { + if (strcmp(text, "dot") == 0) { + *metric = METRIC_DOT; + return 1; + } + if (strcmp(text, "cosine") == 0) { + *metric = METRIC_COSINE; + return 1; + } + return 0; +} + +static int host_is_little_endian(void) { + const uint16_t one = 1; + return *((const unsigned char *)&one) == 1; +} + +static int write_float32_le(FILE *file, float value) { + unsigned char bytes[4]; + uint32_t bits; + memcpy(&bits, &value, sizeof(bits)); + bytes[0] = (unsigned char)(bits & 0xffu); + bytes[1] = (unsigned char)((bits >> 8u) & 0xffu); + bytes[2] = (unsigned char)((bits >> 16u) & 0xffu); + bytes[3] = (unsigned char)((bits >> 24u) & 0xffu); + return fwrite(bytes, sizeof(bytes), 1, file) == 1; +} + +static float read_float32_le(const unsigned char *bytes) { + uint32_t bits = (uint32_t)bytes[0] | ((uint32_t)bytes[1] << 8u) | + ((uint32_t)bytes[2] << 16u) | + ((uint32_t)bytes[3] << 24u); + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static int normalize(float *values, size_t dimensions) { + float squared_norm = 0.0f; + for (size_t i = 0; i < dimensions; ++i) { + squared_norm += values[i] * values[i]; + } + if (!(squared_norm > 0.0f) || !isfinite(squared_norm)) { + return 0; + } + float scale = 1.0f / sqrtf(squared_norm); + for (size_t i = 0; i < dimensions; ++i) { + values[i] *= scale; + } + return 1; +} + +static int parse_vector(char *text, size_t dimensions, float *values) { + char *cursor = text; + for (size_t i = 0; i < dimensions; ++i) { + char *end = NULL; + while (*cursor == ' ' || *cursor == '\t') { + ++cursor; + } + errno = 0; + values[i] = strtof(cursor, &end); + if (errno != 0 || end == cursor || !isfinite(values[i])) { + return 0; + } + cursor = end; + } + while (*cursor == ' ' || *cursor == '\t') { + ++cursor; + } + return *cursor == '\0'; +} + +static void trim_line_end(char *line) { + size_t length = strlen(line); + while (length > 0 && (line[length - 1] == '\n' || line[length - 1] == '\r')) { + line[--length] = '\0'; + } +} + +static int write_manifest_file(const char *path, const struct manifest *manifest) { + FILE *file = fopen(path, "wb"); + if (file == NULL) { + return fail_path("cannot open", path); + } + int bad = fprintf(file, + FORMAT_HEADER "\n" + "backend " BACKEND_NAME "\n" + "scalar f32\n" + "byte_order little\n" + "metric %s\n" + "dimensions %zu\n" + "count %zu\n" + "ids %s\n" + "vectors %s\n", + metric_text(manifest->metric), manifest->dimensions, + manifest->count, manifest->ids_file, + manifest->vectors_file) < 0; + if (!bad && flush_file(file, path) != 0) { + bad = 1; + } + if (fclose(file) != 0) { + bad = 1; + } + return bad ? fail("cannot write index manifest") : 0; +} + +static int manifest_field(char *line, const char *name, char **value) { + size_t length = strlen(name); + if (strncmp(line, name, length) != 0 || line[length] != ' ') { + return 0; + } + *value = line + length + 1; + return **value != '\0'; +} + +static int read_manifest(const char *directory, struct manifest *manifest) { + char path[PATH_BUFFER]; + char *line = NULL; + size_t capacity = 0; + ssize_t length; + int line_number = 0; + int seen_backend = 0, seen_scalar = 0, seen_order = 0; + int seen_metric = 0, seen_dimensions = 0, seen_count = 0; + int seen_ids = 0, seen_vectors = 0; + + memset(manifest, 0, sizeof(*manifest)); + if (!path_join(path, sizeof(path), directory, "format.txt")) { + return 1; + } + FILE *file = fopen(path, "rb"); + if (file == NULL) { + return fail_path("cannot open", path); + } + + while ((length = getline(&line, &capacity, file)) >= 0) { + char *value = NULL; + (void)length; + ++line_number; + trim_line_end(line); + if (line_number == 1) { + if (strcmp(line, FORMAT_HEADER) != 0) { + goto invalid; + } + } else if (manifest_field(line, "backend", &value)) { + seen_backend = strcmp(value, BACKEND_NAME) == 0; + } else if (manifest_field(line, "scalar", &value)) { + seen_scalar = strcmp(value, "f32") == 0; + } else if (manifest_field(line, "byte_order", &value)) { + seen_order = strcmp(value, "little") == 0; + } else if (manifest_field(line, "metric", &value)) { + seen_metric = parse_metric(value, &manifest->metric); + } else if (manifest_field(line, "dimensions", &value)) { + seen_dimensions = parse_size(value, &manifest->dimensions); + } else if (manifest_field(line, "count", &value)) { + char *end = NULL; + unsigned long long parsed; + errno = 0; + parsed = strtoull(value, &end, 10); + seen_count = errno == 0 && end != value && *end == '\0' && + parsed <= SIZE_MAX; + manifest->count = (size_t)parsed; + } else if (manifest_field(line, "ids", &value)) { + seen_ids = plain_filename(value) && + snprintf(manifest->ids_file, sizeof(manifest->ids_file), "%s", + value) < (int)sizeof(manifest->ids_file); + } else if (manifest_field(line, "vectors", &value)) { + seen_vectors = + plain_filename(value) && + snprintf(manifest->vectors_file, sizeof(manifest->vectors_file), "%s", + value) < (int)sizeof(manifest->vectors_file); + } + } + + free(line); + if (ferror(file)) { + fclose(file); + return fail_path("cannot read", path); + } + fclose(file); + if (line_number < 9 || !seen_backend || !seen_scalar || !seen_order || + !seen_metric || !seen_dimensions || !seen_count || !seen_ids || + !seen_vectors) { + return fail("index manifest is incomplete or unsupported"); + } + return 0; + +invalid: + free(line); + fclose(file); + return fail("index manifest has an unsupported format"); +} + +static int checked_vector_bytes(const struct manifest *manifest, size_t *bytes) { + if (manifest->count != 0 && manifest->dimensions > SIZE_MAX / manifest->count) { + return 0; + } + size_t values = manifest->count * manifest->dimensions; + if (values > SIZE_MAX / sizeof(float)) { + return 0; + } + *bytes = values * sizeof(float); + return 1; +} + +static int check_index(const char *directory, struct manifest *manifest, + int announce) { + char ids_path[PATH_BUFFER], vectors_path[PATH_BUFFER]; + struct stat vector_stat; + size_t expected_bytes; + size_t ids = 0; + char *line = NULL; + size_t capacity = 0; + + if (read_manifest(directory, manifest) != 0) { + return 1; + } + if (!path_join(ids_path, sizeof(ids_path), directory, manifest->ids_file) || + !path_join(vectors_path, sizeof(vectors_path), directory, + manifest->vectors_file)) { + return 1; + } + if (!checked_vector_bytes(manifest, &expected_bytes)) { + return fail("index dimensions overflow file size"); + } + if (stat(vectors_path, &vector_stat) != 0) { + return fail_path("cannot inspect", vectors_path); + } + if (vector_stat.st_size < 0 || (uintmax_t)vector_stat.st_size != expected_bytes) { + return fail("vector file size does not match the manifest"); + } + + FILE *ids_file = fopen(ids_path, "rb"); + if (ids_file == NULL) { + return fail_path("cannot open", ids_path); + } + while (getline(&line, &capacity, ids_file) >= 0) { + trim_line_end(line); + if (line[0] == '\0' || strchr(line, '\t') != NULL) { + free(line); + fclose(ids_file); + return fail("ID file contains an empty or tabbed ID"); + } + ++ids; + } + free(line); + if (ferror(ids_file)) { + fclose(ids_file); + return fail_path("cannot read", ids_path); + } + fclose(ids_file); + if (ids != manifest->count) { + return fail("ID count does not match the manifest"); + } + + if (announce) { + printf("check=ok\nbackend=%s\nscalar=f32\nmetric=%s\ndimensions=%zu\ncount=%zu\n", + BACKEND_NAME, metric_text(manifest->metric), manifest->dimensions, + manifest->count); + } + return 0; +} + +static int build_index(const char *directory, const char *dimension_text, + const char *metric_name) { + struct manifest manifest; + char generation[96]; + char ids_path[PATH_BUFFER], vectors_path[PATH_BUFFER]; + char manifest_path[PATH_BUFFER], manifest_temp[PATH_BUFFER]; + char *line = NULL; + size_t line_capacity = 0; + ssize_t line_length; + float *values = NULL; + FILE *ids_file = NULL, *vectors_file = NULL; + int status = 1; + + memset(&manifest, 0, sizeof(manifest)); + if (!parse_size(dimension_text, &manifest.dimensions)) { + return fail("dimensions must be a positive integer"); + } + if (manifest.dimensions > SIZE_MAX / sizeof(*values)) { + return fail("dimensions are too large for this process"); + } + if (!parse_metric(metric_name, &manifest.metric)) { + return fail("metric must be cosine or dot"); + } + if (make_directories(directory) != 0) { + return 1; + } + + struct timespec now; + if (clock_gettime(CLOCK_REALTIME, &now) != 0) { + return fail_path("cannot read clock for", directory); + } + snprintf(generation, sizeof(generation), "%lld-%09ld-%ld", + (long long)now.tv_sec, now.tv_nsec, (long)getpid()); + snprintf(manifest.ids_file, sizeof(manifest.ids_file), "ids-%s.txt", + generation); + snprintf(manifest.vectors_file, sizeof(manifest.vectors_file), + "vectors-%s.f32", generation); + + if (!path_join(ids_path, sizeof(ids_path), directory, manifest.ids_file) || + !path_join(vectors_path, sizeof(vectors_path), directory, + manifest.vectors_file) || + !path_join(manifest_path, sizeof(manifest_path), directory, "format.txt") || + snprintf(manifest_temp, sizeof(manifest_temp), "%s.tmp.%ld", manifest_path, + (long)getpid()) >= (int)sizeof(manifest_temp)) { + return 1; + } + + ids_file = create_exclusive(ids_path); + if (ids_file == NULL) { + return fail_path("cannot create", ids_path); + } + vectors_file = create_exclusive(vectors_path); + if (vectors_file == NULL) { + fail_path("cannot create", vectors_path); + goto cleanup; + } + values = malloc(manifest.dimensions * sizeof(*values)); + if (values == NULL) { + fail("out of memory while reading vectors"); + goto cleanup; + } + + while ((line_length = getline(&line, &line_capacity, stdin)) >= 0) { + char *tab; + (void)line_length; + trim_line_end(line); + if (line[0] == '\0') { + continue; + } + tab = strchr(line, '\t'); + if (tab == NULL || tab == line) { + fail("each input row must be ID, tab, then vector values"); + goto cleanup; + } + *tab = '\0'; + if (strchr(tab + 1, '\n') != NULL || + !parse_vector(tab + 1, manifest.dimensions, values)) { + fail("an input row has the wrong vector dimension or a non-finite value"); + goto cleanup; + } + if (manifest.metric == METRIC_COSINE && + !normalize(values, manifest.dimensions)) { + fail("cosine vectors must have a finite, nonzero norm"); + goto cleanup; + } + if (fprintf(ids_file, "%s\n", line) < 0) { + fail_path("cannot write", ids_path); + goto cleanup; + } + for (size_t i = 0; i < manifest.dimensions; ++i) { + if (!write_float32_le(vectors_file, values[i])) { + fail_path("cannot write", vectors_path); + goto cleanup; + } + } + if (manifest.count == SIZE_MAX) { + fail("too many vector rows"); + goto cleanup; + } + ++manifest.count; + } + if (ferror(stdin)) { + fail("cannot read vector rows from standard input"); + goto cleanup; + } + if (flush_file(ids_file, ids_path) != 0 || + flush_file(vectors_file, vectors_path) != 0) { + goto cleanup; + } + int ids_close_failed = fclose(ids_file) != 0; + ids_file = NULL; + int vectors_close_failed = fclose(vectors_file) != 0; + vectors_file = NULL; + if (ids_close_failed || vectors_close_failed) { + fail("cannot close new index files"); + goto cleanup; + } + + if (write_manifest_file(manifest_temp, &manifest) != 0) { + goto cleanup; + } + if (rename(manifest_temp, manifest_path) != 0) { + fail_path("cannot install", manifest_path); + goto cleanup; + } + + printf("build=ok\nbackend=%s\nscalar=f32\nmetric=%s\ndimensions=%zu\ncount=%zu\n", + BACKEND_NAME, metric_text(manifest.metric), manifest.dimensions, + manifest.count); + status = 0; + +cleanup: + free(values); + free(line); + if (ids_file != NULL) { + fclose(ids_file); + } + if (vectors_file != NULL) { + fclose(vectors_file); + } + if (status != 0) { + unlink(ids_path); + unlink(vectors_path); + unlink(manifest_temp); + } + return status; +} + +static float dot_product(const unsigned char *stored, const float *query, + size_t dimensions) { + float score = 0.0f; + if (host_is_little_endian()) { + for (size_t i = 0; i < dimensions; ++i) { + float value; + memcpy(&value, stored + i * sizeof(float), sizeof(value)); + score += value * query[i]; + } + } else { + for (size_t i = 0; i < dimensions; ++i) { + score += read_float32_le(stored + i * sizeof(float)) * query[i]; + } + } + return score; +} + +static int match_before(float score, size_t row, const struct match *other) { + return score > other->score || (score == other->score && row < other->row); +} + +static void consider_match(struct match *matches, size_t *used, size_t limit, + float score, size_t row, const char *id) { + size_t position = 0; + while (position < *used && !match_before(score, row, &matches[position])) { + ++position; + } + if (position >= limit) { + return; + } + size_t new_used = *used < limit ? *used + 1 : *used; + if (*used == limit) { + free(matches[limit - 1].id); + } + for (size_t i = new_used - 1; i > position; --i) { + matches[i] = matches[i - 1]; + } + matches[position].score = score; + matches[position].row = row; + matches[position].id = strdup(id); + if (matches[position].id == NULL) { + fail("out of memory while retaining matches"); + exit(1); + } + *used = new_used; +} + +static int query_index(const char *directory, const char *limit_text) { + struct manifest manifest; + char ids_path[PATH_BUFFER], vectors_path[PATH_BUFFER]; + char *line = NULL; + size_t line_capacity = 0, limit, used = 0, vector_bytes; + float *query = NULL; + struct match *matches = NULL; + FILE *ids_file = NULL; + int vectors_fd = -1; + unsigned char *mapped = MAP_FAILED; + int status = 1; + + if (!parse_size(limit_text, &limit)) { + return fail("result count must be a positive integer"); + } + if (check_index(directory, &manifest, 0) != 0) { + return 1; + } + if (limit > manifest.count) { + limit = manifest.count; + } + query = malloc(manifest.dimensions * sizeof(*query)); + if (query == NULL) { + return fail("out of memory while reading query"); + } + if (getline(&line, &line_capacity, stdin) < 0) { + fail("query vector is missing on standard input"); + goto cleanup; + } + trim_line_end(line); + if (!parse_vector(line, manifest.dimensions, query)) { + fail("query has the wrong vector dimension or a non-finite value"); + goto cleanup; + } + if (manifest.metric == METRIC_COSINE && + !normalize(query, manifest.dimensions)) { + fail("cosine query must have a finite, nonzero norm"); + goto cleanup; + } + if (getline(&line, &line_capacity, stdin) >= 0) { + trim_line_end(line); + if (line[0] != '\0') { + fail("query accepts exactly one vector"); + goto cleanup; + } + } + + if (limit == 0) { + status = 0; + goto cleanup; + } + matches = calloc(limit, sizeof(*matches)); + if (matches == NULL) { + fail("out of memory while allocating matches"); + goto cleanup; + } + if (!checked_vector_bytes(&manifest, &vector_bytes) || + !path_join(ids_path, sizeof(ids_path), directory, manifest.ids_file) || + !path_join(vectors_path, sizeof(vectors_path), directory, + manifest.vectors_file)) { + goto cleanup; + } + ids_file = fopen(ids_path, "rb"); + if (ids_file == NULL) { + fail_path("cannot open", ids_path); + goto cleanup; + } + vectors_fd = open(vectors_path, O_RDONLY); + if (vectors_fd < 0) { + fail_path("cannot open", vectors_path); + goto cleanup; + } + mapped = mmap(NULL, vector_bytes, PROT_READ, MAP_PRIVATE, vectors_fd, 0); + if (mapped == MAP_FAILED) { + fail_path("cannot map", vectors_path); + goto cleanup; + } + + for (size_t row = 0; row < manifest.count; ++row) { + if (getline(&line, &line_capacity, ids_file) < 0) { + fail("ID file ended during query"); + goto cleanup; + } + trim_line_end(line); + const unsigned char *stored = + mapped + row * manifest.dimensions * sizeof(float); + float score = dot_product(stored, query, manifest.dimensions); + if (manifest.metric == METRIC_COSINE) { + if (score > 1.0f) { + score = 1.0f; + } else if (score < -1.0f) { + score = -1.0f; + } + } + consider_match(matches, &used, limit, score, row, line); + } + for (size_t i = 0; i < used; ++i) { + printf("%s\t%.9g\n", matches[i].id, (double)matches[i].score); + } + status = 0; + +cleanup: + if (mapped != MAP_FAILED) { + munmap(mapped, vector_bytes); + } + if (vectors_fd >= 0) { + close(vectors_fd); + } + if (ids_file != NULL) { + fclose(ids_file); + } + if (matches != NULL) { + for (size_t i = 0; i < used; ++i) { + free(matches[i].id); + } + } + free(matches); + free(query); + free(line); + return status; +} + +static int inspect_index(const char *directory) { + struct manifest manifest; + if (check_index(directory, &manifest, 0) != 0) { + return 1; + } + printf(FORMAT_HEADER "\n" + "backend " BACKEND_NAME "\n" + "scalar f32\n" + "byte_order little\n" + "metric %s\n" + "dimensions %zu\n" + "count %zu\n" + "ids %s\n" + "vectors %s\n", + metric_text(manifest.metric), manifest.dimensions, manifest.count, + manifest.ids_file, manifest.vectors_file); + return 0; +} + +static void usage(FILE *out) { + fprintf(out, + "usage:\n" + " ib-vector-index build INDEX_DIRECTORY DIMENSIONS cosine|dot < rows.tsv\n" + " ib-vector-index query INDEX_DIRECTORY RESULT_COUNT < vector.txt\n" + " ib-vector-index check INDEX_DIRECTORY\n" + " ib-vector-index inspect INDEX_DIRECTORY\n\n" + "build rows are: IDnumber number ...\n" + "query results are: IDscore\n"); +} + +int main(int argc, char **argv) { + if (argc == 6 && strcmp(argv[1], "build") == 0) { + return fail("build received too many arguments"); + } + if (argc == 5 && strcmp(argv[1], "build") == 0) { + return build_index(argv[2], argv[3], argv[4]); + } + if (argc == 4 && strcmp(argv[1], "query") == 0) { + return query_index(argv[2], argv[3]); + } + if (argc == 3 && strcmp(argv[1], "check") == 0) { + struct manifest manifest; + return check_index(argv[2], &manifest, 1); + } + if (argc == 3 && strcmp(argv[1], "inspect") == 0) { + return inspect_index(argv[2]); + } + usage(stderr); + return 2; +} diff --git a/src/IB/Storage.idric b/src/IB/Storage.idric index 07a6328..a4078d8 100644 --- a/src/IB/Storage.idric +++ b/src/IB/Storage.idric @@ -79,11 +79,30 @@ transparent_index name = elem name [ "queries.tsv", "terms.tsv", "summary.json" ] +char_prefix : List Char → List Char → Bool +char_prefix [] _ = True +char_prefix _ [] = False +char_prefix (wanted :: more_wanted) (actual :: more_actual) = + wanted == actual && char_prefix more_wanted more_actual + +begins : String → String → Bool +begins opening value = char_prefix (unpack opening) (unpack value) + +finishes : String → String → Bool +finishes closing value = + char_prefix (reverse (unpack closing)) (reverse (unpack value)) + +vector_text_metadata : List String → Bool +vector_text_metadata ["indexes", "vectors", _, _, "format.txt"] = True +vector_text_metadata ["indexes", "vectors", _, _, name] = + begins "ids-" name && finishes ".txt" name +vector_text_metadata _ = False + public export transparent_path : String → Bool transparent_path path = let parts = path_parts path in - parts == ["visits.jsonl"] || is_tab_record parts || + parts == ["visits.jsonl"] || is_tab_record parts || vector_text_metadata parts || case parts of ["indexes", name] ⇒ transparent_index name _ ⇒ False diff --git a/src/IB/VectorIndex.idric b/src/IB/VectorIndex.idric new file mode 100644 index 0000000..dd5a8c7 --- /dev/null +++ b/src/IB/VectorIndex.idric @@ -0,0 +1,94 @@ +module IB.VectorIndex + +import IB.Storage + +%default total + +public export +choice vector_metric one_of + cosine + dot_product + +public export +vector_metric_text : vector_metric → String +vector_metric_text cosine = "cosine" +vector_metric_text dot_product = "dot" + +public export +record VectorBackend where + constructor Backend + backend_name : String + backend_program : String + scalar_name : String + contract_version : Nat + +public export +flat_f32_exact : VectorBackend +flat_f32_exact = Backend "flat-f32-exact" "ib-vector-index" "f32" 1 + +public export +record VectorIndexSpec where + constructor IndexSpec + backend : VectorBackend + collection : String + embedding_model : String + dimensions : Nat + metric : vector_metric + +safe_component : String → Bool +safe_component value = + value /= "." && value /= ".." && safe_relative_path value && + path_parts value == [value] + +public export +vector_index_relative_directory : VectorIndexSpec → Maybe String +vector_index_relative_directory spec = + if safe_component (collection spec) && safe_component (embedding_model spec) + then Just ("indexes/vectors/" ++ collection spec ++ "/" ++ embedding_model spec) + else Nothing + +public export +data BackendCommand = Run String (List String) + +public export +command_program : BackendCommand → String +command_program (Run program _) = program + +public export +command_arguments : BackendCommand → List String +command_arguments (Run _ arguments) = arguments + +index_directory : String → VectorIndexSpec → Maybe String +index_directory "" _ = Nothing +index_directory store_root spec = + case vector_index_relative_directory spec of + Nothing ⇒ Nothing + Just relative ⇒ Just (store_root ++ "/" ++ relative) + +public export +build_command : String → VectorIndexSpec → Maybe BackendCommand +build_command store_root spec = + case index_directory store_root spec of + Nothing ⇒ Nothing + Just directory ⇒ + if dimensions spec == 0 + then Nothing + else Just (Run + (backend_program (backend spec)) + [ "build" + , directory + , show (dimensions spec) + , vector_metric_text (metric spec) + ]) + +public export +query_command : String → VectorIndexSpec → Nat → Maybe BackendCommand +query_command store_root spec result_count = + case index_directory store_root spec of + Nothing ⇒ Nothing + Just directory ⇒ + if dimensions spec == 0 || result_count == 0 + then Nothing + else Just (Run + (backend_program (backend spec)) + ["query", directory, show result_count]) diff --git a/src/Smoke.idric b/src/Smoke.idric index ed4a6b6..cee92ab 100644 --- a/src/Smoke.idric +++ b/src/Smoke.idric @@ -5,6 +5,7 @@ import IB.Index import IB.Storage import IB.Inspect import IB.DisplayRepair +import IB.VectorIndex %default total @@ -20,6 +21,10 @@ first_or : Nat → List Nat → Nat first_or fallback [] = fallback first_or fallback (value :: _) = value +maybe_string : String → Maybe String → String +maybe_string fallback Nothing = fallback +maybe_string _ (Just value) = value + main : IO () main = do let raw = ingest_raw_lines ["# ignored", " https://example.test/a ", "https://example.test/a", "https://other.test/b"] @@ -32,6 +37,7 @@ main = do FileRow "renderer/passwords" 40 regular_file ] let copy_repair = copy_button "code-1" "git status" code_block + let vector_spec = IndexSpec flat_f32_exact "pages" "test-model" 384 cosine putStrLn ("ingest=" ++ show (length raw)) putStrLn ("duplicate-url-count=" ++ show (row_count "https://example.test/a" (url_rows indices))) putStrLn ("newest-order=" ++ show (first_or 99 (index_chronology indices))) @@ -44,3 +50,8 @@ main = do putStrLn ("copy-label=" ++ repair_label copy_repair) putStrLn ("copy-target=" ++ copy_target_id copy_repair) putStrLn ("copy-payload=" ++ copy_payload copy_repair) + putStrLn ("vector-backend=" ++ backend_name (backend vector_spec)) + putStrLn ("vector-scalar=" ++ scalar_name (backend vector_spec)) + putStrLn ("vector-directory=" ++ maybe_string "invalid" (vector_index_relative_directory vector_spec)) + putStrLn ("vector-format-readable=" ++ show (readable_path "indexes/vectors/pages/test-model/format.txt")) + putStrLn ("vector-bytes-readable=" ++ show (readable_path "indexes/vectors/pages/test-model/vectors-1.f32")) diff --git a/tests/vector-index-smoke.sh b/tests/vector-index-smoke.sh new file mode 100644 index 0000000..bd99a87 --- /dev/null +++ b/tests/vector-index-smoke.sh @@ -0,0 +1,58 @@ +#!/bin/sh +set -eu + +program=${1:?usage: vector-index-smoke.sh PROGRAM} +temporary=$(mktemp -d) +trap 'rm -rf "$temporary"' EXIT HUP INT TERM +index=$temporary/state/indexes/vectors/pages/test-model + +printf '%s\n' \ + 'book-page 1 0 0' \ + 'tool-page 0 1 0' \ + 'mixed-page 1 1 0' | + "$program" build "$index" 3 cosine > "$temporary/build.txt" + +grep -Fx 'build=ok' "$temporary/build.txt" +grep -Fx 'backend=flat-f32-exact' "$temporary/build.txt" +grep -Fx 'scalar=f32' "$temporary/build.txt" +grep -Fx 'metric=cosine' "$temporary/build.txt" +grep -Fx 'dimensions=3' "$temporary/build.txt" +grep -Fx 'count=3' "$temporary/build.txt" + +"$program" check "$index" > "$temporary/check.txt" +grep -Fx 'check=ok' "$temporary/check.txt" + +printf '%s\n' '0.9 0.1 0' | + "$program" query "$index" 2 > "$temporary/results.txt" +sed -n '1s/ .*//p' "$temporary/results.txt" | grep -Fx 'book-page' +sed -n '2s/ .*//p' "$temporary/results.txt" | grep -Fx 'mixed-page' + +cp "$index/format.txt" "$temporary/format-before-failed-build.txt" +printf '%s\n' 'broken-page 1 0' | + if "$program" build "$index" 3 cosine > /dev/null 2> "$temporary/failed-build.txt"; then + echo 'wrong-dimension build unexpectedly succeeded' >&2 + exit 1 + fi +cmp "$temporary/format-before-failed-build.txt" "$index/format.txt" +printf '%s\n' '0.9 0.1 0' | + "$program" query "$index" 1 | sed -n '1s/ .*//p' | grep -Fx 'book-page' + +vectors=$(sed -n 's/^vectors //p' "$index/format.txt") +test -n "$vectors" +test "$(wc -c < "$index/$vectors" | tr -d ' ')" = 36 + +printf '%s\n' '1 0' | + if "$program" query "$index" 1 > /dev/null 2> "$temporary/wrong-dimension.txt"; then + echo 'wrong-dimension query unexpectedly succeeded' >&2 + exit 1 + fi +grep -F 'wrong vector dimension' "$temporary/wrong-dimension.txt" + +printf '%s\n' 'zero 0 0 0' | + if "$program" build "$temporary/zero-index" 3 cosine > /dev/null 2> "$temporary/zero.txt"; then + echo 'zero cosine vector unexpectedly succeeded' >&2 + exit 1 + fi +grep -F 'nonzero norm' "$temporary/zero.txt" + +printf '%s\n' 'vector-index-smoke=ok'