From 53c15c0fd550904d48f52b8a9e56be6d3b515bbd Mon Sep 17 00:00:00 2001 From: dmehala Date: Sat, 31 Oct 2026 16:48:30 +0100 Subject: [PATCH 1/4] Add zml/llmd inference guide Changes: - Add zml/llmd to the GPU inference section with instructions for running LFM3. - Update agent harness guide with zml/llmd. --- deployment/gpu-inference/transformers.mdx | 2 +- deployment/gpu-inference/zml-llmd.mdx | 130 ++++++++++++++++++++++ deployment/on-device/llama-cpp.mdx | 2 +- docs.json | 3 +- examples/agent-harnesses.mdx | 37 +++++- guides/hardware-evaluation.mdx | 1 + guides/migration-guide.mdx | 2 +- guides/use-case-evaluation.mdx | 2 +- lfm/help/faqs.mdx | 1 + lfm/models/complete-library.mdx | 4 +- lfm/models/lfm3-1.7b.mdx | 31 +++++- link-snapshot.yaml | 1 + 12 files changed, 204 insertions(+), 12 deletions(-) create mode 100644 deployment/gpu-inference/zml-llmd.mdx diff --git a/deployment/gpu-inference/transformers.mdx b/deployment/gpu-inference/transformers.mdx index 1610829a..499828a1 100644 --- a/deployment/gpu-inference/transformers.mdx +++ b/deployment/gpu-inference/transformers.mdx @@ -7,7 +7,7 @@ description: "Run Liquid AI LFM models with Hugging Face Transformers. Load, con Use Transformers for simple inference without extra dependencies, research and experimentation, or integration with the Hugging Face ecosystem. -Transformers provides the most flexibility for model development and is ideal for users who want direct access to model internals. For production deployments with high throughput, consider using [vLLM](/deployment/gpu-inference/vllm). +Transformers provides the most flexibility for model development and is ideal for users who want direct access to model internals. For production deployments with high throughput, consider using [vLLM](/deployment/gpu-inference/vllm). For LFM3 text serving on Apple Metal, NVIDIA, AMD, Intel, or Google TPU, see [zml/llmd](/deployment/gpu-inference/zml-llmd).
diff --git a/deployment/gpu-inference/zml-llmd.mdx b/deployment/gpu-inference/zml-llmd.mdx new file mode 100644 index 00000000..83f6ebb2 --- /dev/null +++ b/deployment/gpu-inference/zml-llmd.mdx @@ -0,0 +1,130 @@ +--- +title: "zml/llmd" +description: "Run LFM3 locally with zml/llmd on Apple Metal, NVIDIA, AMD, Intel, and Google TPU." +--- + +[zml/llmd](https://zml.ai/llmd/) is an inference server built on [zml](https://github.com/zml/zml) that runs LFM3 across Apple Metal, NVIDIA CUDA, AMD ROCm, Intel oneAPI, and Google TPU. zml/llmd serves models exclusively through an OpenAI-compatible API. + + + Use zml/llmd for high-throughput serving and batch processing with minimal setup, no Python dependencies, and fast startup across all supported platforms. + + +## Quickstart + +LFM3 support is available in **zml/llmd 20261029.0** and later. The examples below use FP8 weights on NVIDIA, AMD, and Apple Metal, and BF16 weights on Intel and Google TPU. + +### Start the server + +Choose your platform to start the server: + + + + #### Prerequisites + + A Linux host with an NVIDIA GPU, Docker, the NVIDIA driver, and NVIDIA Container Toolkit configured for GPU access. + + #### Run + + ```bash + docker run -p 8000:8000 --shm-size=256GB --gpus=all -e HF_TOKEN -it zmlai/llmd:cuda \ + --model=hf://LiquidAI/LFM3-1.7B-FP8 + ``` + + + #### Prerequisites + + A Linux host with a supported AMD GPU, Docker, and ROCm-compatible drivers. The GPU devices must be available at `/dev/kfd` and `/dev/dri`. + + #### Run + + ```bash + docker run -p 8000:8000 --shm-size=256GB --device=/dev/kfd --device=/dev/dri -e HF_TOKEN -it zmlai/llmd:rocm \ + --model=hf://LiquidAI/LFM3-1.7B-FP8 + ``` + + + #### Prerequisites + + A Linux host with a supported Intel GPU, Docker, and Intel GPU drivers. The GPU device must be available at `/dev/dri`. + + #### Run + + ```bash + docker run -p 8000:8000 --device=/dev/dri -e HF_TOKEN -it zmlai/llmd:oneapi \ + --model=hf://LiquidAI/LFM3-1.7B + ``` + + + #### Prerequisites + + A Google TPU instance with Docker and the TPU runtime configured. Run the command on the TPU host. + + #### Run + + ```bash + docker run --net=host --privileged -e HF_TOKEN -it zmlai/llmd:tpu \ + --model=hf://LiquidAI/LFM3-1.7B + ``` + + + #### Prerequisites + + An Apple Silicon Mac with Homebrew installed. zml/llmd runs directly on macOS with Metal GPU acceleration. + + Install zml/llmd: + + ```bash + brew install zml/zml/llmd + ``` + + #### Run + + ```bash + llmd --model=hf://LiquidAI/LFM3-1.7B-FP8 --batch-size=4 + ``` + + + +zml/llmd loads weights directly from Hugging Face through the `hf://` model path. Once the server is ready, check the loaded model: + +```bash +curl http://localhost:8000/v1/models +``` + +Send your first request with curl to the OpenAI-compatible chat completions endpoint: + +```bash +curl http://localhost:8000/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "LFM3-1.7B-FP8", + "messages": [{"role": "user", "content": "What is the best restaurant in Paris?"}], + "temperature": 0.1 + }' +``` + +## Use local model weights + +If you have already downloaded the model, pass the local directory to `--model` instead of an `hf://` URL. The directory must contain the complete Hugging Face checkpoint, including configuration, tokenizer files, and safetensors weights. + +On Apple Silicon: + +```bash +llmd --model=/absolute/path/to/LFM3-1.7B-FP8 --batch-size=4 +``` + +With Docker, mount the directory into the container and use the container path. For example, on NVIDIA: + +```bash +docker run -p 8000:8000 --shm-size=256GB --gpus=all \ + --mount type=bind,source=/absolute/path/to/LFM3-1.7B-FP8,target=/model,readonly \ + -it zmlai/llmd:cuda --model=/model +``` + +## Model formats + +zml/llmd supports LFM3 text checkpoints in BF16, FP8, and NVFP4. See the [LFM3 model page](/lfm/models/lfm3-1.7b) for model details and prompting guidance. + + + Release notes and platform-specific launch commands for LFM3. + diff --git a/deployment/on-device/llama-cpp.mdx b/deployment/on-device/llama-cpp.mdx index ae640571..36c5d153 100644 --- a/deployment/on-device/llama-cpp.mdx +++ b/deployment/on-device/llama-cpp.mdx @@ -11,7 +11,7 @@ description: "Run Liquid AI LFM models locally with llama.cpp. Setup, GGUF downl Building an application with llama.cpp? Jump to [Build with llama.cpp](#building-applications) for mobile, desktop, chat, tool-calling, structured-output, and multimodal guides. -For GPU-accelerated inference at scale, consider using [vLLM](/deployment/gpu-inference/vllm) instead. +For GPU-accelerated inference at scale, consider using [vLLM](/deployment/gpu-inference/vllm). For LFM3 text serving on Apple Metal, NVIDIA, AMD, Intel, or Google TPU, see [zml/llmd](/deployment/gpu-inference/zml-llmd).
diff --git a/docs.json b/docs.json index f6704b33..afe4ef4e 100644 --- a/docs.json +++ b/docs.json @@ -133,7 +133,8 @@ "pages": [ "deployment/gpu-inference/transformers", "deployment/gpu-inference/sglang", - "deployment/gpu-inference/vllm" + "deployment/gpu-inference/vllm", + "deployment/gpu-inference/zml-llmd" ] }, { diff --git a/examples/agent-harnesses.mdx b/examples/agent-harnesses.mdx index 840e215a..7532ffb6 100644 --- a/examples/agent-harnesses.mdx +++ b/examples/agent-harnesses.mdx @@ -16,7 +16,7 @@ local URL yours prints. You point your harness at that URL. Each backend uses its own default port, so your endpoint depends on which one you run. - llama.cpp and MLX use `8080`, vLLM uses `8000`, SGLang uses `30000`, LM Studio uses `1234`, and Atomic Chat uses `1337`. + llama.cpp and MLX use `8080`, vLLM and zml/llmd use `8000`, SGLang uses `30000`, LM Studio uses `1234`, and Atomic Chat uses `1337`. The examples in this guide use `http://localhost:8080/v1`. When you configure a harness, replace the port with your server's. @@ -45,8 +45,8 @@ vLLM and SGLang run the full-precision weights on GPU. **Need more capability?** [LFM2.5-24B-A2B](/lfm/models/lfm25-24b-a2b) is a 24B-parameter reasoning model with only 2B active parameters per token, post-trained inside Hermes Agent, OpenClaw, Pi, and OpenCode. It thinks before it answers, so it handles longer, multi-step -agent tasks more reliably, at the cost of more memory. Every command in this guide works with it -if you swap the model id. +agent tasks more reliably, at the cost of more memory. The LFM2.5 backend examples below work with it +if you swap the model id; the zml/llmd example uses LFM3-1.7B instead. **Context length**. Agents consume context quickly. If you hit truncation or context-overflow errors mid-run, raise the served context or trim the agent's history. @@ -179,6 +179,34 @@ Install one backend and start it with tool calling enabled. ``` Serves at `http://localhost:30000/v1`. + + **On Mac:** + + ```bash + brew install zml/zml/llmd + llmd --model=hf://LiquidAI/LFM3-1.7B-FP8 --batch-size=4 + ``` + + Serves at `http://localhost:8000/v1`. + + **Other platforms:** + + On Linux, use the Docker image for your accelerator. For example, on NVIDIA with Docker and NVIDIA Container Toolkit configured: + + ```bash + docker run -p 8000:8000 --shm-size=256GB --gpus=all -e HF_TOKEN -it zmlai/llmd:cuda \ + --model=hf://LiquidAI/LFM3-1.7B-FP8 + ``` + + | Platform | Docker image | + | --- | --- | + | NVIDIA CUDA | `zmlai/llmd:cuda` | + | AMD ROCm | `zmlai/llmd:rocm` | + | Intel oneAPI | `zmlai/llmd:oneapi` | + | Google TPU | `zmlai/llmd:tpu` | + + Each platform requires its own device flags, see the [zml/llmd guide](/deployment/gpu-inference/zml-llmd#start-the-server). + Check that the model is loaded and reachable (replace `8080` with your server's port): @@ -191,7 +219,7 @@ curl http://localhost:8080/v1/models Every harness connects the same way: install it, point it at your local server, then run. The examples below use `http://localhost:8080/v1` and model id `LFM2.5-2.6B`. Replace the -port with the one your server prints. Only the exact commands differ per harness. +port with the one your server prints and the model ID with the one returned by `/v1/models`. For the zml/llmd examples above, use `http://localhost:8000/v1` and `LFM3-1.7B-FP8`. Only the exact commands differ per harness. @@ -337,6 +365,7 @@ Now, you have your agent harness running fully locally on your machine. - [llama.cpp deployment](/deployment/on-device/llama-cpp) - [vLLM deployment](/deployment/gpu-inference/vllm) - [SGLang deployment](/deployment/gpu-inference/sglang) +- [zml/llmd deployment](/deployment/gpu-inference/zml-llmd) - [Hermes Agent documentation](https://hermes-agent.nousresearch.com/docs/) - [OpenClaw documentation](https://docs.openclaw.ai/) - [Pi documentation](https://pi.dev/docs/) diff --git a/guides/hardware-evaluation.mdx b/guides/hardware-evaluation.mdx index a068adb4..011614e0 100644 --- a/guides/hardware-evaluation.mdx +++ b/guides/hardware-evaluation.mdx @@ -25,6 +25,7 @@ The LFM2 and LFM2.5 architecture interleaves short convolutions with grouped-que | Transformers | [Transformers](/deployment/gpu-inference/transformers) | GPU baseline, correctness check | | vLLM | [vLLM](/deployment/gpu-inference/vllm) | GPU serving | | SGLang | [SGLang](/deployment/gpu-inference/sglang) | GPU serving | +| zml/llmd | [zml/llmd](/deployment/gpu-inference/zml-llmd) | Apple Silicon, NVIDIA, AMD, Intel, Google TPU, custom silicon | | llama.cpp | [llama.cpp](/deployment/on-device/llama-cpp) | CPU, embedded, custom silicon | | Ollama | [Ollama](/deployment/on-device/ollama) | Local development | | MLX | [MLX](/deployment/on-device/mlx) | Apple Silicon | diff --git a/guides/migration-guide.mdx b/guides/migration-guide.mdx index 3a1b8b5f..12eb53fc 100644 --- a/guides/migration-guide.mdx +++ b/guides/migration-guide.mdx @@ -34,7 +34,7 @@ See the [Complete Model Library](/lfm/models/complete-library) for the full cata Across runtimes, avoid carrying over hand-written Qwen, Llama, or Gemma prompt templates. Use the model-provided chat template, and configure the `lfm2` tool-call parser in vLLM or SGLang when tool use is in scope. -For GPU inference, see [Transformers](/deployment/gpu-inference/transformers), [vLLM](/deployment/gpu-inference/vllm), or [SGLang](/deployment/gpu-inference/sglang). +For GPU inference, see [Transformers](/deployment/gpu-inference/transformers), [vLLM](/deployment/gpu-inference/vllm), [SGLang](/deployment/gpu-inference/sglang), or [zml/llmd](/deployment/gpu-inference/zml-llmd). For edge and on-device inference, see [llama.cpp](/deployment/on-device/llama-cpp), [Ollama](/deployment/on-device/ollama), [Atomic Chat](/deployment/on-device/atomic-chat), [LM Studio](/deployment/on-device/lm-studio), [MLX](/deployment/on-device/mlx), or [ONNX](/deployment/on-device/onnx). To embed a model in a mobile or desktop app, see [Build with llama.cpp](/deployment/on-device/llama-cpp/mobile). diff --git a/guides/use-case-evaluation.mdx b/guides/use-case-evaluation.mdx index 5d608d27..1053fa23 100644 --- a/guides/use-case-evaluation.mdx +++ b/guides/use-case-evaluation.mdx @@ -15,7 +15,7 @@ Use this guide when you want to validate LFMs on an actual product task. The goa ## Integration -LFMs serve through OpenAI-compatible endpoints on [vLLM](/deployment/gpu-inference/vllm) and [SGLang](/deployment/gpu-inference/sglang), load in [Transformers](/deployment/gpu-inference/transformers), and run locally through [llama.cpp](/deployment/on-device/llama-cpp), [Ollama](/deployment/on-device/ollama), [Atomic Chat](/deployment/on-device/atomic-chat), [MLX](/deployment/on-device/mlx), and [LM Studio](/deployment/on-device/lm-studio). +LFMs serve through OpenAI-compatible endpoints on [vLLM](/deployment/gpu-inference/vllm), [SGLang](/deployment/gpu-inference/sglang) and [zml/llmd](/deployment/gpu-inference/zml-llmd), load in [transformers](/deployment/gpu-inference/transformers), and run locally through [llama.cpp](/deployment/on-device/llama-cpp), [ollama](/deployment/on-device/ollama), [atomic chat](/deployment/on-device/atomic-chat), [mlx](/deployment/on-device/mlx), and [lm studio](/deployment/on-device/lm-studio). Read these three pages before judging quality: diff --git a/lfm/help/faqs.mdx b/lfm/help/faqs.mdx index ce245dd7..8a930025 100644 --- a/lfm/help/faqs.mdx +++ b/lfm/help/faqs.mdx @@ -18,6 +18,7 @@ LFM models are compatible with the frameworks below. Support varies by model (fo - [Transformers](/deployment/gpu-inference/transformers) - For research and development - [llama.cpp](/deployment/on-device/llama-cpp) - For efficient CPU inference, including [iOS and Android apps](/deployment/on-device/llama-cpp/mobile) - [vLLM](/deployment/gpu-inference/vllm) - For high-throughput production serving +- [zml/llmd](/deployment/gpu-inference/zml-llmd) - For high-throughput and fast startup - [MLX](/deployment/on-device/mlx) - For Apple Silicon optimization - [Ollama](/deployment/on-device/ollama) - For easy local deployment diff --git a/lfm/models/complete-library.mdx b/lfm/models/complete-library.mdx index fa37002d..9c3701b4 100644 --- a/lfm/models/complete-library.mdx +++ b/lfm/models/complete-library.mdx @@ -8,7 +8,7 @@ description: "Browse every Liquid Foundation Model (LFM) for local AI: text, vis Across the library: - Context length depends on the model: 32K tokens for most text, vision, audio, and omni models, 256K for LFM3-1.7B, 128K for LFM3-VL-2B, LFM2.5-8B-A1B, d1-2B, and d1-omni-2B, 131K for LFM2.5-24B-A2B, 8K for the LFM2.5 encoders, and 16K for d1-omni-600M. Each model page lists its exact limit. -- Text and vision models run with [Transformers](/deployment/gpu-inference/transformers), [llama.cpp](/deployment/on-device/llama-cpp), [vLLM](/deployment/gpu-inference/vllm), [SGLang](/deployment/gpu-inference/sglang), [MLX](/deployment/on-device/mlx), [Ollama](/deployment/on-device/ollama), and [Atomic Chat](/deployment/on-device/atomic-chat). Audio, omni, retrieval, encoder, and decision models support fewer runtimes (d1-omni-600M: Transformers, llama.cpp, and mlx-vlm); check each model page. +- Text and vision models run with [Transformers](/deployment/gpu-inference/transformers), [llama.cpp](/deployment/on-device/llama-cpp), [vLLM](/deployment/gpu-inference/vllm), [SGLang](/deployment/gpu-inference/sglang), [zml/llmd](/deployment/gpu-inference/zml-llmd), [MLX](/deployment/on-device/mlx), [Ollama](/deployment/on-device/ollama), and [Atomic Chat](/deployment/on-device/atomic-chat). Audio, omni, retrieval, encoder, and decision models support fewer runtimes (d1-omni-600M: Transformers, llama.cpp, and mlx-vlm); check each model page. - Most open-weight models can be fine-tuned with [Liquid Finetune](/lfm/fine-tuning/liquid-finetune), [TRL](/lfm/fine-tuning/trl), or [Unsloth](/lfm/fine-tuning/unsloth). The **Trainable?** column in the matrix below lists the documented path for each model.
@@ -57,7 +57,7 @@ Start with the model family that matches your input and output shape, then choos - Use [vLLM](/deployment/gpu-inference/vllm) or [SGLang](/deployment/gpu-inference/sglang) for high-throughput serving, and [Transformers](/deployment/gpu-inference/transformers) for direct Python inference. + Use [vLLM](/deployment/gpu-inference/vllm) or [SGLang](/deployment/gpu-inference/sglang) for high-throughput serving, and [Transformers](/deployment/gpu-inference/transformers) for direct Python inference. For LFM3 text serving across Apple Metal, NVIDIA, AMD, Intel, and Google TPU, use [zml/llmd](/deployment/gpu-inference/zml-llmd). diff --git a/lfm/models/lfm3-1.7b.mdx b/lfm/models/lfm3-1.7b.mdx index 0c24c279..1dbccc3d 100644 --- a/lfm/models/lfm3-1.7b.mdx +++ b/lfm/models/lfm3-1.7b.mdx @@ -97,6 +97,35 @@ LFM3-1.7B is designed for local agents, tool calling, structured outputs, and do + + Run LFM3 with zml/llmd on apple metal, NVIDIA CUDA, AMD ROCm, Intel oneAPI, or Google TPU. On Apple Silicon: + **On Mac:** + + ```bash + brew install zml/zml/llmd + llmd --model=hf://LiquidAI/LFM3-1.7B-FP8 --batch-size=4 + ``` + + Serves at `http://localhost:8000/v1`. + + **Other platforms:** + + On Linux, use the Docker image for your accelerator. For example, on NVIDIA with Docker and NVIDIA Container Toolkit configured: + + ```bash + docker run -p 8000:8000 --shm-size=256GB --gpus=all -e HF_TOKEN -it zmlai/llmd:cuda \ + --model=hf://LiquidAI/LFM3-1.7B-FP8 + ``` + + | Platform | Docker image | + | --- | --- | + | NVIDIA CUDA | `zmlai/llmd:cuda` | + | AMD ROCm | `zmlai/llmd:rocm` | + | Intel oneAPI | `zmlai/llmd:oneapi` | + | Google TPU | `zmlai/llmd:tpu` | + + Each platform requires its own device flags, see the [zml/llmd guide](/deployment/gpu-inference/zml-llmd#start-the-server). + For local agent setup, see the [Agent Harnesses guide](/examples/agent-harnesses). @@ -113,4 +142,4 @@ LFM3-1.7B supports deferred tools, so tool definitions stay out of the prompt un ## FP8 and NVFP4 Checkpoints -[LFM3-1.7B-FP8](https://huggingface.co/LiquidAI/LFM3-1.7B-FP8) and [LFM3-1.7B-NVFP4](https://huggingface.co/LiquidAI/LFM3-1.7B-NVFP4) are FP8 and 4-bit floating-point versions of LFM3-1.7B for GPU inference with SGLang. To use one, replace the model ID in the SGLang example above. +[LFM3-1.7B-FP8](https://huggingface.co/LiquidAI/LFM3-1.7B-FP8) and [LFM3-1.7B-NVFP4](https://huggingface.co/LiquidAI/LFM3-1.7B-NVFP4) are FP8 and 4-bit floating-point versions of LFM3-1.7B for inference with SGLang and [zml/llmd](/deployment/gpu-inference/zml-llmd). To use one with SGLang, replace the model ID in the example above. diff --git a/link-snapshot.yaml b/link-snapshot.yaml index 483af0c3..c60a7aa6 100644 --- a/link-snapshot.yaml +++ b/link-snapshot.yaml @@ -18,6 +18,7 @@ active: - /deployment/gpu-inference/sglang - /deployment/gpu-inference/transformers - /deployment/gpu-inference/vllm + - /deployment/gpu-inference/zml-llmd - /deployment/on-device/android/advanced-features - /deployment/on-device/android/ai-agent-usage-guide - /deployment/on-device/android/android-quick-start-guide From ea4f9cdb41c2fa1213681c9b9e51ac9cf63e08c4 Mon Sep 17 00:00:00 2001 From: Leonie Monigatti Date: Tue, 10 Nov 2026 10:13:21 +0100 Subject: [PATCH 2/4] Revert unrelated changes to Transformers, llama.cpp, and use-case evaluation pages --- deployment/gpu-inference/transformers.mdx | 2 +- deployment/on-device/llama-cpp.mdx | 2 +- guides/use-case-evaluation.mdx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment/gpu-inference/transformers.mdx b/deployment/gpu-inference/transformers.mdx index 499828a1..1610829a 100644 --- a/deployment/gpu-inference/transformers.mdx +++ b/deployment/gpu-inference/transformers.mdx @@ -7,7 +7,7 @@ description: "Run Liquid AI LFM models with Hugging Face Transformers. Load, con Use Transformers for simple inference without extra dependencies, research and experimentation, or integration with the Hugging Face ecosystem. -Transformers provides the most flexibility for model development and is ideal for users who want direct access to model internals. For production deployments with high throughput, consider using [vLLM](/deployment/gpu-inference/vllm). For LFM3 text serving on Apple Metal, NVIDIA, AMD, Intel, or Google TPU, see [zml/llmd](/deployment/gpu-inference/zml-llmd). +Transformers provides the most flexibility for model development and is ideal for users who want direct access to model internals. For production deployments with high throughput, consider using [vLLM](/deployment/gpu-inference/vllm).
diff --git a/deployment/on-device/llama-cpp.mdx b/deployment/on-device/llama-cpp.mdx index 36c5d153..ae640571 100644 --- a/deployment/on-device/llama-cpp.mdx +++ b/deployment/on-device/llama-cpp.mdx @@ -11,7 +11,7 @@ description: "Run Liquid AI LFM models locally with llama.cpp. Setup, GGUF downl Building an application with llama.cpp? Jump to [Build with llama.cpp](#building-applications) for mobile, desktop, chat, tool-calling, structured-output, and multimodal guides. -For GPU-accelerated inference at scale, consider using [vLLM](/deployment/gpu-inference/vllm). For LFM3 text serving on Apple Metal, NVIDIA, AMD, Intel, or Google TPU, see [zml/llmd](/deployment/gpu-inference/zml-llmd). +For GPU-accelerated inference at scale, consider using [vLLM](/deployment/gpu-inference/vllm) instead.
diff --git a/guides/use-case-evaluation.mdx b/guides/use-case-evaluation.mdx index 1053fa23..f928bbda 100644 --- a/guides/use-case-evaluation.mdx +++ b/guides/use-case-evaluation.mdx @@ -15,7 +15,7 @@ Use this guide when you want to validate LFMs on an actual product task. The goa ## Integration -LFMs serve through OpenAI-compatible endpoints on [vLLM](/deployment/gpu-inference/vllm), [SGLang](/deployment/gpu-inference/sglang) and [zml/llmd](/deployment/gpu-inference/zml-llmd), load in [transformers](/deployment/gpu-inference/transformers), and run locally through [llama.cpp](/deployment/on-device/llama-cpp), [ollama](/deployment/on-device/ollama), [atomic chat](/deployment/on-device/atomic-chat), [mlx](/deployment/on-device/mlx), and [lm studio](/deployment/on-device/lm-studio). +LFMs serve through OpenAI-compatible endpoints on [vLLM](/deployment/gpu-inference/vllm), [SGLang](/deployment/gpu-inference/sglang), and [zml/llmd](/deployment/gpu-inference/zml-llmd), load in [Transformers](/deployment/gpu-inference/transformers), and run locally through [llama.cpp](/deployment/on-device/llama-cpp), [Ollama](/deployment/on-device/ollama), [Atomic Chat](/deployment/on-device/atomic-chat), [MLX](/deployment/on-device/mlx), and [LM Studio](/deployment/on-device/lm-studio). Read these three pages before judging quality: From b34db5e5eda56b15f8e8cf76cd922fcc74e56db1 Mon Sep 17 00:00:00 2001 From: Leonie Monigatti Date: Tue, 10 Nov 2026 10:13:21 +0100 Subject: [PATCH 3/4] Fix heading levels and remove external links on zml/llmd page --- deployment/gpu-inference/zml-llmd.mdx | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/deployment/gpu-inference/zml-llmd.mdx b/deployment/gpu-inference/zml-llmd.mdx index 83f6ebb2..0212babd 100644 --- a/deployment/gpu-inference/zml-llmd.mdx +++ b/deployment/gpu-inference/zml-llmd.mdx @@ -19,11 +19,11 @@ Choose your platform to start the server: - #### Prerequisites + ### Prerequisites A Linux host with an NVIDIA GPU, Docker, the NVIDIA driver, and NVIDIA Container Toolkit configured for GPU access. - #### Run + ### Run ```bash docker run -p 8000:8000 --shm-size=256GB --gpus=all -e HF_TOKEN -it zmlai/llmd:cuda \ @@ -31,11 +31,11 @@ Choose your platform to start the server: ``` - #### Prerequisites + ### Prerequisites A Linux host with a supported AMD GPU, Docker, and ROCm-compatible drivers. The GPU devices must be available at `/dev/kfd` and `/dev/dri`. - #### Run + ### Run ```bash docker run -p 8000:8000 --shm-size=256GB --device=/dev/kfd --device=/dev/dri -e HF_TOKEN -it zmlai/llmd:rocm \ @@ -43,11 +43,11 @@ Choose your platform to start the server: ``` - #### Prerequisites + ### Prerequisites A Linux host with a supported Intel GPU, Docker, and Intel GPU drivers. The GPU device must be available at `/dev/dri`. - #### Run + ### Run ```bash docker run -p 8000:8000 --device=/dev/dri -e HF_TOKEN -it zmlai/llmd:oneapi \ @@ -55,11 +55,11 @@ Choose your platform to start the server: ``` - #### Prerequisites + ### Prerequisites A Google TPU instance with Docker and the TPU runtime configured. Run the command on the TPU host. - #### Run + ### Run ```bash docker run --net=host --privileged -e HF_TOKEN -it zmlai/llmd:tpu \ @@ -67,7 +67,7 @@ Choose your platform to start the server: ``` - #### Prerequisites + ### Prerequisites An Apple Silicon Mac with Homebrew installed. zml/llmd runs directly on macOS with Metal GPU acceleration. @@ -77,7 +77,7 @@ Choose your platform to start the server: brew install zml/zml/llmd ``` - #### Run + ### Run ```bash llmd --model=hf://LiquidAI/LFM3-1.7B-FP8 --batch-size=4 @@ -123,8 +123,4 @@ docker run -p 8000:8000 --shm-size=256GB --gpus=all \ ## Model formats -zml/llmd supports LFM3 text checkpoints in BF16, FP8, and NVFP4. See the [LFM3 model page](/lfm/models/lfm3-1.7b) for model details and prompting guidance. - - - Release notes and platform-specific launch commands for LFM3. - +zml/llmd supports LFM3 text checkpoints in BF16, FP8, and NVFP4. From a589922e824a8c1fd2ee409e9a5ccb1e18ce27a7 Mon Sep 17 00:00:00 2001 From: Leonie Monigatti Date: Tue, 10 Nov 2026 10:13:21 +0100 Subject: [PATCH 4/4] Use LFM3-1.7B throughout the agent harnesses guide --- examples/agent-harnesses.mdx | 75 ++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/examples/agent-harnesses.mdx b/examples/agent-harnesses.mdx index 7532ffb6..c8ccdfda 100644 --- a/examples/agent-harnesses.mdx +++ b/examples/agent-harnesses.mdx @@ -23,34 +23,33 @@ local URL yours prints. You point your harness at that URL. ### Model configuration -[LFM2.5-2.6B](/lfm/models/lfm25-2.6b) is a dense 2.6B-parameter model built for on-device deployment. It runs fast on -consumer hardware and supports tool calling, which makes it a good fit for agentic workloads. +[LFM3-1.7B](/lfm/models/lfm3-1.7b) is a 1.7B-parameter text model post-trained for agent harnesses such as Hermes Agent, +OpenClaw, and Pi. It runs fast on consumer hardware and supports native tool calling, which makes it a good fit for agentic workloads. The two settings worth choosing up front are the quantization and the context length. Both trade memory for quality or capacity, so pick them to fit your hardware. -**Quantization.** Because LFM2.5-2.6B is small, you have room to trade size for quality. +**Quantization.** Because LFM3-1.7B is small, you have room to trade size for quality. For the GGUF quants, which cover llama.cpp and LM Studio, we recommend starting with `Q4_K_M` and stepping up to `Q8_0` or `BF16` depending on your available memory. | Quant | Size | Notes | | -------- | ------- | -------------------------------------------------------- | -| `Q4_K_M` | 1.67 GB | Best balance of size and quality (recommended) | -| `Q6_K` | 2.22 GB | Better quality | -| `Q8_0` | 2.87 GB | Near-lossless and a safe choice for tool-heavy agentic work | -| `BF16` | 5.4 GB | Full precision for maximum fidelity and benchmarking | +| `Q4_K_M` | 1.1 GB | Best balance of size and quality (recommended) | +| `Q6_K` | 1.43 GB | Better quality | +| `Q8_0` | 1.85 GB | Near-lossless and a safe choice for tool-heavy agentic work | +| `BF16` | 3.47 GB | Full precision for maximum fidelity and benchmarking | -MLX uses its own quantization. Pick the 4-bit, 6-bit, 8-bit, or bf16 build from the MLX repo. -vLLM and SGLang run the full-precision weights on GPU. +MLX uses its own quantization. Pick the [4-bit](https://huggingface.co/LiquidAI/LFM3-1.7B-MLX-4bit) or [bf16](https://huggingface.co/LiquidAI/LFM3-1.7B-MLX-bf16) build. +vLLM and SGLang run the full-precision weights on GPU, and zml/llmd runs the [FP8](https://huggingface.co/LiquidAI/LFM3-1.7B-FP8) weights. **Need more capability?** [LFM2.5-24B-A2B](/lfm/models/lfm25-24b-a2b) is a 24B-parameter reasoning model with only 2B active parameters per token, post-trained inside Hermes Agent, -OpenClaw, Pi, and OpenCode. It thinks before it answers, so it handles longer, multi-step -agent tasks more reliably, at the cost of more memory. The LFM2.5 backend examples below work with it -if you swap the model id; the zml/llmd example uses LFM3-1.7B instead. +OpenClaw, Pi, and OpenCode. It handles longer, multi-step agent tasks and agentic coding more +reliably, at the cost of more memory. The backend examples below, except zml/llmd, work with it if you swap the model ID. **Context length**. Agents consume context quickly. If you hit truncation or context-overflow errors mid-run, raise the served context or trim the agent's history. -LFM2.5-2.6B supports up to 128K tokens. The examples serve the full window, but if you're +LFM3-1.7B supports up to 256K tokens. The examples serve the full window, but if you're memory-constrained, serve a smaller window such as 32K tokens, which is usually plenty for a single agent task. @@ -66,17 +65,17 @@ Install one backend and start it with tool calling enabled. brew install llama.cpp # macOS winget install llama.cpp # Windows ``` - For Linux and build-from-source options, see the [llama.cpp guide](/deployment/on-device/llama-cpp). + LFM3 requires llama.cpp [b11960](https://github.com/ggml-org/llama.cpp/releases/tag/b11960) or newer. For Linux and build-from-source options, see the [llama.cpp guide](/deployment/on-device/llama-cpp). **Run:** The `-hf` flag auto-downloads the GGUF. ```bash - llama-server -hf LiquidAI/LFM2.5-2.6B-GGUF:Q4_K_M \ + llama-server -hf LiquidAI/LFM3-1.7B-GGUF:Q4_K_M \ --jinja \ --port 8080 \ - -c 131072 \ + -c 262144 \ -fa on \ -ngl 99 \ --temp 0.1 \ @@ -87,14 +86,14 @@ Install one backend and start it with tool calling enabled. | Flag | Meaning | | ---------- | ------------------------------------------------- | | `--jinja` | **Enables tool calling** via the model's template | - | `-c 131072` | Context window (128K) | + | `-c 262144` | Context window (256K) | | `-fa on` | Flash attention (needs a Metal or CUDA build) | | `-ngl 99` | Offload all layers to GPU | **Install:** - Download and install [LM Studio](https://lmstudio.ai), then search for **LFM2.5-2.6B** in + Download and install [LM Studio](https://lmstudio.ai) with llama.cpp engine 2.65.1 or newer, then search for **LFM3-1.7B** in the model catalog and download the `Q4_K_M` GGUF. See the [LM Studio guide](/deployment/on-device/lm-studio). @@ -102,7 +101,7 @@ Install one backend and start it with tool calling enabled. Open the **Developer / Local Server** tab, then: - 1. Load the **LFM2.5-2.6B** model. + 1. Load the **LFM3-1.7B** model. 2. Enable **tool use** in the model settings. 3. Set the context length in the model settings. 4. Click **Start Server**. It serves at `http://localhost:1234`. @@ -110,13 +109,13 @@ Install one backend and start it with tool calling enabled. **Install:** - Download and install [Atomic Chat](https://atomic.chat), then search for **LFM2.5-2.6B** in + Download and install [Atomic Chat](https://atomic.chat), then search for **LFM3-1.7B** in the model library and download the `Q4_K_M` GGUF. See the [Atomic Chat guide](/deployment/on-device/atomic-chat). **Run:** - 1. Load **LFM2.5-2.6B** in a chat. + 1. Load **LFM3-1.7B** in a chat. 2. Open the **Integrations** tab. 3. Click **Start Server**. It serves the loaded model at `http://localhost:1337` by default; use the port shown in the app. @@ -135,7 +134,7 @@ Install one backend and start it with tool calling enabled. `mlx_lm.server` exposes an OpenAI-compatible endpoint: ```bash - mlx_lm.server --model LiquidAI/LFM2.5-2.6B-MLX --port 8080 + mlx_lm.server --model LiquidAI/LFM3-1.7B-MLX-4bit --port 8080 ``` Confirm your `mlx-lm` version forwards tools to the chat template. @@ -152,9 +151,10 @@ Install one backend and start it with tool calling enabled. Tool calling requires explicit flags: ```bash - vllm serve LiquidAI/LFM2.5-2.6B \ + vllm serve LiquidAI/LFM3-1.7B \ + --reasoning-parser qwen3 \ --enable-auto-tool-choice \ - --tool-call-parser lfm2 + --tool-call-parser qwen3_coder ``` Serves at `http://localhost:8000/v1`. @@ -172,10 +172,11 @@ Install one backend and start it with tool calling enabled. ```bash sglang serve \ - --model-path LiquidAI/LFM2.5-2.6B \ + --model-path LiquidAI/LFM3-1.7B \ --host 0.0.0.0 \ --port 30000 \ - --tool-call-parser lfm2 + --reasoning-parser qwen3-thinking \ + --tool-call-parser qwen3_coder ``` Serves at `http://localhost:30000/v1`. @@ -218,8 +219,8 @@ curl http://localhost:8080/v1/models ## Connect your agent harness Every harness connects the same way: install it, point it at your local server, then run. -The examples below use `http://localhost:8080/v1` and model id `LFM2.5-2.6B`. Replace the -port with the one your server prints and the model ID with the one returned by `/v1/models`. For the zml/llmd examples above, use `http://localhost:8000/v1` and `LFM3-1.7B-FP8`. Only the exact commands differ per harness. +The examples below use `http://localhost:8080/v1` and model ID `LFM3-1.7B`. Replace the +port with the one your server prints and the model ID with the one returned by `/v1/models`. For zml/llmd, use `http://localhost:8000/v1` and `LFM3-1.7B-FP8`. Only the exact commands differ per harness. @@ -241,7 +242,7 @@ port with the one your server prints and the model ID with the one returned by ` # choose "Custom endpoint (self-hosted / vLLM / etc.)" # API base URL: http://localhost:8080/v1 # API key: (leave empty for local) - # Model name: LFM2.5-2.6B + # Model name: LFM3-1.7B ``` Or set it directly, then **enable tool-use enforcement** (without it, the model tends to @@ -250,8 +251,8 @@ port with the one your server prints and the model ID with the one returned by ` ```bash hermes config set model.provider custom hermes config set model.base_url http://localhost:8080/v1 - hermes config set model.default LFM2.5-2.6B - hermes config set model.context_length 131072 + hermes config set model.default LFM3-1.7B + hermes config set model.context_length 262144 hermes config set model.api_mode chat_completions hermes config set agent.tool_use_enforcement true ``` @@ -290,10 +291,10 @@ port with the one your server prints and the model ID with the one returned by ` api: "openai-completions", models: [ { - id: "LFM2.5-2.6B", - name: "LFM2.5-2.6B", + id: "LFM3-1.7B", + name: "LFM3-1.7B", input: ["text"], - contextWindow: 131072, + contextWindow: 262144, maxTokens: 8192, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, }, @@ -307,7 +308,7 @@ port with the one your server prints and the model ID with the one returned by ` Select it as the active model: ```json5 - { agents: { defaults: { model: { primary: "local/LFM2.5-2.6B" } } } } + { agents: { defaults: { model: { primary: "local/LFM3-1.7B" } } } } ``` **Run:** @@ -339,7 +340,7 @@ port with the one your server prints and the model ID with the one returned by ` "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "local", - "models": [{ "id": "LFM2.5-2.6B" }] + "models": [{ "id": "LFM3-1.7B" }] } } } @@ -361,7 +362,7 @@ Now, you have your agent harness running fully locally on your machine. ## References -- [LFM2.5-2.6B](/lfm/models/lfm25-2.6b) +- [LFM3-1.7B](/lfm/models/lfm3-1.7b) - [llama.cpp deployment](/deployment/on-device/llama-cpp) - [vLLM deployment](/deployment/gpu-inference/vllm) - [SGLang deployment](/deployment/gpu-inference/sglang)