diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 2b00d1abaa..caca8907f2 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -37,7 +37,7 @@ jobs: id: get_version shell: bash run: | - VERSION=$(python -c "import llama_cpp; print(llama_cpp.__version__)") + VERSION=$(python -c "import importlib.metadata; print(importlib.metadata.version('llama-cpp-python'))") echo "Detected version: $VERSION" echo "version=$VERSION" >> $GITHUB_OUTPUT diff --git a/.gitignore b/.gitignore index fad7f43313..b5d60bf894 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,15 @@ local_settings.py models/ docker/open_llama/*.bin +# Repository-only ABI tool inputs and generated reports. +# Keep only the directory instructions and local ignore rules tracked. +/tools/abi/artifacts/* +!/tools/abi/artifacts/.gitignore +!/tools/abi/artifacts/README.md +/tools/abi/output/* +!/tools/abi/output/.gitignore +!/tools/abi/output/README.md + # C extensions (llama_cpp bindings) llama_cpp/*.so llama_cpp/*.dylib @@ -208,4 +217,4 @@ docs/_build/ # Installer logs pip-log.txt -pip-delete-this-directory.txt \ No newline at end of file +pip-delete-this-directory.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 1865195db3..075b978847 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,477 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [0.3.45] Reactivated Built-in Embeddings, Modern Model Loading, and Stronger Cross-Platform Reliability + +- fix(ctypes): correct llama-ext binding signatures + - use uint32_t for layer IDs + - fix void return type for embedding extraction control + - return target layer count as uint32_t + +- feat(llama): expose additional model loading options + - add `no_alloc` and `load_mtp` parameters + - enable `extra buffer types` by default + +- feat(llama): support llama_model_params `load_mode` + - Update model loading configuration to use the new `load_mode` field from + llama_model_params and align with the latest llama.cpp API changes. + - Remove deprecated internal handling of legacy loading flags and keep + backward compatibility by warning users when `use_mmap`, `use_direct_io`, + or `use_mlock` are still used. + - This prepares the Python bindings for the updated llama.cpp model loading + interface while providing a smoother migration path for existing users. + - docs: document `load_mode` migration + - Replace references to the legacy model loading flags with load_mode, document all supported loading modes for the Python API and server, and update the performance tuning example. + +- feat(tools): add cross-platform ABI inspection utility + - Inspect `PE`, `ELF`, and `Mach-O` exports and normalize platform-specific symbol names. + - Validate optional `llama_ext` ctypes aliases across Windows, Linux, and macOS builds. Keep artifacts and timestamped privacy-safe reports local to the repository. + - More information see here: [Cross-platform ABI inspection](https://github.com/JamePeng/llama-cpp-python/tree/main/tools/abi) + +- fix(ctypes): support GCC/Clang mangled symbols for optional llama_ext APIs + - Add missing `_Z` Itanium C++ ABI symbol variants to ctypes function + lookup lists. This improves compatibility with Linux and macOS builds + where C++ symbols are exported using GCC/Clang name mangling. + - Issue report from **@ckcfcc** (https://github.com/JamePeng/llama-cpp-python/issues/159) + +- fix(loader): guard `HIP_PATH` and `VULKAN_SDK` dirs with os.path.exists +os.add_dll_directory() raises FileNotFoundError [WinError 3] when the +directory does not exist, so a stale `HIP_PATH` or `VULKAN_SDK` left behind by +an uninstalled SDK makes "import llama_cpp" fail outright on Windows.(by **@emptyngton**) + + The CUDA_PATH branch above already guards each candidate directory with + os.path.exists(); this applies the same pattern to the HIP and Vulkan + branches. Valid directories are still added individually, so a partially + removed SDK contributes whichever of bin/lib remain instead of raising. + +- fix(_internals): clean up native resources on initialization failures + - Register native model and batch ownership immediately after allocation + so later validation failures cannot leak llama.cpp resources. + Free a loaded model when vocab lookup fails, and route mixed-batch setup + failures through idempotent cleanup. + - Initialize sampling-context resource fields before fallible setup and + make partial teardown safe to repeat. This prevents missing attributes + from interrupting cleanup when sampler-chain construction fails. + - Clear model, vocabulary, and sampling parameter references after native + context and sampler resources have been released. This prevents closed + wrapper objects from unnecessarily keeping models and related Python + objects alive. + - Add failure-injection tests that verify model and batch handles are freed + exactly once and partially initialized sampling contexts release their resources + idempotently.Extend lifecycle tests to verify that parent references are cleared + and that repeated close calls remain safe. + +- test(chat-format): modernize coverage with Qwen3.5-style templates + - Replace the legacy Mistral-focused chat format tests with self-contained + Qwen3.5-style Jinja template coverage: + - verify ChatML system, user, and assistant message rendering + - cover enabled and disabled thinking generation prompts + - test image and video placeholders with vision identifiers + - validate tool definitions, tool calls, and tool response history + - add clear error coverage for invalid message structures + - verify model-specific stop token criteria + - keep the tests independent of tokenizer files and model weights + +- docs(readme): replace the new logo with fork project branding + - Add the new llama-cpp-python logo asset under docs and update the README + header to reference the repository-local image. + - the new logo which combined llama, C++, and Project branding remains readable. + +- docs(embedding): add end-to-end embeddings and reranking guide + - Create a schema-compliant feature guide covering sentence embeddings, + token-level vectors, reranking workflows, pooling modes, normalization, + streaming batch configuration, return shapes, and output formats. + - Add complete examples for the standard Llama API, LlamaEmbedding, + pre-tokenized inputs, cosine-similarity output, and cross-encoder + reranking. + - Document common configuration problems, implementation limitations, and + the embedding and reranking model families currently listed as supported + by the project. + - Expose the new feature guide through the Wiki index. + +- docs(llama): expand embedding parameters and API guidance + - Add a role overview and reorganize constructor options into focused, + readable parameter groups. + - Document embedding, pooling, attention, KV cache, sequence capacity, and + recurrent-state settings with their defaults and runtime behavior. + - Expand the embed() and create_embedding() sections with normalization + modes, return shapes, batching semantics, pooling recommendations, + OpenAI compatibility notes, and resource-safe examples. + - Fix the YAML frontmatter and improve Markdown spacing for cleaner Wiki + rendering. + +- docs(embedding): document maintained APIs and sequence batch capacity + - Replace the deprecated Llama embedding guidance with current embed() and + create_embedding() usage. + - Document the roles of n_batch, n_ubatch, and n_seq_max, including + parallel batching examples, resource considerations, common sequence ID + errors, and the required configuration changes. + - Clarify that LlamaEmbedding remains a convenience interface for + embedding-oriented defaults and reranking workflows. + +- docs(example): refresh the built-in embedding usage example + - Fix the Llama constructor option from embedding=True to embeddings=True + and demonstrate L2-normalized output through create_embedding(). + +- test(embedding): cover built-in and streaming embedding workflows + - Add coverage for actionable LlamaBatch sequence-capacity errors and the + maintained embedding APIs on the standard Llama class. + - Verify pre-tokenized batches, normalization, separator-based inputs, + token accounting, OpenAI-compatible responses, and LlamaEmbedding + streaming behavior with n_seq_max=1. + - Explicitly close embedding models after integration tests to release + native context and model resources. + +- fix(embedding): respect n_seq_max when streaming embedding batches + - Use the configured sequence capacity instead of n_ubatch when deciding + when to decode the current LlamaEmbedding batch. + - This prevents invalid sequence IDs for multi-document inputs and allows + the default n_seq_max=1 configuration to process documents sequentially + without failing. + +- refactor(batch): improve sequence capacity validation guidance + - Make LlamaBatch sequence validation errors explain the configured + n_seq_max value, valid sequence ID range, and minimum capacity required + for parallel batching. + - Handle negative sequence IDs separately and provide actionable setup + guidance for Llama, LlamaEmbedding, and direct LlamaBatch users. + - Remove the unused normalize_embedding helper now that normalization is + handled by the embedding pipeline. + +- feat(embedding): modernize the built-in Llama embedding API + - Replace the legacy embedding path with sequence-aware streaming batch + processing based on the current LlamaBatch interface. + - Support string, batched string, and pre-tokenized inputs, token-level and + rank pooling outputs, separator-based splitting, token accounting, and + llama.cpp-compatible normalization modes. + - Restore embed() and create_embedding() as maintained Llama APIs while + preserving the existing boolean normalization behavior. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/876a4321163249c43ca4e986818fab5ab081f282](https://github.com/ggml-org/llama.cpp/commit/876a4321163249c43ca4e986818fab5ab081f282) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260801 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/ebf6099b81cf67cfb5eec569466367c9fa04e9d4...aafc6fb74ebfba6a044510f80b5e9ad277109c12 + +## [0.3.44] Improved Windows DLL(OpenMP) Loading Reliability for GGML Backends + +- fix(ggml): preload bundled OpenMP runtime before loading ggml-base + - Preload the packaged `libomp140.x86_64.dll` on Windows before initializing + ggml-base to ensure CPU backend DLLs can resolve their OpenMP runtime + dependency. + - This only applies to Windows builds with llama-cpp-python >= 0.3.39 and + uses the bundled runtime from the package lib directory, avoiding the need + for users to configure system PATH or install additional OpenMP runtimes. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/846e991ec3c7ccec49112ff2c5b00b710e5f551d](https://github.com/ggml-org/llama.cpp/commit/846e991ec3c7ccec49112ff2c5b00b710e5f551d) + +## [0.3.43] Better llama.cpp ABI Compatibility, MTMD Performance and Extension API Support + +- patch(Gemma4ChatHandler): Synchronize huggingface gemma4 latest chat template + - fix: chat template — null handling, reasoning preservation, turn-tag balance, input validation + - https://huggingface.co/google/gemma-4-31B-it/commit/68abe48010cbe15293462fa11e901a60639a44e5 + +- feat(llama_ext): support optional llama-ext.h API bindings + - Add Python ctypes bindings for the experimental APIs exposed by + llama-ext.h, including NextN/MTP embeddings, and model metadata extraction. + - Extension symbols are loaded optionally to handle ABI changes, renamed + symbols, and builds that do not export experimental APIs without breaking + the main Python bindings. + +- feat(ctypes): handle missing optional symbols gracefully + - Allow ctypes bindings to mark symbols as optional through the `required` + flag. + - Missing symbols caused by ABI naming differences, API changes, or experimental + extensions will no longer break library loading. Optional APIs emit diagnostic + warnings and provide runtime unavailable stubs instead. + +- fix(ctypes): validate argument types before binding shared library functions + - Add explicit validation for ctypes function argument declarations before + assigning them to the loaded shared library function. + - This provides clearer error messages when invalid Python types are passed + to `argtypes`, instead of exposing the internal ctypes error about missing + `from_param()` methods. + +- fix(ctypes): validate argument types before binding shared library functions + - Add explicit validation for ctypes function argument declarations before + assigning them to the loaded shared library function. + - This provides clearer error messages when invalid Python types are passed + to `argtypes`, instead of exposing the internal ctypes error about missing + `from_param()` methods. + +- feat(ctypes): support ABI-compatible symbol aliases + - Allow ctypes_function_for_shared_library to accept either a single + symbol name or an ordered iterable of ABI-compatible aliases. + - Resolve aliases in order and bind the first exported symbol found while + preserving the selected symbol name for runtime diagnostics. Also improve + error reporting for empty alias lists and missing symbols. + +- refactor(mtmd): cache Generic MTMD chat template resolution for accelerate the processing speed of `__call__`. + - Refactor MTMD chat template handling to resolve and analyze the chat template only + once per handler instance instead of on every request. + - Add template initialization state, cache parsed media placeholder tags, and support + explicit chat template overrides through a dedicated field. Improve lifecycle cleanup + by resetting cached template state and MTMD resources during handler close. + - This keeps `GenericMTMDChatHandler` runtime processing focused on message rendering and media tokenization + while avoiding repeated chat template resolution overhead. + +- fix(mtmd): preserve subclass chat format during MTMD initialization + - Ensure MTMDChatHandler initialization remains compatible with specialized chat + handlers that define their own chat_format before calling super().__init__(). + - Initialize chat_format only when it is not already provided by the subclass, + then apply chat_format_override or fallback to the built-in MTMD template. + This prevents AttributeError during inherited handler initialization while + keeping template override behavior unchanged. + +- refactor(embedding): rename `llama_cpp` import alias to `llama_cpp_lib` + - Rename the `llama_cpp.llama_cpp` import alias to `llama_cpp_lib` to avoid potential namespace conflicts with the local `.llama_cpp` imports. Update all affected call sites in `llama_embedding.py`. + +- patch(Llama): Increase chunk preview limit to 128 in Llama.eval exception + - Raises the maximum tokens captured for the error message preview from 16 to 128, improving visibility into the offending chunk during fatal backend crashes. + +- ci(metal): get package version from importlib metadata + * Avoid importing llama_cpp when detecting the package version. + * This prevents initialization side effects and keeps CI version extraction reliable. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/86d86ed4396b4130922f7b9af26e3d9fc11a591b](https://github.com/ggml-org/llama.cpp/commit/86d86ed4396b4130922f7b9af26e3d9fc11a591b) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260716 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/e522cecb93907c67ffe2e339b7009c93d3fb0f59...a64128351a1d04c6dd644e3908070f7ea2002f20 + +## [0.3.42] More Reliable Dynamic Backend Loading, Safer MTMD Processing, and Advanced Batch Support + +- fix(loader): improve Windows DLL search path handling and diagnostics + - Remove duplicated Windows DLL directory registration logic + - Add optional CUDA, HIP, and Vulkan runtime DLL search paths + - Keep bundled library paths with correct priority order for loading, need `/lib` > `/bin` + - Add comments explaining DLL search path ordering behavior + - Add load source diagnostics for system and bundled libraries + - Improve visibility when debugging shared library loading issues + + **Note**: + * For most single-DLL backends, the bin directory can still work as a fallback search path. However, some cases may fail due to missing dependencies such as `libomp140.x86_64.dll`. + * For `multi-DLL backends`, such as the `SYCL backend`, which depends on multiple DLLs (`dnnl.dll`, `tbb12.dll`, `mk_*.dll`, etc.), loading ggml-sycl.dll may fail when its dependent DLLs cannot be found, potentially resulting in an `access violation` crash. + * This update ensures that the DLL search path prioritizes /lib instead of /bin during the initial lookup stage, improving backend loading reliability. + * Special thanks to **@allanmeng** for reporting and testing the SYCL backend issue. + +- fix(ggml): load ggml-base before ggml library + - Load ggml-base shared library before ggml to ensure the base + runtime dependency is initialized prior to loading the main ggml + library. + + - This improves dynamic library loading reliability on platforms + where ggml depends on ggml-base during initialization. + +- fix(mtmd): validate MTMD inputs before tokenization + - Add Python-side MTMD input validation before calling the native mtmd_tokenize + path. Normalize missing bitmap lists to empty lists for pure text prompts, check + that rendered media markers match decoded bitmap inputs, reject missing bitmap + entries, and validate that the media marker is available. + + - Improve media placeholder mismatch errors with marker counts and marker details, + and surface mtmd_tokenize failures with richer diagnostic context including media + counts and backend support flags. + +- feat(LlamaBatch): add mixed token embedding batch support + - Add optional mixed=True initialization for LlamaBatch so token+embedding rows can + be represented in a single llama_batch. Mixed batches keep the native embd buffer + from llama_batch_init and attach a Python-owned token buffer, which is cleared + before llama_batch_free() to avoid invalid ownership. + + - Route token-only and embedding-only write APIs away from mixed batches, add + mixed-batch validation, and introduce add_token_embedding for EAGLE3/MTP-style + decoder inputs containing both token ids and embedding vectors. + + - This prepares LlamaBatch for speculative decoding paths that require mixed + token+hidden-state inputs, especially EAGLE3 and MTP. It keeps ordinary + token-only and embedding-only APIs separated while providing a dedicated + add_token_embedding path for mixed decoder rows. + +- feat(LlamaBatch): add embedding rows to LlamaBatch + - Add shared seq_id validation for token and embedding batch writes. + + - Introduce embedding-buffer checks plus add_embedding and add_embeddings helpers + for embd-only llama_batch inputs, enabling decoder paths that consume external + embedding rows while keeping token writes restricted to token buffers. + + - This prepares LlamaBatch for embedding-only decode paths, such as speculative + decoding feature injection or external encoder/projector outputs. + + - It does not implement mixed token+embedding batches yet; those still need a + separate ownership-safe design for the token buffer. + +- fix(LlamaBatch): harden LlamaBatch token writes + - Clarify llama_batch token vs embedding allocation semantics and keep future + embedding/mixed-batch support open. + + - Add token-buffer checks before add_token/add_sequence, validate add_sequence + input lengths and seq_ids, and improve error messages for invalid batch + configuration. + +- fix(eval): validate eval tokens before native decode + - Add token-id validation at the Llama.eval() boundary before context shifting, + batch construction, or llama_decode execution. This prevents invalid token + types, negative token ids, and out-of-vocabulary ids from reaching the native + decode path, where they may otherwise cause hard crashes instead of Python + exceptions. + + - Wrap llama_decode with defensive exception handling in LlamaContext.decode() so + native exceptions are surfaced with clearer diagnostic context. + + - Also include a small token preview in Llama.eval() fatal decode errors to make + backend failures easier to debug without changing the existing recoverable KV + slot handling behavior. + +- fix(types): make assistant message name optional + - Mark the assistant message `name` field as `NotRequired[Optional[str]]` + to match the optional nature of assistant message metadata and avoid + requiring callers to provide `name` in typed chat completion requests. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/e3546c7948e3af463d0b401e6421d5a4c2faf565](https://github.com/ggml-org/llama.cpp/commit/e3546c7948e3af463d0b401e6421d5a4c2faf565) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260711 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/169d5e1a43fb6ff4e5b6f5d0f26f1ec8acbd97b8...3da4c603612c3344031b32ffbeb1da1c84bb205a + +## [0.3.41] Template-Driven MTMD, Broader Multimodal Inputs, and Smarter N-Gram Drafting + +- refactor(mtmd): extract prompt rendering and media marker normalization + - Add extra_template_arguments to MTMD chat handlers and pass them through to the Jinja chat template render call. This allows generic model templates to receive render-time options such as enable_thinking, add_vision_id, or model-specific template jinja variables. + - Extract MTMD prompt rendering into dedicated helpers: + * _render_mtmd_prompt() for pure chat template rendering + * _replace_media_placeholders() for normalizing rendered media tags and URLs into the MTMD runtime marker + * _render_and_replace_media() for the combined render-and-normalize stage + - This removes inline render/replace logic from _process_mtmd_prompt(), keeps media marker validation after normalization, and improves separation between prompt construction and MTMD tokenization. + +- docs(README): add GenericMTMDChatHandler usage guide + - Replace the legacy Llava multimodal loading example with a GenericMTMDChatHandler + usage guide for template-driven multimodal GGUF models. + - Document loading mmproj through Llama, chat template resolution order, + extra_template_arguments for model-specific Jinja variables, and when to prefer + a dedicated multimodal chat handler. + - Also clarify the mmproj_path naming, llama_multimodal migration, and note that + the generic handler is intended as a flexible fallback for models without + dedicated handlers and may require additional testing for model-specific + prompting behavior. + - Update Generic MTMD Chat Handler directory index. + +- refactor(mtmd): extract mtmd_tokenize into _mtmd_tokenize standalone helper + - Introduce `_mtmd_tokenize()` to encapsulate llama.cpp mtmd_tokenize binding + - Decouple hybrid tokenization logic from `_process_mtmd_prompt` + - Improve separation of concerns between prompt construction and C++ binding + - Preserve strict media marker validation to ensure token/bitmap alignment + +- feat(speculative): Improve ngram-map draft selection and accept feedback + - Store accepted draft lengths per key/value and truncate future drafts accordingly + - Make key-only mode draft on any key match without applying min_hits + - Select k4v continuations by frequency instead of latest occurrence + - Skip ambiguous k4v drafts when the top continuation is not dominant + - Track fixed-size k4v continuations to keep frequency statistics comparable + +- feat(mtmd): broaden multimodal media extraction + - Broaden MTMD media extraction to support common multimodal content shapes used + by model chat templates. + - In addition to OpenAI-style image_url/audio_url/video_url chunks, accept + image/audio/video typed chunks and direct media keys such as {"image": "..."}, + {"audio": "..."}, or {"video": "..."}. This keeps the extracted media list + aligned with templates that emit placeholders for image, audio, or video content + without requiring URL-specific chunk names. + - Add a shared helper for extracting URLs, local paths, existing data URIs, or + inline base64 payloads from media content items. Preserve capability checks, + strict input_audio format validation, and explicit errors for missing or + ambiguous media payloads. + +- feat(mtmd): enhance generic chat template support + - Enhance GenericMTMDChatHandler to better support model-provided chat templates. + - Allow the generic handler to accept an optional named chat template, load it + from the model at call time via llama_model_chat_template(), fall back to the + model's default chat template, and finally use the built-in MTMD CHAT_FORMAT + when no model template is available. + - Also expand the generic media placeholder list for common multimodal templates + and document the handler as a template-driven MTMD implementation. This prepares + the generic path for a later render-driven placeholder replacement pass. + +- fix(model): handle missing chat templates + - Update `LlamaModel.model_chat_template()` to return Optional[str] and accept + name=None for the default model chat template. + - `llama_model_chat_template()` may return nullptr when no chat template is + available. Handle that case explicitly instead of decoding a null pointer, and + return None so callers can apply their own fallback logic. + +- fix(vocab): update `LlamaModel.vocab_type` to use self.vocab and add None checks + +- refactor(mtmd): move multimodal handlers to separate module `llama_multimodal` + - Move `MTMDChatHandler`, `GenericMTMDChatHandler``, and model-specific multimodal + chat handlers out of `llama_chat_format.py` into `llama_multimodal.py`. + - `llama_chat_format.py` has grown too large and difficult to maintain, especially + as MTMD support expands beyond image-only use cases. Splitting multimodal + handling into its own module makes the chat formatting layer smaller and keeps + media loading, MTMD tokenization, multimodal KV-cache bookkeeping, and handler + implementations in a dedicated place. + - This also prepares the codebase for broader multimodal support and future video + frame / image batch evaluation, where the media-processing path will need to + evolve independently from text-only chat formatting. + - Keep backward-compatible re-exports from `llama_chat_format.py` so existing + imports continue to work. + - Also keep `clip_model_path` as a deprecated initialization alias for + `mmproj_path` in the base MTMD handler. + - docs: update mtmd chat handler import paths in README + - Update import statements for multi-modal chat handlers from llama_cpp.llama_chat_format to llama_cpp.llama_multimodal in the documentation examples. + +- feat: Implemented generic multimodal chat handler prototype (by **@alcoftTAO**) + +- docs(README): Added command prompt scenario for README.md (by **@patrikpatrik**) + - Updated command prompt scenario under Configuration -> Environment Variables + - Sanity checking after successful installation of wheel + +- feat(MTMDChatHandler): add chunk type helpers + - Add small helper methods `_is_text_chunk`/`_is_image_chunk`/`_is_audio_chunk` for checking + MTMD text, image, and audio chunk type enum values. + - This keeps MTMD prompt processing easier to read and avoids repeating direct + enum comparisons when building token spans for text and media chunks. + +- feat(mtmd): add video input support to `MTMDChatHandler` + - Add video_url handling to the MTMD chat template and media extraction + pipeline. Detect whether the loaded libmtmd build supports video helpers + and reject video inputs early when MTMD_VIDEO is unavailable. + - Update media loading and bitmap creation for the new helper wrapper API. + mtmd_helper_bitmap_init_from_buf now returns a bitmap wrapper containing + both the decoded bitmap and an optional video helper context, so keep the + video context alive until mtmd_tokenize completes and release it afterward. + - Also consolidate duplicated audio/video byte loading into a shared + _load_bytes helper, reuse it for image loading, and add richer default HTTP + headers for remote media requests. + +- build(CMakelists): Improve Windows LLVM OpenMP runtime `libomp140.x86_64.dll` discovery + - Also improve diagnostics by reporting the selected runtime source and path, + warning when an explicit override points to a missing file, and keeping a clear + runtime warning when no OpenMP DLL can be found. + - prefer VS 2022 VC143 OpenMP redist and keep System32 as final fallback。 + +- feat(_ctypes_extensions): improve error diagnostics for shared library loading + When `load_shared_library` fails, the resulting `RuntimeError` now + includes a listing of the contents of the searched directories. This + provides immediate context to help developers diagnose missing, misplaced, + or incorrectly named library files. + + - Added `_format_library_dir_contents` to safely format directory listings. + - Appended the directory listing to the failure message. + - Confined this diagnostic work strictly to the failure path to avoid any + performance overhead during successful imports. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/3899b39ce2acc2e019f149b7107f24b6ca297390](https://github.com/ggml-org/llama.cpp/commit/3899b39ce2acc2e019f149b7107f24b6ca297390) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260707 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/12861b918f67b62f78f28c5cabb7223f766e1097...b9b58594023ab673c2dda6723f8909d85d65a2e5 + + ## [0.3.40-Milestone] Reasoning Budget Control, Gemma 4 12B Support, Enhanced Jinja2ChatFormatter, NGram k/k4v Speculative Decoding, Faster Native Sampling and Multimodal Improvements - feat(internals): Add `ReasoningBudgetSampler` support diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b2cfeeb8c..2286fe5eed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -336,6 +336,7 @@ if (LLAMA_BUILD) set(GGML_BACKEND_TARGETS ggml-cann ggml-cuda + ggml-et ggml-hexagon ggml-hip ggml-metal diff --git a/README.md b/README.md index 433e031ae0..c07fafa730 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- + llama-cpp-python logo

# Python Bindings for [`llama.cpp`](https://github.com/ggml-org/llama.cpp) @@ -29,6 +29,7 @@ This package provides: - [How to use the ReasoningBudgetSampler](https://github.com/JamePeng/llama-cpp-python#reasoning-budget-first-reasoning-block) - [Multi-modal Models Support](https://github.com/JamePeng/llama-cpp-python#multi-modal-models) - Support Models Lists + - [Introducing Generic MTMD Chat Handler](https://github.com/JamePeng/llama-cpp-python#generic-mtmd-chat-handler) - [Loading a Local Image With Qwen3VL(Thinking/Instruct)](https://github.com/JamePeng/llama-cpp-python#loading-a-local-image-with-qwen3vlthinkinginstruct) - [Speech Recognition With Qwen3-ASR (Speech-to-Text)](https://github.com/JamePeng/llama-cpp-python#speech-recognition-with-qwen3-asr-speech-to-text) - [Comprehensive Omni MultiModal Example: Gemma-4 (Vision + Audio + Text)](https://github.com/JamePeng/llama-cpp-python#comprehensive-omni-multimodal-example-gemma-4-vision--audio--text) @@ -110,12 +111,22 @@ CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" \ ``` ```powershell -# Windows +# Windows powershell $env:CMAKE_ARGS = "-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" ``` + +```command prompt +# Windows command prompt +set CMAKE_ARGS = "-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" +pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` +**Sanity Checking** +Use this line to check if installation was successful before moving further. +```python.exe -c "from llama_cpp import Llama; print('llama-cpp import OK')"``` +
CLI / requirements.txt @@ -1043,68 +1054,142 @@ Below are the supported multi-modal models and their respective chat handlers (P | [qwen3.6](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF) | `Qwen35ChatHandler` | `qwen3.6` | | [step3-vl](https://huggingface.co/JamePeng2023/Step3-VL-10B-GGUF) | `Step3VLChatHandler` | `step3-vl` | -Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. +Then you'll need to load the multimodal projection model (`mmproj`) together with the main language model. + +Starting from `0.3.41-preview`, new multimodal implementations are recommended to use the updated interfaces in `llama_multimodal`. For backward compatibility, the legacy `llama_chat_format` path is still retained, but may be deprecated in future versions. + +The parameter `clip_model_path` has been renamed to `mmproj_path` to better reflect its purpose and align with llama.cpp's multimodal projection model naming convention. New code should use `mmproj_path` exclusively. + +### Generic MTMD Chat Handler + +For multimodal GGUF models that already include a valid `tokenizer.chat_template`, you can use the generic MTMD handler through `mmproj_path`. + +This is especially useful for newer multimodal models that have not yet received a dedicated Python chat handler. The generic handler renders the model-provided Jinja chat template, then normalizes rendered media placeholders or media URLs into the canonical llama.cpp MTMD media marker, usually `<__media__>`, before calling `mtmd_tokenize`. + +> **Note:** `GenericMTMDChatHandler` is intended as a flexible fallback for template-driven multimodal models. Because different model families may use different media ordering rules, reasoning switches, stop tokens, or special template variables, some models may still require a dedicated chat handler. Please test carefully and report issues if you encounter incorrect prompts, missing media markers, or mismatched media counts. ```python from llama_cpp import Llama -from llama_cpp.llama_chat_format import Llava15ChatHandler -model_path="path/to/llava/ggml-model-f16.gguf" -mmproj_path="path/to/llava/mmproj-model-f16.gguf" +# Model and multimodal projection paths +MODEL_PATH = r"path/to/model.gguf" +MMPROJ_PATH = r"path/to/mmproj.gguf" llm = Llama( - model_path=model_path, - chat_handler=Llava15ChatHandler(clip_model_path=mmproj_path), - n_ctx=2048, + model_path=MODEL_PATH, + mmproj_path=MMPROJ_PATH, + n_gpu_layers=-1, + n_ctx=10240, + verbose=True, + verbosity=2, + chat_handler_kwargs={ + "verbose": True, + }, ) -llm.create_chat_completion( - messages = [ - {"role": "system", "content": "You are an assistant who perfectly describes images."}, +response = llm.create_chat_completion( + messages=[ { "role": "user", "content": [ - {"type" : "text", "text": "What's in this image?"}, - {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" } } - ] + { + "type": "image_url", + "image_url": { + "url": "path/to/image.jpg", + }, + }, + { + "type": "text", + "text": "Describe this image in detail.", + }, + ], } ] ) + +print(response["choices"][0]["message"]["content"]) +```` + +#### Chat Template Resolution Order + +`GenericMTMDChatHandler` resolves the chat template in the following order: + +1. Use the explicit `chat_format` passed through `chat_handler_kwargs`, if provided. +2. Use the named model chat template if `chat_template_name` is provided. +3. Fall back to the default `tokenizer.chat_template` stored in the GGUF model metadata. +4. Fall back to the built-in MTMD chat template if no model template is available. + +Example using a named chat template: + +```python +llm = Llama( + model_path=r"path/to/model.gguf", + mmproj_path=r"path/to/mmproj.gguf", + # chat_template_name="default", + n_gpu_layers=-1, + n_ctx=4096, + chat_handler_kwargs={ + "verbose": False, + }, +) ``` -You can also pull the model from the Hugging Face Hub using the `from_pretrained` method. +#### Passing Extra Template Arguments + +Some model chat templates expose optional Jinja variables such as `enable_thinking`, `add_vision_id`, or model-specific media token switches. Further details can be obtained by analyzing the chat templates provided in `chat_template.jinja` or `tokenizer_config.json` for each model. + +You can pass those values through `chat_handler_kwargs["extra_template_arguments"]`: ```python from llama_cpp import Llama -from llama_cpp.llama_chat_format import MoondreamChatHandler -chat_handler = MoondreamChatHandler.from_pretrained( - repo_id="vikhyatk/moondream2", - filename="*mmproj*", -) +# Model and multimodal projection paths +MODEL_PATH = r"path/to/model.gguf" +MMPROJ_PATH = r"path/to/mmproj.gguf" -llm = Llama.from_pretrained( - repo_id="vikhyatk/moondream2", - filename="*text-model*", - chat_handler=chat_handler, - n_ctx=2048, # n_ctx should be increased to accommodate the image embedding +llm = Llama( + model_path=MODEL_PATH, + mmproj_path=MMPROJ_PATH, + n_gpu_layers=-1, + n_ctx=10240, + verbose=False, + verbosity=1, + chat_handler_kwargs={ + "extra_template_arguments": { + "enable_thinking": True, + }, + "verbose": False, + }, ) +... +``` -response = llm.create_chat_completion( - messages = [ - { - "role": "user", - "content": [ - {"type" : "text", "text": "What's in this image?"}, - {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" } } +The values inside `extra_template_arguments` are passed directly into the Jinja template render call. - ] - } - ] +For models that already have a dedicated handler, you can still instantiate that handler directly: + +```python +from llama_cpp import Llama +from llama_cpp.llama_multimodal import PaddleOCRChatHandler + +MODEL_PATH = r"path/to/model.gguf" +MMPROJ_PATH = r"path/to/mmproj.gguf" + +llm = Llama( + model_path=MODEL_PATH, + chat_handler=PaddleOCRChatHandler( + mmproj_path=MMPROJ_PATH, + ), + n_gpu_layers=-1, # Use all available GPU layers + n_ctx = 0, # Context window size + n_batch=2048, ) -print(response["choices"][0]["text"]) +... ``` +Use `GenericMTMDChatHandler` when the model-provided `tokenizer.chat_template` already works correctly. Prefer a dedicated handler when the model requires custom prompt construction, special reasoning behavior, custom stop tokens, OCR/ASR-specific handling, or non-standard media ordering. + + **Note**: Multi-modal models also support tool calling and JSON mode. @@ -1118,7 +1203,8 @@ print(response["choices"][0]["text"]) ```python # Import necessary libraries from llama_cpp import Llama -from llama_cpp.llama_chat_format import Qwen3VLChatHandler +# from llama_cpp.llama_chat_format import Qwen3VLChatHandler +from llama_cpp.llama_multimodal import Qwen3VLChatHandler import base64 import os @@ -1275,7 +1361,8 @@ The `Qwen3ASRChatHandler` is specifically designed for the Qwen3 Automatic Speec ```python from llama_cpp import Llama -from llama_cpp.llama_chat_format import Qwen3ASRChatHandler +# from llama_cpp.llama_chat_format import Qwen3ASRChatHandler +from llama_cpp.llama_multimodal import Qwen3ASRChatHandler import base64 import os @@ -1380,7 +1467,8 @@ Below is a complete, production-ready example demonstrating how to dynamically r ```python from llama_cpp import Llama -from llama_cpp.llama_chat_format import Gemma4ChatHandler +# from llama_cpp.llama_chat_format import Gemma4ChatHandler +from llama_cpp.llama_multimodal import Gemma4ChatHandler import base64 import os @@ -1560,7 +1648,12 @@ To generate embeddings, use the `LlamaEmbedding` class. It automatically configu from llama_cpp.llama_embedding import LlamaEmbedding, LLAMA_POOLING_TYPE_NONE # Initialize the model (automatically sets embeddings=True) -llm = LlamaEmbedding(model_path="path/to/bge-m3.gguf", n_gpu_layers=-1, pooling_type=LLAMA_POOLING_TYPE_NONE) +llm = LlamaEmbedding( + model_path="path/to/bge-m3.gguf", + n_gpu_layers=-1, + pooling_type=LLAMA_POOLING_TYPE_NONE, + n_seq_max=128, # Maximum independent sequences in one decode batch +) # 1. Simple usage (OpenAI-compatible format) response = llm.create_embedding("Hello, world!") @@ -1574,6 +1667,14 @@ embeddings = llm.embed(documents) # Returns a list of lists (vectors) print(f"Generated {len(embeddings)} vectors.") ``` +> **Parallel batch capacity:** `n_seq_max` controls how many independent +> sequence IDs may coexist in one decode batch; it is not the total number of +> documents accepted by `embed()`. For batch embedding, set it high enough for +> the number of short documents that can fit within `n_batch`. If an error says +> `seq_id=1` exceeds `n_seq_max=1`, initialize the model with at least +> `n_seq_max=2`. For example, use `n_seq_max=8` for up to eight parallel +> sequences. Larger values can use more context resources. + **Advanced Output Formats:** You can request raw arrays or cosine similarity matrices directly: @@ -1667,14 +1768,28 @@ vec_int16 = llm.embed("text", normalize=NORM_MODE_MAX_INT16) embeddings_raw = llm.embed(["search query", "document text"], normalize=NORM_MODE_NONE) ``` -### Legacy Usage (Deprecated) +### Using the standard `Llama` class -The standard `Llama` class still supports basic embedding generation, but it lacks the memory optimizations and reranking capabilities of `LlamaEmbedding`. +The standard `Llama` class also supports the maintained streaming embedding +implementation. Initialize it with `embeddings=True`, then call `embed()` for +raw results or `create_embedding()` for an OpenAI-compatible response. +`LlamaEmbedding` remains a convenient specialized interface because it enables +embedding-oriented defaults and provides the `rank()` helper. ```python -# Old method - Not recommended for large batches or reranking -llm = llama_cpp.Llama(model_path="...", embeddings=True) -emb = llm.create_embedding("text") +llm = llama_cpp.Llama( + model_path="path/to/model.gguf", + embeddings=True, + n_batch=512, + n_seq_max=8, + kv_unified=True, +) + +# OpenAI-compatible response; normalize=True selects L2 normalization. +response = llm.create_embedding(["query", "document"], normalize=True) + +# Raw vectors. Integer normalization modes are also supported. +vectors = llm.embed(["query", "document"], normalize=2) ``` --- diff --git a/docs/icon.png b/docs/icon.png new file mode 100644 index 0000000000..d2d754d746 Binary files /dev/null and b/docs/icon.png differ diff --git a/docs/server.md b/docs/server.md index cd6f86c513..3bdc0c7e69 100644 --- a/docs/server.md +++ b/docs/server.md @@ -37,6 +37,26 @@ CLI arguments and environment variables are available for all of the fields defi Additionally the server supports configuration check out the [configuration section](#configuration-and-multi-model-support) for more information and examples. +#### Model loading mode + +Use `load_mode` to select how the server loads model data. The corresponding +CLI option is `--load_mode`, the environment variable is `LOAD_MODE`, and a +multi-model JSON configuration can set `"load_mode"` for each model. +`use_mmap`, `use_direct_io`, and `use_mlock` are no longer server settings. + +| Value | Mode | Description | +|---:|---|---| +| `0` | `LLAMA_LOAD_MODE_NONE` | Use no special model-loading mode. | +| `1` | `LLAMA_LOAD_MODE_MMAP` | Memory-map the model. This is the default. | +| `2` | `LLAMA_LOAD_MODE_MLOCK` | Keep the loaded model in RAM rather than allowing it to be swapped or compressed. | +| `3` | `LLAMA_LOAD_MODE_MMAP_MLOCK` | Memory-map the model and keep its mapped pages in RAM. | +| `4` | `LLAMA_LOAD_MODE_DIRECT_IO` | Use direct I/O when it is available. | + +For example, start the server with memory mapping plus memory locking: + +```bash +python3 -m llama_cpp.server --model --load_mode 3 +``` ## Guides diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 1f7cce206b..00add6ea4c 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -1,22 +1,34 @@ -```yaml --- title: Llama Class module_name: llama_cpp.llama source_file: llama_cpp/llama.py class_name: Llama -last_updated: 2026-05-16 +last_updated: 2026-07-29 version_target: "latest" --- -``` ## Overview + The `Llama` class is the core, high-level Python wrapper for a `llama.cpp` model. It handles model loading, memory management (KV cache), tokenization, and generation (both base text completion and chat formatting). It includes advanced features like dynamic LoRA routing, dual-mode hybrid/recurrent checkpointing, speculative decoding, and context shifting. +## Role in the Library + +`Llama` is the main user-facing entry point for loading a GGUF model and +creating a native `llama.cpp` context. It exposes completion, chat, tokenization, +embedding, state, sampling, and runtime configuration APIs through one managed +object. + +Use `Llama` when one application needs a general-purpose model interface. +For embedding-only applications, `LlamaEmbedding` provides embedding-oriented +defaults and additional reranking helpers while inheriting the same model and +context lifecycle. + ## Constructor (`__init__`) Initialize the model and context. Note that model loading will immediately allocate RAM/VRAM based on the selected offloading parameters. ### Core Model & Hardware Parameters + | Parameter | Type | Default | Description | | :--- | :--- | :--- | :--- | | `model_path` | `str` | **Required** | Model file path (GGUF format) | @@ -24,28 +36,78 @@ Initialize the model and context. Note that model loading will immediately alloc | `cpu_moe` | `bool` | `False` | Whether to keep all MoE weights on CPU | | `n_cpu_moe` | `int` | `0` | Number of first N MoE layers to keep on CPU (compatible with `cpu_moe`) | | `split_mode` | `int` | `LLAMA_SPLIT_MODE_LAYER` | Model GPU split mode:
• `LLAMA_SPLIT_MODE_NONE`: single GPU
• `LLAMA_SPLIT_MODE_ROW`: row-level split
• `LLAMA_SPLIT_MODE_LAYER`: layer-level split | +| `load_mode` | `int` (`llama_load_mode`) | `LLAMA_LOAD_MODE_MMAP` | How model data is loaded. Select one of the `LLAMA_LOAD_MODE_*` values described below. | | `main_gpu` | `int` | `0` | The primary GPU to use for intermediate results or the entire model. | | `tensor_split` | `List[float]` | `None` | Proportional split of tensors across GPUs (max `LLAMA_MAX_DEVICES`). | -| `use_mmap` | `bool` | `True` | Whether to use memory mapping (mmap) if possible. | -| `use_mlock` | `bool` | `False` | Force the system to keep the model in RAM, preventing swapping. | | `kv_overrides` | `Dict` | `None` | Key-value overrides for the model metadata (supports bool, int, float, str). | -| `numa` | `Union[bool, int]`| `False` | NUMA strategy (e.g., `GGML_NUMA_STRATEGY_DISTRIBUTE`). | +| `numa` | `Union[bool, int]` | `False` | NUMA strategy (e.g., `GGML_NUMA_STRATEGY_DISTRIBUTE`). | + +#### Model Load Modes + +`load_mode` replaces the legacy `use_mmap`, `use_direct_io`, and `use_mlock` +arguments. It accepts a member of `llama_cpp.llama_load_mode`: + +| Value | Integer | Description | +| :--- | :---: | :--- | +| `LLAMA_LOAD_MODE_NONE` | `0` | Use no special model-loading mode. | +| `LLAMA_LOAD_MODE_MMAP` | `1` | Memory-map the model. This is the default. | +| `LLAMA_LOAD_MODE_MLOCK` | `2` | Keep the loaded model in RAM rather than allowing it to be swapped or compressed. | +| `LLAMA_LOAD_MODE_MMAP_MLOCK` | `3` | Memory-map the model and keep its mapped pages in RAM. | +| `LLAMA_LOAD_MODE_DIRECT_IO` | `4` | Use direct I/O when it is available. | + +```python +import llama_cpp + +llm = llama_cpp.Llama( + model_path="models/model.gguf", + load_mode=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP_MLOCK, +) +``` + +The legacy loading arguments are retained only for call compatibility. They no +longer configure the underlying model parameters and may emit a deprecation +warning; set `load_mode` explicitly instead. Use the following migration +mapping: + +| Legacy configuration | Replacement | +| :--- | :--- | +| `use_mmap=False, use_mlock=False` | `load_mode=LLAMA_LOAD_MODE_NONE` | +| `use_mmap=True, use_mlock=False` | `load_mode=LLAMA_LOAD_MODE_MMAP` | +| `use_mmap=False, use_mlock=True` | `load_mode=LLAMA_LOAD_MODE_MLOCK` | +| `use_mmap=True, use_mlock=True` | `load_mode=LLAMA_LOAD_MODE_MMAP_MLOCK` | +| `use_direct_io=True` | `load_mode=LLAMA_LOAD_MODE_DIRECT_IO` | + +### Context & Batch Parameters -### Context & Performance Parameters | Parameter | Type | Default | Description | | :--- | :--- | :--- | :--- | | `n_ctx` | `int` | `512` | Text context size. Set to `0` to load from model metadata. | -| `n_batch` | `int` | `2048` | Maximum batch size for prompt processing. | -| `n_ubatch` | `int` | `512` | Physical batch size. | +| `n_keep` | `int` | `256` | Preferred number of leading tokens to preserve during automatic context shifting. | +| `n_batch` | `int` | `2048` | Maximum number of tokens in a logical prompt-processing batch. The effective value cannot exceed `n_ctx`. | +| `n_ubatch` | `int` | `512` | Maximum number of tokens in a physical micro-batch processed by llama.cpp. | +| `n_seq_max` | `int` | `1` | Maximum independent sequence states in one decode batch. Embedding calls split automatically at this limit; larger values enable more parallel sequences. | +| `n_rs_seq` | `int` | `0` | Experimental recurrent-state snapshots retained per sequence for rollback. `0` disables rollback snapshots. | +| `n_outputs_max` | `int` | `0` | Maximum outputs in a physical batch. `0` is converted to the effective `n_batch`. | | `n_threads` | `int` | `None` | Number of threads for generation (defaults to CPU count // 2). | -| `n_threads_batch`| `int` | `None` | Number of threads for batch processing (defaults to CPU count). | -| `flash_attn_type`| `int` | `AUTO` | Controls Flash Attention activation (`LLAMA_FLASH_ATTN_TYPE_AUTO`). | -| `swa_full` | `bool` | `None` | Whether to use full-size SWA cache | -| `kv_unified` | `bool` | `None` | Use single unified KV buffer for the KV cache of all sequences | -| `type_k` / `type_v`| `int` | `None` | KV cache data type for K and V (defaults to `f16`). | -| `offload_kqv` | `bool` | `True` | Whether to offload K, Q, V tensors to GPU. | +| `n_threads_batch` | `int` | `None` | Number of threads for batch processing (defaults to CPU count). | +| `ctx_type` | `int` | `LLAMA_CONTEXT_TYPE_DEFAULT` | Context implementation selected by llama.cpp. Keep the default unless a model or backend requires another context type. | + +### Embedding, Attention & KV Parameters + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `embeddings` | `bool` | `False` | Enable embedding extraction alongside logits. Must be `True` before calling `embed()` or `create_embedding()`. | +| `pooling_type` | `int` | `LLAMA_POOLING_TYPE_UNSPECIFIED` | Pooling strategy for embedding output. `UNSPECIFIED` follows model metadata, `NONE` returns token-level vectors, and `RANK` returns classifier or reranking output. | +| `attention_type` | `int` | `LLAMA_ATTENTION_TYPE_UNSPECIFIED` | Attention mode used by the context. `UNSPECIFIED` lets llama.cpp select the model-compatible behavior. | +| `logits_all` | `bool` | `False` | Retain logits for every evaluated token instead of only requested outputs. Completion log probabilities require this mode. | +| `flash_attn_type` | `int` | `LLAMA_FLASH_ATTN_TYPE_AUTO` | Controls when Flash Attention is enabled. | +| `offload_kqv` | `bool` | `True` | Offload K, Q, and V tensor operations to the selected device when supported. | +| `swa_full` | `Optional[bool]` | `None` | Use a full-size sliding-window-attention cache. `None` keeps llama.cpp's default. | +| `kv_unified` | `Optional[bool]` | `None` | Use a unified KV buffer for all sequences. `LlamaEmbedding` enables this automatically. | +| `type_k` / `type_v` | `Optional[int]` | `None` | KV cache data types for keys and values. `None` uses llama.cpp defaults. | ### Advanced & Chat Parameters + | Parameter | Type | Default | Description | | :--- | :--- | :--- | :--- | | `chat_format` | `str` | `None` | String specifying the chat template (e.g., `"llama-2"`, `"chatml"`). Guessed from GGUF if None. | @@ -71,7 +133,9 @@ Initialize the model and context. Note that model loading will immediately alloc ## Core Methods ### `create_chat_completion` + Generates a chat response using the configured `chat_format` or `chat_handler`. + ```python import llama_cpp @@ -89,7 +153,9 @@ print(response["choices"][0]["message"]["content"]) ``` ### `create_completion` / `__call__` + Generates standard text completion from a raw string prompt. + ```python import llama_cpp @@ -99,7 +165,9 @@ print(output["choices"][0]["text"]) ``` ### `generate` + A low-level generator yielding token IDs one by one. Highly customizable with sampling parameters, dynamic LoRA mounting, and control vectors. + ```python import llama_cpp @@ -111,14 +179,18 @@ for token in model.generate(tokens, top_k=40, top_p=0.95, temp=0.2): ``` ### `eval` + Low-level method to ingest and evaluate a sequence of tokens. Used internally to update the KV cache and logits. Handles **Context Shifting** automatically to prevent OOM when the token count exceeds `n_ctx`. + ```python # Evaluates a chunk of tokens and updates internal state model.eval(tokens=[1, 453, 234, 987], active_loras=[{"name": "coding_adapter", "scale": 1.0}]) ``` ### `abort` + Immediately halts an active generation loop safely. + * **Usage**: Typically called from a separate monitoring thread (like a timer). When triggered, the running stream will exit and the final chunk will contain `"finish_reason": "abort"`. ### Runtime Logging Control @@ -158,7 +230,9 @@ llm.set_verbosity(1) ``` ### Dynamic LoRA Management + The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dynamically per-generation or per-eval. + * `load_lora(name: str, path: str)`: Loads an adapter into VRAM (does not apply it yet). * `unload_lora(name: str)`: Releases the specific LoRA from VRAM. * `list_loras() -> List[str]`: Returns names of all registered LoRAs. @@ -430,15 +504,123 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn --- -## Deprecated / Changed APIs +## Embeddings + +The `Llama` embedding methods are maintained and use streaming batches. Create +the model with `embeddings=True` before calling them. + +```python +from llama_cpp import Llama, LLAMA_POOLING_TYPE_UNSPECIFIED + +llm = Llama( + model_path="path/to/embedding-model.gguf", + embeddings=True, + pooling_type=LLAMA_POOLING_TYPE_UNSPECIFIED, + n_batch=512, + n_ubatch=512, + n_seq_max=8, + kv_unified=True, +) -> ⚠️ **Warning:** The internal embedding methods on the `Llama` class are deprecated and will be removed. +try: + # Raw sequence embeddings with explicit L2 normalization. + vectors = llm.embed(["query", "document"], normalize=2) -* `embed()` ➔ **Deprecated.** -* `create_embedding()` ➔ **Deprecated.** + # OpenAI-compatible response. + response = llm.create_embedding( + ["query", "document"], + normalize=True, + ) +finally: + llm.close() +``` -**Migration Note:** Do not use `Llama(..., embeddings=True)` combined with `model.create_embedding(...)`. Instead, use the dedicated `LlamaEmbedding` class, which offers optimized batching and reranking support. -*See: [[LlamaEmbedding]]* +### `embed(input, normalize=False, truncate=True, separator=None, return_count=False)` + +Generate raw embedding values for strings or pre-tokenized inputs. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `input` | `Union[str, List[str], List[List[int]]]` | Required | A single string, a list of strings, or a list containing pre-tokenized token-ID lists. | +| `normalize` | `Union[bool, int]` | `False` | `False` returns raw values, while `True` applies L2 normalization. Integer modes are listed below. Rank outputs are not normalized. | +| `truncate` | `bool` | `True` | Truncate each input to the smaller of the context capacity and logical batch capacity. If disabled, an input longer than `n_batch` raises `ValueError`. | +| `separator` | `Optional[str]` | `None` | Split a single string into multiple independent inputs. When set, the result uses the batch return shape. | +| `return_count` | `bool` | `False` | Return `(result, total_token_count)` instead of only the embedding result. | + +Normalization modes follow the llama.cpp embedding example: + +| Value | Behavior | +|---|---| +| `False` or `-1` | No normalization | +| `True` or `2` | Euclidean/L2 normalization | +| `0` | Scale by the maximum absolute value to a maximum magnitude of `32760` | +| `1` | Taxicab/L1 normalization | +| Integer greater than `2` | Corresponding p-norm normalization | + +Unlike `LlamaEmbedding.embed()`, the standard `Llama.embed()` method defaults to +raw, unnormalized output for backward compatibility. + +The return shape depends on the input and pooling type: + +| Input / pooling mode | Return shape | +|---|---| +| Single string with sequence pooling | `List[float]` | +| String list or separator-split string with sequence pooling | `List[List[float]]` | +| `LLAMA_POOLING_TYPE_NONE` | One token embedding matrix per input: `List[List[float]]` for a single string or `List[List[List[float]]]` for a batch | +| `LLAMA_POOLING_TYPE_RANK` with one classifier output | A scalar for a single string or a list of scalars for a batch | +| `LLAMA_POOLING_TYPE_RANK` with multiple classifier outputs | A classifier vector for each input | +| Any mode with `return_count=True` | `(result, total_token_count)` | + +Use `LLAMA_POOLING_TYPE_UNSPECIFIED` for ordinary sentence embeddings unless +the model documentation requires a specific sequence pooling strategy. +`LLAMA_POOLING_TYPE_NONE` is token-level output and should not be used when one +vector per input document is expected. + +### `create_embedding(input, model=None, normalize=False, truncate=True)` + +Wrap sequence or token-level embedding output in an OpenAI-compatible response: + +```python +{ + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [...], + "index": 0, + } + ], + "model": "path/to/embedding-model.gguf", + "usage": { + "prompt_tokens": 12, + "total_tokens": 12, + }, +} +``` + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `input` | `Union[str, List[str]]` | Required | One string or a list of strings. | +| `model` | `Optional[str]` | `None` | Model name placed in the response. Defaults to `model_path`. | +| `normalize` | `Union[bool, int]` | `False` | Passed directly to `embed()`. | +| `truncate` | `bool` | `True` | Passed directly to `embed()`. | + +For parallel batches, `n_seq_max` must cover every sequence ID active in a +single decode batch. The default `n_seq_max=1` is valid and processes multiple +inputs sequentially. Increasing it allows more inputs to be decoded in +parallel; for example, `n_seq_max=8` permits IDs `0` through `7` in one batch. +`n_batch` limits logical input tokens, `n_ubatch` controls the physical token +batch, and `n_seq_max` limits independent sequences. + +`LlamaEmbedding` remains available as the specialized convenience class. It +automatically enables embedding-oriented context options, defaults to L2 +normalization, provides additional output formats, and adds the `rank()` helper +for formatting query/document pairs. + +> **OpenAI compatibility:** use sequence pooling when calling +> `create_embedding()` through an OpenAI-compatible client. Token-level pooling +> (`LLAMA_POOLING_TYPE_NONE`) produces nested token vectors rather than the +> single flat vector normally expected for each input. --- diff --git a/docs/wiki/features/embeddings-rerank.md b/docs/wiki/features/embeddings-rerank.md index e69de29bb2..b5cb54ed44 100644 --- a/docs/wiki/features/embeddings-rerank.md +++ b/docs/wiki/features/embeddings-rerank.md @@ -0,0 +1,419 @@ +--- +title: Embeddings and Reranking +feature_name: Embeddings and Reranking +source_files: + - llama_cpp/llama.py + - llama_cpp/llama_embedding.py + - llama_cpp/_internals.py +last_updated: 2026-07-26 +version_target: "latest" +--- + +# Embeddings and Reranking + +## Overview + +`llama-cpp-python` can use compatible GGUF models for three related inference +workflows: + +- **Sentence or document embeddings** produce one vector per input. +- **Token embeddings** produce one vector per token. +- **Reranking** scores each query/document pair with a cross-encoder model. + +The general-purpose `Llama` class and the specialized `LlamaEmbedding` class +share the same native model and context implementation. Both support streaming +batches, pre-tokenized inputs, multiple pooling modes, and configurable vector +normalization. + +`LlamaEmbedding` adds embedding-oriented defaults, extra output formats, and +the `rank()` helper. The standard `Llama` API is useful when an application +already manages models through the main class or needs both generation and +embedding capabilities. + +## When to Use + +| Goal | Recommended API | Pooling | +|---|---|---| +| Store one vector per sentence or document | `Llama.embed()` or `LlamaEmbedding.embed()` | `LLAMA_POOLING_TYPE_UNSPECIFIED`, or the model-required MEAN/CLS/LAST mode | +| Return an OpenAI-style embedding response | `create_embedding()` | Sequence pooling | +| Inspect a vector for every token | `embed()` | `LLAMA_POOLING_TYPE_NONE` | +| Score documents against a query | `LlamaEmbedding.rank()` | `LLAMA_POOLING_TYPE_RANK` | +| Return raw arrays or a cosine-similarity matrix | `LlamaEmbedding.create_embedding()` | Sequence pooling | + +Use the pooling configuration documented by the model author whenever one is +provided. `LLAMA_POOLING_TYPE_UNSPECIFIED` lets model metadata select the +sequence-pooling behavior and is the safest general default for ordinary +sentence embeddings. + +## Supported Models + +The project README currently lists the following GGUF model families as working +with the embedding and reranking APIs: + +| Model family | Task | GGUF model | +|---|---|---| +| `bge-m3` | Embedding | [bge-m3-GGUF](https://huggingface.co/gpustack/bge-m3-GGUF) | +| `jina-embeddings-v2-base-zh` | Embedding | [jina-embeddings-v2-base-zh-GGUF](https://huggingface.co/gpustack/jina-embeddings-v2-base-zh-GGUF) | +| `jina-embeddings-v3` | Embedding | [jina-embeddings-v3-GGUF](https://huggingface.co/second-state/jina-embeddings-v3-GGUF) | +| `bge-reranker-v2-m3` | Reranking | [bge-reranker-v2-m3-GGUF](https://huggingface.co/gpustack/bge-reranker-v2-m3-GGUF) | +| `qwen3-reranker` | Reranking | [Qwen3-Reranker-GGUF](https://huggingface.co/JamePeng2023/Qwen3-Reranker-GGUF) | + +This is a known-compatible list, not an exhaustive compatibility matrix. +Support for a specific file still depends on its GGUF metadata, pooling +configuration, classifier head, tokenizer, and reranking template. Validate +the output shape and quality before deploying a new model or quantization. + +## Related APIs + +| API | Role | +|---|---| +| `Llama(..., embeddings=True)` | General-purpose model interface with maintained `embed()` and `create_embedding()` methods | +| `LlamaEmbedding(...)` | Specialized subclass that forces `embeddings=True` and `kv_unified=True` | +| `Llama.embed()` | Raw sequence, token-level, or rank output with optional token counting | +| `Llama.create_embedding()` | OpenAI-compatible response wrapper; defaults to raw vectors | +| `LlamaEmbedding.embed()` | Specialized raw embedding API; defaults to L2 normalization | +| `LlamaEmbedding.create_embedding()` | Adds `json`, `json+`, and `array` output formats | +| `LlamaEmbedding.rank()` | Formats query/document pairs and returns reranking scores | +| `Llama.tokenize()` | Converts text into token IDs for pre-tokenized embedding input | + +See [[core/Llama|Llama]] for the general model lifecycle and +[[modules/LlamaEmbedding|Llama Embedding]] for the complete specialized class +reference. + +## Code Examples + +All examples assume that `MODEL_PATH` points to a compatible GGUF embedding or +reranking model. Pooling requirements and output dimensions are model-specific. + +### Sentence Embeddings with `Llama` + +```python +from llama_cpp import Llama, LLAMA_POOLING_TYPE_UNSPECIFIED + + +MODEL_PATH = "path/to/embedding-model.gguf" + +model = Llama( + model_path=MODEL_PATH, + embeddings=True, + pooling_type=LLAMA_POOLING_TYPE_UNSPECIFIED, + n_ctx=512, + n_batch=512, + n_ubatch=512, + n_seq_max=8, + kv_unified=True, + n_gpu_layers=-1, + verbose=False, +) + +try: + documents = [ + "The weather is pleasant today.", + "A storm is expected tomorrow.", + "Vector search compares semantic meaning.", + ] + + vectors, token_count = model.embed( + documents, + normalize=True, + return_count=True, + ) + + print("vectors:", len(vectors)) + print("dimension:", len(vectors[0])) + print("processed tokens:", token_count) + + response = model.create_embedding( + documents, + normalize=2, + ) + print(response["usage"]) +finally: + model.close() +``` + +For `Llama.embed()`, `normalize=False` is the backward-compatible default. +`True` and integer mode `2` both select L2 normalization. + +### Specialized Batch Embeddings and Similarity + +```python +from llama_cpp import LLAMA_POOLING_TYPE_UNSPECIFIED +from llama_cpp.llama_embedding import ( + LlamaEmbedding, + NORM_MODE_EUCLIDEAN, +) + + +MODEL_PATH = "path/to/embedding-model.gguf" + +model = LlamaEmbedding( + model_path=MODEL_PATH, + pooling_type=LLAMA_POOLING_TYPE_UNSPECIFIED, + n_ctx=512, + n_batch=512, + n_ubatch=512, + n_seq_max=8, + n_gpu_layers=-1, + verbose=False, +) + +try: + texts = ["apple", "fruit", "automobile"] + + # "array" always returns one vector entry per input. + vectors = model.create_embedding( + texts, + normalize=NORM_MODE_EUCLIDEAN, + output_format="array", + ) + print("first vector dimension:", len(vectors[0])) + + response = model.create_embedding( + texts, + normalize=NORM_MODE_EUCLIDEAN, + output_format="json+", + ) + print(response["cosineSimilarity"]) +finally: + model.close() +``` + +`json+` extends the OpenAI-style response with `cosineSimilarity` when at least +two compatible sequence vectors are available. + +### Token-Level Embeddings + +```python +from llama_cpp import Llama, LLAMA_POOLING_TYPE_NONE + + +model = Llama( + model_path="path/to/embedding-model.gguf", + embeddings=True, + pooling_type=LLAMA_POOLING_TYPE_NONE, + n_ctx=256, + n_batch=256, + verbose=False, +) + +try: + token_vectors = model.embed("Token-level example", normalize=True) + + print("tokens:", len(token_vectors)) + print("dimension per token:", len(token_vectors[0])) +finally: + model.close() +``` + +Token-level output is a matrix, not one flat vector per document. It is useful +for token analysis and custom pooling, but it is not the normal shape expected +by OpenAI-compatible vector-store clients. + +### Pre-tokenized and Separator-Split Inputs + +```python +from llama_cpp import Llama, LLAMA_POOLING_TYPE_UNSPECIFIED + + +model = Llama( + model_path="path/to/embedding-model.gguf", + embeddings=True, + pooling_type=LLAMA_POOLING_TYPE_UNSPECIFIED, + n_ctx=256, + n_batch=256, + verbose=False, +) + +try: + token_batches = [ + model.tokenize(b"first document"), + model.tokenize(b"second document"), + ] + vectors = model.embed(token_batches, normalize=2) + + split_vectors = model.embed( + "first document\nsecond document", + separator="\n", + normalize=2, + ) + + print(len(vectors), len(split_vectors)) +finally: + model.close() +``` + +When `separator` is set, a single string is treated as a batch and the return +value uses the batch shape. + +### Reranking Query/Document Pairs + +```python +from llama_cpp import LLAMA_POOLING_TYPE_RANK +from llama_cpp.llama_embedding import LlamaEmbedding + + +RERANK_MODEL_PATH = "path/to/reranker-model.gguf" + +ranker = LlamaEmbedding( + model_path=RERANK_MODEL_PATH, + pooling_type=LLAMA_POOLING_TYPE_RANK, + n_ctx=1024, + n_batch=1024, + n_ubatch=512, + n_seq_max=8, + n_gpu_layers=-1, + verbose=False, +) + +try: + query = "What causes rain?" + documents = [ + "Rain forms when atmospheric water vapor condenses and falls.", + "A cake is made from flour, eggs, and sugar.", + "Cloud droplets grow until gravity pulls them toward the ground.", + ] + + scores = ranker.rank(query, documents) + ranked = sorted( + zip(documents, scores), + key=lambda item: item[1], + reverse=True, + ) + + for document, score in ranked: + print(f"{score:.6f} {document}") +finally: + ranker.close() +``` + +`rank()` first checks for a model-provided `rerank` chat template. If no +template exists, it constructs a sequence from the model's BOS, separator, and +EOS tokens. + +## Configuration Notes + +### Pooling Modes + +| Constant | Output behavior | Typical use | +|---|---|---| +| `LLAMA_POOLING_TYPE_UNSPECIFIED` | Uses the model-configured pooling behavior | Default for sentence embedding models | +| `LLAMA_POOLING_TYPE_NONE` | One vector per token | Token analysis or custom pooling | +| `LLAMA_POOLING_TYPE_MEAN` | Mean-pooled sequence vector | Models trained for mean pooling | +| `LLAMA_POOLING_TYPE_CLS` | Vector from the classification token | Models trained with CLS pooling | +| `LLAMA_POOLING_TYPE_LAST` | Vector from the final token | Models trained with last-token pooling | +| `LLAMA_POOLING_TYPE_RANK` | Classifier or reranking output | Cross-encoder reranking models | + +Do not select `LLAMA_POOLING_TYPE_NONE` when one vector per input is required. +It changes both the amount of output and its nesting depth. + +### Normalization Modes + +| Mode | Value | Behavior | +|---|---:|---| +| `NORM_MODE_NONE` | `-1` | Return raw values | +| `NORM_MODE_MAX_INT16` | `0` | Scale the maximum absolute component to `32760` | +| `NORM_MODE_TAXICAB` | `1` | L1/taxicab normalization | +| `NORM_MODE_EUCLIDEAN` | `2` | L2/Euclidean normalization | +| p-norm | Any integer greater than `2` | Normalize using the corresponding p-norm | + +The constant `NORM_MODE_PNORM` currently has value `6`; callers may also pass a +different integer greater than `2`. + +Normalization defaults differ between the two classes: + +| API | Default | +|---|---| +| `Llama.embed()` / `Llama.create_embedding()` | Raw output (`False`) | +| `LlamaEmbedding.embed()` / `LlamaEmbedding.create_embedding()` | L2 (`NORM_MODE_EUCLIDEAN`) | +| Rank output | Never normalized | + +L2-normalized vectors are convenient for cosine similarity because their dot +product is their cosine similarity. + +### Batch and Context Capacity + +| Parameter | Controls | +|---|---| +| `n_ctx` | Maximum context length available to an input sequence | +| `n_batch` | Maximum tokens in one logical decode batch | +| `n_ubatch` | Physical token micro-batch size used by llama.cpp | +| `n_seq_max` | Maximum independent sequences decoded together | + +Embedding input lists are streamed through multiple decode batches. The +default `n_seq_max=1` is valid and processes inputs sequentially. Increasing it +allows more independent sequences to be decoded together, but may use more +context resources. + +Each individual tokenized sequence must fit the configured logical batch +capacity. Choose `n_batch` large enough for the longest intended input and use +`truncate=True` when truncation is acceptable. + +### Input and Return Shapes + +| Input and mode | Direct `embed()` result | +|---|---| +| Single string with sequence pooling | `List[float]` | +| String list with sequence pooling | `List[List[float]]` | +| Separator-split string with sequence pooling | `List[List[float]]` | +| Single string with token-level pooling | `List[List[float]]` | +| String list with token-level pooling | `List[List[List[float]]]` | +| Rank model with one classifier output | Scalar for one string; list of scalars for a batch | +| Rank model with multiple classifier outputs | Classifier vector per input | +| Any input with `return_count=True` | `(result, processed_token_count)` | + +Token counts are measured after tokenization and any applied truncation. + +### Output Wrappers + +`Llama.create_embedding()` returns an OpenAI-compatible dictionary containing +`object`, `data`, `model`, and token `usage`. + +`LlamaEmbedding.create_embedding()` supports: + +| `output_format` | Result | +|---|---| +| `"json"` | OpenAI-style response | +| `"json+"` | OpenAI-style response plus a cosine-similarity matrix when available | +| `"array"` | Raw list containing one output entry per input | + +For OpenAI-compatible vector-store clients, use sequence pooling so each +`data[i]["embedding"]` value is a flat vector. + +### Common Configuration Problems + +| Symptom | Cause | Action | +|---|---|---| +| `Llama model must be created with embeddings=True` | Standard `Llama` was initialized without embedding extraction | Recreate it with `embeddings=True` | +| Output is a matrix for each document | `LLAMA_POOLING_TYPE_NONE` selects token-level output | Use `UNSPECIFIED` or the pooling mode required by the model | +| `seq_id` exceeds `n_seq_max` in custom batch code | A manual sequence ID is outside the configured capacity | Increase `n_seq_max` or use IDs within `0..n_seq_max-1` | +| A long input exceeds `n_batch` | One tokenized sequence is larger than the logical batch | Increase `n_batch`, shorten the input, or enable truncation | +| Local source changes are not visible | Python imported an installed `site-packages` build | Print `llama_cpp.__file__`, then reinstall or adjust the development environment | + +## Limitations + +- Embedding dimensions, valid pooling modes, tokenization, and reranking heads + are determined by the GGUF model. A model that was not exported for the + requested task may not produce meaningful output. +- `rank()` returns raw model scores. They are not automatically calibrated as + probabilities and should primarily be compared within the same query. +- For a two-output reranking head, `rank()` uses the first output as the score. + It does not apply softmax. +- The fallback reranking prompt depends on the model's BOS, separator, and EOS + tokens. Prefer a GGUF model containing a suitable `rerank` chat template. +- `json+` similarity output is intended for at least two compatible, + fixed-length sequence vectors. It is not suitable for ragged token-level + matrices or scalar rank scores. +- Embedding calls clear the context memory used by the operation. Do not expect + a previous completion KV-cache state to remain reusable after embedding on + the same model instance. +- Model and reranking support still requires broader testing across GGUF + architectures. Validate output quality and shape before production use. + +## Related Features +- [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] +- [[core/Llama|Llama](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] — General model lifecycle and built-in embedding APIs. +- [[modules/LlamaEmbedding|Llama Embedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] — Specialized API reference, + normalization constants, and reranking methods. +- [[install|Installation](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/install.md)] — Backend selection, GPU acceleration, and source + installation. diff --git a/docs/wiki/index.md b/docs/wiki/index.md index 8e5dbed14b..bc029f739c 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -44,6 +44,17 @@ These pages document major source modules and related classes. --- +### Features + +Workflow guides combine related classes and configuration into complete usage +patterns. + +| Page | Description | +|---|---| +| [features/embeddings-rerank\|Embeddings and Reranking] | Sentence embeddings, token-level vectors, normalization, streaming batches, similarity output, and cross-encoder reranking. | + +--- + ### Development This section contains maintainer-facing development notes, workflows, and LLM-assisted helper tools for working on `llama-cpp-python`. @@ -99,6 +110,7 @@ Currently available pages: - `modules/LlamaGrammar.md` - `modules/LlamaSpeculative.md` - `modules/Logger.md` +- `features/embeddings-rerank.md` - `development/git-commit-generation-agent.md` - `SCHEMA.md` - `contributing-to-wiki.md` diff --git a/docs/wiki/modules/LlamaEmbedding.md b/docs/wiki/modules/LlamaEmbedding.md index 3aa2427227..5aa3bd8e0e 100644 --- a/docs/wiki/modules/LlamaEmbedding.md +++ b/docs/wiki/modules/LlamaEmbedding.md @@ -3,7 +3,7 @@ title: Llama Embedding module_name: llama_cpp.llama_embedding source_file: llama_cpp/llama_embedding.py class_name: LlamaEmbedding -last_updated: 2026-05-31 +last_updated: 2026-07-26 version_target: "latest" --- @@ -38,6 +38,7 @@ version_target: "latest" | `n_ctx` | int | 0 | Text context window size (0 = model default). | | `n_batch` | int | 512 | Maximum prompt processing batch size. | | `n_ubatch` | int | 512 | Physical batch size. | +| `n_seq_max` | int | 1 (inherited) | Maximum number of independent sequence IDs available in a decode batch. Increase this for parallel embedding batches. | | `pooling_type` | int | `LLAMA_POOLING_TYPE_UNSPECIFIED` (-1) | Pooling strategy used by the model: `LLAMA_POOLING_TYPE_RANK` (4) for rerankers, `LLAMA_POOLING_TYPE_UNSPECIFIED` (-1) for embeddings. | | `n_gpu_layers` | int | 0 | Number of layers offloaded to GPU (0 = CPU only, -1 = all layers). | | `verbose` | bool | True | Whether to print debug information. | @@ -46,9 +47,44 @@ version_target: "latest" ### Initialization Logic 1. Forces `embeddings=True` to enable embedding support. -2. Sets `kv_unified=True` to enable unified KV Cache, allowing arbitrary sequence IDs in a batch without "invalid seq_id" errors. +2. Sets `kv_unified=True` to enable unified KV Cache. Sequence IDs must still + fit within the configured `n_seq_max`. 3. Passes `pooling_type` to the parent class constructor. +### Parallel Batch Capacity + +`n_batch`, `n_ubatch`, and `n_seq_max` control different limits: + +- `n_batch`: maximum number of input tokens in a logical decode batch. +- `n_ubatch`: physical token batch size used by llama.cpp. +- `n_seq_max`: number of independent sequence IDs that may coexist in a decode + batch. + +For multiple documents, set `n_seq_max` to the desired parallel sequence +capacity: + +```python +model = LlamaEmbedding( + model_path="path/to/model.gguf", + n_batch=512, + n_ubatch=512, + n_seq_max=8, +) +``` + +If the configuration is too small, the error includes the current capacity, +valid ID range, and required minimum: + +```text +LlamaBatch.add_sequence: seq_id=1 exceeds the configured sequence capacity +(n_seq_max=1; valid IDs are 0 through 0). For parallel batching, initialize +Llama or LlamaEmbedding with n_seq_max>=2 ... +``` + +`n_seq_max` is not the total number of documents passed to `embed()`; it is the +number that can be active in one decode batch. Increase it carefully because +larger values may require more context resources. + ## Core Methods ### `embed(input, normalize=NORM_MODE_EUCLIDEAN, truncate=True, separator=None, return_count=False)` @@ -129,7 +165,10 @@ version_target: "latest" - Token-level embeddings: `LLAMA_POOLING_TYPE_NONE (0)`. 2. **Batch Optimization for Large Datasets**: - - Adjust `n_batch` and `n_ubatch` to balance performance and memory. + - Adjust `n_batch`, `n_ubatch`, and `n_seq_max` to balance parallelism, + performance, and memory. + - If `seq_id` exceeds the configured capacity, increase `n_seq_max` to at + least `seq_id + 1`. - Streaming processing avoids OOM for large datasets. 3. **Normalization Selection**: @@ -153,7 +192,12 @@ To generate embeddings, use the `LlamaEmbedding` class. It automatically configu from llama_cpp.llama_embedding import LlamaEmbedding, LLAMA_POOLING_TYPE_NONE # Initialize the model (automatically sets embeddings=True) -llm = LlamaEmbedding(model_path="path/to/bge-m3.gguf", n_gpu_layers=-1, pooling_type=LLAMA_POOLING_TYPE_NONE) +llm = LlamaEmbedding( + model_path="path/to/bge-m3.gguf", + n_gpu_layers=-1, + pooling_type=LLAMA_POOLING_TYPE_NONE, + n_seq_max=128, +) # 1. Simple usage (OpenAI-compatible format) response = llm.create_embedding("Hello, world!") @@ -263,7 +307,8 @@ embeddings_raw = llm.embed(["search query", "document text"], normalize=NORM_MOD ## Notes - This class is in development; some features may be unstable, especially reranking model support. -- Performance issues can be addressed by adjusting `n_batch`, `n_ubatch`, and `n_gpu_layers`. +- Performance issues can be addressed by adjusting `n_batch`, `n_ubatch`, + `n_seq_max`, and `n_gpu_layers`. - For custom models, manual `pooling_type` configuration may be required to match model behavior. ## Related Links diff --git a/examples/high_level_api/high_level_api_embedding.py b/examples/high_level_api/high_level_api_embedding.py index feb0ed68d9..bf96213213 100644 --- a/examples/high_level_api/high_level_api_embedding.py +++ b/examples/high_level_api/high_level_api_embedding.py @@ -6,6 +6,6 @@ parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-model.bin") args = parser.parse_args() -llm = Llama(model_path=args.model, embedding=True) +llm = Llama(model_path=args.model, embeddings=True) -print(llm.create_embedding("Hello world!")) +print(llm.create_embedding("Hello world!", normalize=True)) diff --git a/examples/low_level_api/common.py b/examples/low_level_api/common.py index 8adb2923cc..601f5cebdf 100644 --- a/examples/low_level_api/common.py +++ b/examples/low_level_api/common.py @@ -60,9 +60,6 @@ class GptParams: instruct: bool = False perplexity: bool = False - use_mmap: bool = True - use_direct_io: bool = False - use_mlock: bool = False mem_test: bool = False verbose_prompt: bool = False diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py index 1f4f5b3e79..96c4121f4f 100644 --- a/examples/low_level_api/low_level_api_chat_cpp.py +++ b/examples/low_level_api/low_level_api_chat_cpp.py @@ -76,9 +76,6 @@ def __init__(self, params: GptParams) -> None: self.lparams.n_parts = self.params.n_parts self.lparams.seed = self.params.seed self.lparams.memory_f16 = self.params.memory_f16 - self.lparams.use_mlock = self.params.use_mlock - self.lparams.use_mmap = self.params.use_mmap - self.lparams.use_direct_io = self.params.use_direct_io self.model = llama_cpp.llama_load_model_from_file( self.params.model.encode("utf8"), self.lparams diff --git a/examples/notebooks/PerformanceTuning.ipynb b/examples/notebooks/PerformanceTuning.ipynb index ba74e4a41f..43772a5b7e 100644 --- a/examples/notebooks/PerformanceTuning.ipynb +++ b/examples/notebooks/PerformanceTuning.ipynb @@ -24,7 +24,13 @@ "# Hyperparameters\n", "space = [\n", " Categorical([True, False], name=\"f16_kv\"),\n", - " Categorical([True, False], name=\"use_mlock\"),\n", + " Categorical(\n", + " [\n", + " llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP,\n", + " llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP_MLOCK,\n", + " ],\n", + " name=\"load_mode\",\n", + " ),\n", " Integer(1, multiprocessing.cpu_count(), name=\"n_threads\"),\n", " Integer(1, 2048, name=\"n_batch\"),\n", "]\n", @@ -46,13 +52,13 @@ "@use_named_args(space)\n", "def objective(**params):\n", " f16_kv = params[\"f16_kv\"]\n", - " use_mlock = params[\"use_mlock\"]\n", + " load_mode = params[\"load_mode\"]\n", " n_threads = params[\"n_threads\"]\n", " n_batch = params[\"n_batch\"]\n", " llm = llama_cpp.Llama(\n", " model_path=MODEL_PATH,\n", " f16_kv=f16_kv,\n", - " use_mlock=use_mlock,\n", + " load_mode=load_mode,\n", " n_threads=n_threads,\n", " n_batch=n_batch,\n", " )\n", diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 1650e6af69..b359355f9e 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.40" +__version__ = "0.3.45" diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index 1a9f8eb8c5..3634720681 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -5,10 +5,12 @@ import ctypes import functools import pathlib +import importlib.metadata from ctypes.util import find_library from typing import ( Any, Callable, + Iterable, List, Union, Optional, @@ -18,6 +20,15 @@ ) from typing_extensions import TypeAlias +def _version_at_least(version: str) -> bool: + """Check whether installed llama-cpp-python version meets requirement.""" + try: + current = importlib.metadata.version("llama-cpp-python") + from packaging.version import Version + return Version(current) >= Version(version) + except Exception: + return False + def _format_library_dir_contents(base_paths: list[pathlib.Path]) -> str: """Format directory contents for diagnostics after library loading fails.""" sections = [] @@ -90,17 +101,8 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list # Add the library directory to the DLL search path on Windows (if needed) if sys.platform == "win32": - for base_path in base_paths: - p = pathlib.Path(base_path) - if p.exists() and p.is_dir(): - os.add_dll_directory(str(p)) - os.environ["PATH"] = str(p) + os.pathsep + os.environ["PATH"] - if sys.platform == "win32" and sys.version_info >= (3, 9): - for base_path in base_paths: - p = pathlib.Path(base_path) - if p.exists() and p.is_dir(): - os.add_dll_directory(str(p)) + # Add CUDA runtime DLL directories if CUDA is available. if "CUDA_PATH" in os.environ: cuda_path = os.environ["CUDA_PATH"] sub_dirs_to_add = [ @@ -114,13 +116,41 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list if os.path.exists(full_path): os.add_dll_directory(full_path) + # Add HIP runtime DLL directories when HIP backend is available. if "HIP_PATH" in os.environ: - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) + hip_path = os.environ["HIP_PATH"] + for sub_dir in ["bin", "lib"]: + full_path = os.path.join(hip_path, sub_dir) + if os.path.exists(full_path): + os.add_dll_directory(full_path) + # Add Vulkan SDK DLL directories when Vulkan backend is enabled. if "VULKAN_SDK" in os.environ: - os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Bin")) - os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Lib")) + vulkan_sdk = os.environ["VULKAN_SDK"] + for sub_dir in ["Bin", "Lib"]: + full_path = os.path.join(vulkan_sdk, sub_dir) + if os.path.exists(full_path): + os.add_dll_directory(full_path) + + # Add package-provided library directories. + # + # The paths are added in reverse order intentionally. + # This ensures that the first entry in base_paths gets prepended + # to PATH last, making it the highest priority search location. + # + # Example: + # base_paths = [ + # package/lib, + # package/bin, + # ] + # + # After reversed iteration: + # PATH = package/lib;package/bin;... + for base_path in reversed(base_paths): + p = pathlib.Path(base_path) + if p.exists() and p.is_dir(): + os.add_dll_directory(str(p)) + os.environ["PATH"] = str(p) + os.pathsep + os.environ["PATH"] cdll_args["winmode"] = ctypes.RTLD_GLOBAL @@ -130,7 +160,9 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list lib_path = find_library(lib_base_name) if lib_path: try: - return ctypes.CDLL(lib_path, **cdll_args) + lib = ctypes.CDLL(lib_path, **cdll_args) + print(f"[llama-cpp-python].find_library: loaded library from {lib_path}") + return lib except Exception as e: errors.append(f"{lib_path}: {e}") @@ -141,7 +173,9 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list if lib_path.exists(): try: - return ctypes.CDLL(str(lib_path), **cdll_args) + lib = ctypes.CDLL(str(lib_path), **cdll_args) + print(f"[llama-cpp-python].provided_path: loaded library from {lib_path}") + return lib except Exception as e: errors.append(f"{lib_path}: {e}") @@ -184,20 +218,100 @@ class CtypesRef(Generic[CtypesCData]): def ctypes_function_for_shared_library(lib: ctypes.CDLL): - """Decorator for defining ctypes functions with type hints""" + """Create a decorator used to bind typed Python declarations to C symbols. + + The returned decorator accepts either a single exported symbol name or an + iterable of ABI-compatible aliases. When aliases are provided, they are + checked in order and the first available symbol is selected. + """ def ctypes_function( - name: str, argtypes: List[Any], restype: Any, enabled: bool = True + name: Union[str, Iterable[str]], + argtypes: List[Any], + restype: Any, + enabled: bool = True, + required: bool = True, ): + """Bind a Python declaration to one of the requested C symbols. + + Args: + name: A symbol name or an ordered iterable of compatible aliases. + argtypes: The ctypes argument types assigned to the C function. + restype: The ctypes return type assigned to the C function. + enabled: Return the original Python declaration when disabled. + required: Raise if symbol is missing. If False, create a runtime unavailable stub. + + Raises: + ValueError: If no symbol names are provided. + AttributeError: If none of the requested symbols exist in the + shared library. + """ + symbol_names = (name,) if isinstance(name, str) else tuple(name) + + if not symbol_names: + raise ValueError("At least one shared library symbol name is required") + def decorator(f: F) -> F: - if enabled: - func = getattr(lib, name) + if not enabled: + return f + + for symbol_name in symbol_names: + try: + func = getattr(lib, symbol_name) + except AttributeError: + continue + # Validate ctypes argument declarations before assigning them. + # ctypes requires every argtype to provide from_param(). + for index, argtype in enumerate(argtypes): + if not hasattr(argtype, "from_param"): + raise TypeError( + "Invalid ctypes argument type:\n" + f" function: {f.__name__}\n" + f" symbol: {symbol_name}\n" + f" arg index: {index}\n" + f" arg type: {argtype!r}\n" + f" expected: a ctypes type with from_param()" + ) + func.argtypes = argtypes func.restype = restype - functools.wraps(f)(func) + functools.update_wrapper(func, f) + + # Preserve the actual exported symbol selected at runtime for + # diagnostics, especially when ABI aliases are being used. + func.__ctypes_symbol_name__ = symbol_name return func - else: - return f + + message = ( + "None of the shared library symbols were found: " + + ", ".join(symbol_names) + ) + + if required: + raise AttributeError(message) + + # Optional extension API. + # Keep import working when the symbol is unavailable. + print( + "[llama-cpp-python].ctypes_function: WARNING! optional API unavailable\n" + f" symbols: {', '.join(symbol_names)}\n" + f" library: {getattr(lib, '_name', '')}" + ) + + def unavailable(*args, **kwargs): + raise RuntimeError( + "This llama.cpp extension API is unavailable.\n" + f"Required symbol(s): {', '.join(symbol_names)}\n" + f"Library: {getattr(lib, '_name', '')}" + ) + + functools.update_wrapper(unavailable, f) + + # Mark unavailable extension API. + unavailable.__ctypes_symbol_name__ = None + unavailable.__ctypes_optional__ = True + + return unavailable return decorator diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index c4ae7c94bf..ee1a101870 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -6,10 +6,9 @@ import enum import os import pathlib - from llama_cpp._ctypes_extensions import ( + _version_at_least, load_shared_library, - byref, ctypes_function_for_shared_library, ) @@ -21,20 +20,60 @@ TYPE_CHECKING, ) +def _preload_openmp_runtime(): + """Preload bundled OpenMP runtime before loading ggml-base. + + This is required on Windows when CPU backends depend on the packaged + OpenMP runtime DLL. + """ + + # Only Windows DLL loading requires this workaround. + if os.name != "nt": + return + + # Keep compatibility with older package versions. + if not _version_at_least("0.3.39"): + return + + # Some ComfyUI environments include complex software packages and may also contain + # additional OpenMP libraries (such as `libiomp5md.dll`); + # the best approach is to delete the conflicting libraries + # (i.e., OpenMP dynamic libraries that are not the VC143 version). + os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" + + libomp_path = (pathlib.Path(__file__).parent / "lib" / "libomp140.x86_64.dll") + + if not libomp_path.exists(): + print(f"[llama-cpp-python] WARNING: bundled OpenMP runtime not found: {libomp_path}") + return + + try: + ctypes.CDLL(str(libomp_path), winmode=ctypes.RTLD_GLOBAL) + print(f"[llama-cpp-python] loaded bundled OpenMP runtime: {libomp_path}") + except Exception as e: + print( + "[llama-cpp-python] WARNING: failed to load bundled OpenMP runtime:\n" + f" path: {libomp_path}\n" + f" error: {e}" + ) + libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) libggml_base_paths = [ libggml_base_path / "lib", - libggml_base_path / "bin", + # libggml_base_path / "bin", # The `bin` path is no longer used as a search path for dynamic ggml libraries. ] -libggml = load_shared_library("ggml", libggml_base_paths) - -ggml_function = ctypes_function_for_shared_library(libggml) +# Load bundled OpenMP runtime before ggml-base on Windows. +_preload_openmp_runtime() libggml_base = load_shared_library("ggml-base", libggml_base_paths) ggml_base_function = ctypes_function_for_shared_library(libggml_base) +libggml = load_shared_library("ggml", libggml_base_paths) + +ggml_function = ctypes_function_for_shared_library(libggml) + # // ====== ggml.h ====== GGML_FILE_MAGIC = 0x67676d6c # b"ggml" @@ -122,6 +161,7 @@ class GGMLStatus(enum.IntEnum): # GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) # GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) # GGML_TYPE_Q1_0 = 41, +# GGML_TYPE_Q2_0 = 42, # GGML_TYPE_COUNT = 42, # }; class GGMLType(enum.IntEnum): @@ -159,7 +199,8 @@ class GGMLType(enum.IntEnum): GGML_TYPE_MXFP4 = 39 GGML_TYPE_NVFP4 = 40 GGML_TYPE_Q1_0 = 41 - GGML_TYPE_COUNT = 42 + GGML_TYPE_Q2_0 = 42 + GGML_TYPE_COUNT = 43 # // precision @@ -201,6 +242,7 @@ class GGMLPrec(enum.IntEnum): # GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors # GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors # GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors +# GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors # }; class GGMLFType(enum.IntEnum): GGML_FTYPE_UNKNOWN = -1 @@ -230,6 +272,7 @@ class GGMLFType(enum.IntEnum): GGML_FTYPE_MOSTLY_MXFP4 = 25 GGML_FTYPE_MOSTLY_NVFP4 = 26 GGML_FTYPE_MOSTLY_Q1_0 = 27 + GGML_FTYPE_MOSTLY_Q2_0 = 28 # // available tensor operations: @@ -292,6 +335,7 @@ class GGMLFType(enum.IntEnum): # GGML_OP_IM2COL, # GGML_OP_IM2COL_BACK, # GGML_OP_IM2COL_3D, +# GGML_OP_COL2IM_1D, # GGML_OP_CONV_2D, # GGML_OP_CONV_3D, # GGML_OP_CONV_2D_DW, @@ -324,6 +368,7 @@ class GGMLFType(enum.IntEnum): # GGML_OP_RWKV_WKV7, # GGML_OP_SOLVE_TRI, # GGML_OP_GATED_DELTA_NET, +# GGML_OP_LIGHTNING_INDEXER, # GGML_OP_UNARY, @@ -401,55 +446,57 @@ class GGML_OP(enum.IntEnum): GGML_OP_IM2COL = 52 GGML_OP_IM2COL_BACK = 53 GGML_OP_IM2COL_3D = 54 - GGML_OP_CONV_2D = 55 - GGML_OP_CONV_3D = 56 - GGML_OP_CONV_2D_DW = 57 - GGML_OP_CONV_TRANSPOSE_2D = 58 - GGML_OP_POOL_1D = 59 - GGML_OP_POOL_2D = 60 - GGML_OP_POOL_2D_BACK = 61 - GGML_OP_UPSCALE = 62 - GGML_OP_PAD = 63 - GGML_OP_PAD_REFLECT_1D = 64 - GGML_OP_ROLL = 65 - GGML_OP_ARANGE = 66 - GGML_OP_TIMESTEP_EMBEDDING = 67 - GGML_OP_ARGSORT = 68 - GGML_OP_TOP_K = 69 - GGML_OP_LEAKY_RELU = 70 - GGML_OP_TRI = 71 - GGML_OP_FILL = 72 - - GGML_OP_FLASH_ATTN_EXT = 73 - GGML_OP_FLASH_ATTN_BACK = 74 - GGML_OP_SSM_CONV = 75 - GGML_OP_SSM_SCAN = 76 - GGML_OP_WIN_PART = 77 - GGML_OP_WIN_UNPART = 78 - GGML_OP_GET_REL_POS = 79 - GGML_OP_ADD_REL_POS = 80 - GGML_OP_RWKV_WKV6 = 81 - GGML_OP_GATED_LINEAR_ATTN = 82 - GGML_OP_RWKV_WKV7 = 83 - GGML_OP_SOLVE_TRI = 84 - GGML_OP_GATED_DELTA_NET = 85 - - GGML_OP_UNARY = 86 - - GGML_OP_MAP_CUSTOM1 = 87 - GGML_OP_MAP_CUSTOM2 = 88 - GGML_OP_MAP_CUSTOM3 = 89 - - GGML_OP_CUSTOM = 90 - - GGML_OP_CROSS_ENTROPY_LOSS = 91 - GGML_OP_CROSS_ENTROPY_LOSS_BACK = 92 - GGML_OP_OPT_STEP_ADAMW = 93 - GGML_OP_OPT_STEP_SGD = 94 - - GGML_OP_GLU = 95 - - GGML_OP_COUNT = 96 + GGML_OP_COL2IM_1D = 55 + GGML_OP_CONV_2D = 56 + GGML_OP_CONV_3D = 57 + GGML_OP_CONV_2D_DW = 58 + GGML_OP_CONV_TRANSPOSE_2D = 59 + GGML_OP_POOL_1D = 60 + GGML_OP_POOL_2D = 61 + GGML_OP_POOL_2D_BACK = 62 + GGML_OP_UPSCALE = 63 + GGML_OP_PAD = 64 + GGML_OP_PAD_REFLECT_1D = 65 + GGML_OP_ROLL = 66 + GGML_OP_ARANGE = 67 + GGML_OP_TIMESTEP_EMBEDDING = 68 + GGML_OP_ARGSORT = 69 + GGML_OP_TOP_K = 70 + GGML_OP_LEAKY_RELU = 71 + GGML_OP_TRI = 72 + GGML_OP_FILL = 73 + + GGML_OP_FLASH_ATTN_EXT = 74 + GGML_OP_FLASH_ATTN_BACK = 75 + GGML_OP_SSM_CONV = 76 + GGML_OP_SSM_SCAN = 77 + GGML_OP_WIN_PART = 78 + GGML_OP_WIN_UNPART = 79 + GGML_OP_GET_REL_POS = 80 + GGML_OP_ADD_REL_POS = 81 + GGML_OP_RWKV_WKV6 = 82 + GGML_OP_GATED_LINEAR_ATTN = 83 + GGML_OP_RWKV_WKV7 = 84 + GGML_OP_SOLVE_TRI = 85 + GGML_OP_GATED_DELTA_NET = 86 + GGML_OP_LIGHTNING_INDEXER = 87 + + GGML_OP_UNARY = 88 + + GGML_OP_MAP_CUSTOM1 = 89 + GGML_OP_MAP_CUSTOM2 = 90 + GGML_OP_MAP_CUSTOM3 = 91 + + GGML_OP_CUSTOM = 92 + + GGML_OP_CROSS_ENTROPY_LOSS = 93 + GGML_OP_CROSS_ENTROPY_LOSS_BACK = 94 + GGML_OP_OPT_STEP_ADAMW = 95 + GGML_OP_OPT_STEP_SGD = 96 + + GGML_OP_GLU = 97 + + GGML_OP_COUNT = 98 # enum ggml_unary_op { # GGML_UNARY_OP_ABS, diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 91befb2247..7cbd87e4b5 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -54,11 +54,14 @@ def __init__( self.params = params self.verbose = verbose self._exit_stack = ExitStack() + self.model = None + self.vocab = None + self._lora_registry: Dict[str, LlamaLoraAdapter] = {} model = None if not os.path.exists(path_model): - raise ValueError(f"Model path does not exist: {path_model}") + raise ValueError(f"LlamaModel[__init__]: Model path does not exist: {path_model}") with suppress_stdout_stderr(disable=verbose): model = llama_cpp.llama_model_load_from_file( @@ -68,15 +71,20 @@ def __init__( if model is None: raise ValueError(f"Failed to load model from file: {path_model}") - vocab = llama_cpp.llama_model_get_vocab(model) - - if vocab is None: - raise ValueError(f"Failed to get vocab from model: {path_model}") - + # Record ownership immediately so every later failure can release the + # native model. In particular, a failed vocab lookup must not leak the + # successfully loaded model. self.model = model - self.vocab = vocab + try: + vocab = llama_cpp.llama_model_get_vocab(model) + if vocab is None: + raise ValueError(f"LlamaModel[__init__]: Failed to get vocab from model: {path_model}") + except BaseException: + llama_cpp.llama_model_free(model) + self.model = None + raise - self._lora_registry: Dict[str, LlamaLoraAdapter] = {} + self.vocab = vocab def close(self): """Manually free LlamaModel and Vocab/Lora resources.""" @@ -100,9 +108,13 @@ def __del__(self): self.close() def vocab_type(self) -> int: - return llama_cpp.llama_vocab_type(self.model) + if self.vocab is None: + raise RuntimeError("LlamaModel.vocab_type: vocab is None") + return llama_cpp.llama_vocab_type(self.vocab) def n_vocab(self) -> int: + if self.vocab is None: + raise RuntimeError("LlamaModel.n_vocab: vocab is None") return llama_cpp.llama_vocab_n_tokens(self.vocab) def n_ctx_train(self) -> int: @@ -123,6 +135,9 @@ def n_embd_out(self) -> int: def n_layer(self) -> int: return llama_cpp.llama_model_n_layer(self.model) + def n_layer_nextn(self) -> int: + return llama_cpp.llama_model_n_layer_nextn(self.model) + def n_head(self) -> int: return llama_cpp.llama_model_n_head(self.model) @@ -146,6 +161,12 @@ def model_desc(self) -> str: llama_cpp.llama_model_desc(self.model, buf, 256) return buf.value.decode("utf-8") + def model_ftype(self) -> int: + """ + Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 + """ + return llama_cpp.llama_model_ftype(self.model) + def model_size(self) -> int: """ Returns the total size of all the tensors in the model in bytes @@ -561,6 +582,10 @@ def close(self): self._exit_stack.close() self._exit_stack = None + # The context no longer needs to keep its parent model alive once the + # native context and its callbacks have been released. + self.model = None + def __del__(self): self.close() @@ -733,7 +758,14 @@ def decode(self, batch: 'LlamaBatch') -> int: (e.g., negative error codes or invalid batch structures). """ self._assert_ctx() - return_code = llama_cpp.llama_decode(self.ctx, batch.batch) + try: + return_code = llama_cpp.llama_decode(self.ctx, batch.batch) + except Exception as e: + raise RuntimeError( + "llama_decode raised a native exception before returning a status code. " + "This may indicate an invalid batch, invalid token id, corrupted context, " + "backend memory issue, or native access violation." + ) from e if return_code == 0: return 0 @@ -836,6 +868,63 @@ def get_embeddings_seq(self, seq_id: int): self._assert_ctx() return llama_cpp.llama_get_embeddings_seq(self.ctx, seq_id) + def set_embeddings_nextn(self, enabled: bool, masked: bool) -> None: + """ + Set whether the context outputs nextn embeddings or not + If masked == true, output the embeddings only for the tokens with batch.logits != 0 + If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits + """ + self._assert_ctx() + llama_cpp.llama_set_embeddings_nextn(self.ctx, enabled, masked) + + def get_embeddings_nextn(self): + self._assert_ctx() + embeddings = llama_cpp.llama_get_embeddings_nextn(self.ctx) + if not embeddings: + raise RuntimeError("LlamaContext.get_embeddings_nextn: output is unavailable") + return embeddings + + def get_embeddings_nextn_ith(self, i: int): + self._assert_ctx() + embeddings = llama_cpp.llama_get_embeddings_nextn_ith(self.ctx, i) + if not embeddings: + raise RuntimeError( + f"LlamaContext.get_embeddings_nextn_ith: invalid output index {i}" + ) + return embeddings + + def set_embeddings_layer_inp(self, layer_id: int, enabled: bool) -> None: + self._assert_ctx() + if layer_id < 0: + raise ValueError("layer_id must be non-negative") + llama_cpp.llama_set_embeddings_layer_inp(self.ctx, layer_id, enabled) + + def get_embeddings_layer_inp(self, layer_id: int): + self._assert_ctx() + if layer_id < 0: + raise ValueError("layer_id must be non-negative") + embeddings = llama_cpp.llama_get_embeddings_layer_inp(self.ctx, layer_id) + if not embeddings: + raise RuntimeError( + f"LlamaContext.get_embeddings_layer_inp: layer {layer_id} output is unavailable" + ) + return embeddings + + def set_nextn_layer_offset(self, offset: int) -> None: + """ + Select which appended NextN block the DECODER_MTP graph runs (offset past + the trunk: il = n_layer() + offset). Used by the speculative NextN driver to + chain multiple trained NextN heads. Default 0 (first head). + """ + self._assert_ctx() + if offset < 0: + raise ValueError("NextN layer offset must be non-negative") + llama_cpp.llama_set_nextn_layer_offset(self.ctx, offset) + + def get_ctx_other(self): + self._assert_ctx() + return llama_cpp.llama_get_ctx_other(self.ctx) + def reset_timings(self): llama_cpp.llama_perf_context_reset(self.ctx) @@ -984,38 +1073,90 @@ def __init__( n_tokens: int, embd: int, n_seq_max: int, + mixed: bool = False, verbose: bool = True ): # logical validity of parameters if n_tokens <= 0: - raise ValueError(f"n_tokens must be positive, got {n_tokens}") + raise ValueError(f"LlamaBatch[__init__]: n_tokens must be positive, got {n_tokens}") + if embd < 0: + raise ValueError(f"LlamaBatch[__init__]: embd must be non-negative, got {embd}") if n_seq_max <= 0: - raise ValueError(f"n_seq_max must be positive, got {n_seq_max}") + raise ValueError(f"LlamaBatch[__init__]: n_seq_max must be positive, got {n_seq_max}") + if mixed and embd <= 0: + raise ValueError("LlamaBatch[__init__]: mixed batch requires embd > 0.") self.n_tokens_capacity = n_tokens self.embd = embd self.n_seq_max = n_seq_max + self.mixed = mixed self.verbose = verbose + self._token_buf = None + self._owns_token = False self._exit_stack = ExitStack() + self.batch = None - batch = llama_cpp.llama_batch_init(self.n_tokens_capacity, self.embd, self.n_seq_max) + # llama_batch_init allocates either batch.token or batch.embd: + # + # embd == 0 -> token batch + # embd > 0 -> embedding batch + # + # Some llama.cpp paths, such as EAGLE3/MTP, manually create mixed + # token+embd batches after initialization. This wrapper keeps that + # possibility open, but add_token/add_sequence only support token input. + batch = llama_cpp.llama_batch_init( + self.n_tokens_capacity, + self.embd, + self.n_seq_max, + ) if batch is None: raise MemoryError( - f"Failed to allocate memory for llama_batch via llama_batch_init({n_tokens},{embd},{n_seq_max})" + f"Failed to allocate memory for llama_batch via " + f"llama_batch_init({n_tokens},{embd},{n_seq_max})" ) + # Take ownership before validating or allocating supplementary Python + # buffers so close() can release the native allocation on every failure. self.batch = batch + try: + if mixed: + if bool(batch.token): + raise RuntimeError( + "LlamaBatch[__init__]: expected batch.token to be NULL for " + "mixed embedding batch initialized with embd > 0." + ) + if not bool(batch.embd): + raise RuntimeError( + "LlamaBatch[__init__]: expected batch.embd to be non-NULL " + "for mixed batch." + ) + + self._token_buf = ( + llama_cpp.llama_token * self.n_tokens_capacity + )() + batch.token = self._token_buf + self._owns_token = True + except BaseException: + self.close() + raise def close(self): """Manually free LlamaBatch resources.""" if getattr(self, "batch", None) is not None: try: + if getattr(self, "_owns_token", False): + # batch.token points to a Python-owned ctypes buffer in mixed mode. + # llama_batch_free() would call free(batch.token), so clear it first. + self.batch.token = None llama_cpp.llama_batch_free(self.batch) except Exception: pass self.batch = None + self._token_buf = None + self._owns_token = False + if getattr(self, "_exit_stack", None) is not None and hasattr(self._exit_stack, "close"): self._exit_stack.close() self._exit_stack = None @@ -1046,17 +1187,90 @@ def space_left(self) -> int: return self.n_tokens_capacity - self.batch.n_tokens else: raise RuntimeError( - f"LlamaBatch Critical Error: n_tokens ({self.batch.n_tokens}) exceeds capacity ({self.n_tokens_capacity}). " - "This implies a buffer overflow or corrupted internal state." + f"LlamaBatch Critical Error: n_tokens ({self.batch.n_tokens}) exceeds capacity " + f"({self.n_tokens_capacity}). This implies a buffer overflow or " + "corrupted internal state." ) def reset(self): """ - Resets the batch counter to 0. Does not free memory, just resets the index. - Call this before starting a new decoding step. + Reset the logical batch counter. + + This does not free or clear the underlying C buffers. llama_decode only + reads entries in [0, batch.n_tokens), so resetting n_tokens is enough and + matches llama.cpp's reusable batch pattern. + """ + if self.batch is None: + return + self.batch.n_tokens = 0 + + def _require_open(self, where: str) -> None: + if self.batch is None: + raise RuntimeError(f"LlamaBatch.{where}: batch has been closed.") + + def _require_token_buffer(self, where: str) -> None: + """ + Require that batch.token is available. + + llama_batch_init allocates batch.token only when embd == 0. Some advanced + llama.cpp paths manually create mixed token+embd batches, but this Python + token API should only write token ids when batch.token is non-null. """ - if self.batch is not None: - self.batch.n_tokens = 0 + self._require_open(where) + + if self.mixed: + raise RuntimeError( + f"LlamaBatch.{where} is for token-only batches. " + "Use add_token_embedding for mixed batches." + ) + + if not bool(self.batch.token): + raise RuntimeError( + f"LlamaBatch.{where} requires a token buffer, but batch.token is NULL. " + "This batch was likely initialized as an embedding batch. Use a " + "separate embedding or mixed-batch path instead." + ) + + def _validate_seq_ids(self, seq_ids: Sequence[int], where: str) -> int: + n_seq_id = len(seq_ids) + + if n_seq_id <= 0: + raise ValueError(f"LlamaBatch.{where}: seq_ids must not be empty.") + + if n_seq_id > self.n_seq_max: + raise ValueError( + f"LlamaBatch.{where}: token belongs to {n_seq_id} sequences, " + f"but this batch was initialized with n_seq_max={self.n_seq_max}. " + f"Increase n_seq_max to at least {n_seq_id} when constructing " + "Llama, LlamaEmbedding, or LlamaBatch." + ) + + for seq_id in seq_ids: + if not isinstance(seq_id, int): + raise ValueError( + f"LlamaBatch.{where}: seq_id must be int, got " + f"{type(seq_id).__name__}." + ) + + if seq_id < 0: + raise ValueError( + f"LlamaBatch.{where}: invalid seq_id {seq_id}; " + "sequence IDs must be non-negative integers." + ) + + if seq_id >= self.n_seq_max: + required_n_seq_max = seq_id + 1 + raise ValueError( + f"LlamaBatch.{where}: seq_id={seq_id} exceeds the configured " + f"sequence capacity (n_seq_max={self.n_seq_max}; valid IDs " + f"are 0 through {self.n_seq_max - 1}). For parallel batching, " + f"initialize Llama or LlamaEmbedding with " + f"n_seq_max>={required_n_seq_max}, or create LlamaBatch " + "with that value. Use seq_id=0 when processing only one " + "sequence." + ) + + return n_seq_id def add_token(self, token: int, pos: int, seq_ids: Sequence[int], logits: bool): """ @@ -1071,6 +1285,8 @@ def add_token(self, token: int, pos: int, seq_ids: Sequence[int], logits: bool): A single token can be part of multiple sequences simultaneously. logits: A boolean flag indicating whether the backend should compute logits for this token. """ + self._require_token_buffer("add_token") + idx = self.batch.n_tokens if idx >= self.n_tokens_capacity: raise IndexError(f"LlamaBatch overflow[add_token]: Cannot add token. Capacity {self.n_tokens_capacity} reached.") @@ -1078,10 +1294,8 @@ def add_token(self, token: int, pos: int, seq_ids: Sequence[int], logits: bool): self.batch.token[idx] = token self.batch.pos[idx] = pos - n_seq_id = len(seq_ids) - if n_seq_id > self.n_seq_max: - raise ValueError(f"LlamaBatch Error[add_token]: Token belongs to {n_seq_id} sequences, " - f"but n_seq_max was initialized to {self.n_seq_max}.") + n_seq_id = self._validate_seq_ids(seq_ids, "add_token") + self.batch.n_seq_id[idx] = n_seq_id for i, seq_id in enumerate(seq_ids): @@ -1094,36 +1308,48 @@ def add_sequence( self, token_array: Sequence[int], pos_array: Sequence[int], - seq_ids: Sequence[Sequence[int]], + seq_ids: Sequence[int], logits_array: Sequence[bool] ): """ - Adds a sequence of tokens to the batch in a vectorized manner. - Strictly maps the provided arrays to the underlying C++ batch structure without subjective overriding. + Adds a sequence of tokens to the batch. Args: - token_array: A sequence of token IDs to be evaluated. - pos_array: A sequence of logical positions corresponding to each token. - seq_id_array: A sequence of lists, where each list contains the sequence IDs for the respective token. - (e.g., [[0], [0], [0]] for 3 tokens belonging to sequence 0). - logits_array: A sequence of boolean flags indicating whether to compute logits for each token. + token_array: Token ids to evaluate. + pos_array: Logical positions for each token. + seq_ids: Sequence ids shared by every token in this call, usually [0]. + A token can belong to multiple sequences, for example [0, 1], + matching llama.cpp's per-token seq_id list. + logits_array: Whether to request logits/output for each token. """ + self._require_token_buffer("add_sequence") + n_tokens = len(token_array) current_count = self.batch.n_tokens + if len(pos_array) != n_tokens: + raise ValueError( + f"LlamaBatch.add_sequence: pos_array length mismatch: " + f"{len(pos_array)} != {n_tokens}." + ) + + if len(logits_array) != n_tokens: + raise ValueError( + f"LlamaBatch.add_sequence: logits_array length mismatch: " + f"{len(logits_array)} != {n_tokens}." + ) + if current_count + n_tokens > self.n_tokens_capacity: raise IndexError( f"LlamaBatch overflow[add_sequence]: Cannot add {n_tokens} tokens. " f"Space left: {self.n_tokens_capacity - current_count}" ) - n_seq_id = len(seq_ids) - if n_seq_id > self.n_seq_max: - raise ValueError(f"LlamaBatch Error[add_sequence]: Token belongs to {n_seq_id} sequences, " - f"but n_seq_max was initialized to {self.n_seq_max}.") + n_seq_id = self._validate_seq_ids(seq_ids, "add_sequence") for i in range(n_tokens): j = current_count + i + self.batch.token[j] = token_array[i] self.batch.pos[j] = pos_array[i] @@ -1135,14 +1361,220 @@ def add_sequence( self.batch.n_tokens += n_tokens + def _require_embedding_buffer(self, where: str) -> None: + self._require_open(where) + + if self.mixed: + raise RuntimeError( + f"LlamaBatch.{where} is for embedding-only batches. " + "Use add_token_embedding for mixed batches." + ) + + if self.embd <= 0: + raise RuntimeError( + f"LlamaBatch.{where} requires an embedding batch, but embd={self.embd}." + ) -# Embedding functions -def normalize_embedding(embedding): - norm = float(np.linalg.norm(embedding)) - if norm == 0.0: - return embedding - return [v / norm for v in embedding] + if not bool(self.batch.embd): + raise RuntimeError( + f"LlamaBatch.{where} requires batch.embd, but batch.embd is NULL." + ) + + def add_embedding( + self, + embedding: Sequence[float], + pos: int, + seq_ids: Sequence[int], + logits: bool = False, + ) -> None: + """ + Add one embedding row to an embedding batch. + This is for embd-only llama_batch input: + batch.token == NULL + batch.embd != NULL + + Args: + embedding: One embedding vector of length self.embd. + pos: Logical sequence position. + seq_ids: Sequence ids this embedding belongs to, usually [0]. + logits: Whether to request output for this row. + """ + self._require_embedding_buffer("add_embedding") + + if len(embedding) != self.embd: + raise ValueError( + f"LlamaBatch.add_embedding: embedding length mismatch: " + f"{len(embedding)} != embd({self.embd})." + ) + + idx = self.batch.n_tokens + if idx >= self.n_tokens_capacity: + raise IndexError( + f"LlamaBatch overflow[add_embedding]: capacity " + f"{self.n_tokens_capacity} reached." + ) + + n_seq_id = self._validate_seq_ids(seq_ids, "add_embedding") + + base = idx * self.embd + for d, value in enumerate(embedding): + self.batch.embd[base + d] = float(value) + + self.batch.pos[idx] = pos + self.batch.n_seq_id[idx] = n_seq_id + + for i, seq_id in enumerate(seq_ids): + self.batch.seq_id[idx][i] = seq_id + + self.batch.logits[idx] = logits + self.batch.n_tokens += 1 + + def add_embeddings( + self, + embeddings: Sequence[float], + *, + pos_array: Sequence[int], + seq_ids: Sequence[int], + logits_array: Optional[Sequence[bool]] = None, + ) -> None: + """ + Add multiple embedding rows to an embedding batch. + + embeddings layout: + row-major [n_tokens, self.embd] + + The number of rows is inferred from pos_array. This method supports + embedding-only llama_batch inputs: + + batch.token == NULL + batch.embd != NULL + + It only supports one logical position per embedding row. M-RoPE media + embedding batches should continue to use MTMD helper APIs. + """ + self._require_embedding_buffer("add_embeddings") + + n_tokens = len(pos_array) + if n_tokens <= 0: + raise ValueError("LlamaBatch.add_embeddings: pos_array must not be empty.") + + if logits_array is None: + logits_array = [False] * n_tokens + elif len(logits_array) != n_tokens: + raise ValueError( + f"LlamaBatch.add_embeddings: logits_array length mismatch: " + f"{len(logits_array)} != {n_tokens}." + ) + + expected = n_tokens * self.embd + if len(embeddings) != expected: + raise ValueError( + f"LlamaBatch.add_embeddings: embeddings length mismatch: " + f"{len(embeddings)} != n_tokens({n_tokens}) * embd({self.embd}) = {expected}." + ) + + current_count = self.batch.n_tokens + if current_count + n_tokens > self.n_tokens_capacity: + raise IndexError( + f"LlamaBatch overflow[add_embeddings]: cannot add {n_tokens} rows. " + f"Space left: {self.n_tokens_capacity - current_count}." + ) + + n_seq_id = self._validate_seq_ids(seq_ids, "add_embeddings") + + for i in range(n_tokens): + j = current_count + i + + src_base = i * self.embd + dst_base = j * self.embd + + for d in range(self.embd): + self.batch.embd[dst_base + d] = float(embeddings[src_base + d]) + + self.batch.pos[j] = int(pos_array[i]) + self.batch.n_seq_id[j] = n_seq_id + + for k, seq_id in enumerate(seq_ids): + self.batch.seq_id[j][k] = int(seq_id) + + self.batch.logits[j] = int(logits_array[i]) + + self.batch.n_tokens += n_tokens + + def _require_mixed_buffer(self, where: str) -> None: + self._require_open(where) + + if not self.mixed: + raise RuntimeError( + f"LlamaBatch.{where} requires mixed=True batch." + ) + + if self.embd <= 0: + raise RuntimeError( + f"LlamaBatch.{where} requires mixed token+embedding batch, " + f"but embd={self.embd}." + ) + + if not bool(self.batch.token): + raise RuntimeError( + f"LlamaBatch.{where} requires batch.token, but batch.token is NULL." + ) + + if not bool(self.batch.embd): + raise RuntimeError( + f"LlamaBatch.{where} requires batch.embd, but batch.embd is NULL." + ) + + def add_token_embedding( + self, + token: int, + embedding: Sequence[float], + pos: int, + seq_ids: Sequence[int], + logits: bool, + ) -> None: + """ + Add one mixed token+embedding row. + + This is for EAGLE3/MTP-style decoder inputs where each batch row contains: + token id + embedding vector + position + seq ids + logits flag + """ + self._require_mixed_buffer("add_token_embedding") + + if len(embedding) != self.embd: + raise ValueError( + f"LlamaBatch.add_token_embedding: embedding length mismatch: " + f"{len(embedding)} != embd({self.embd})." + ) + + idx = self.batch.n_tokens + if idx >= self.n_tokens_capacity: + raise IndexError( + f"LlamaBatch overflow[add_token_embedding]: capacity " + f"{self.n_tokens_capacity} reached." + ) + + n_seq_id = self._validate_seq_ids(seq_ids, "add_token_embedding") + + self.batch.token[idx] = token + + base = idx * self.embd + for d, value in enumerate(embedding): + self.batch.embd[base + d] = float(value) + + self.batch.pos[idx] = pos + self.batch.n_seq_id[idx] = n_seq_id + + for i, seq_id in enumerate(seq_ids): + self.batch.seq_id[idx][i] = seq_id + + self.batch.logits[idx] = logits + self.batch.n_tokens += 1 class LlamaTokenDataArray: """ @@ -1537,6 +1969,19 @@ def __init__( self.model = model self.params = params + # Initialize every resource-bearing attribute before performing work + # that can fail. This keeps close() safe for partially initialized + # instances. + self.prev = None + self._cur_p = None + self.sampler_chain = None + self.grammar_sampler = None + self.reasoning_budget_sampler = None + self._logits_view = None + self._logits_ptr_addr = None + self._single_token = None + self._single_array = None + self.vocab = llama_cpp.llama_model_get_vocab(model.model) self.n_vocab = model.n_vocab() @@ -1583,7 +2028,6 @@ def __init__( self._build_sampler_chain() # Grammar sampler - self.grammar_sampler = None if params.grammar: self.grammar_sampler = GrammarSampler( model, @@ -1620,6 +2064,7 @@ def _build_sampler_chain(self): # Note: In some implementations, penalties come before other samplers if CommonSamplerType.PENALTIES in p.samplers: s.add_penalties( + self.n_vocab, p.penalty_last_n, p.penalty_repeat, p.penalty_freq, @@ -1899,12 +2344,12 @@ def close(self): # Free grammar sampler if it was initialized. # This releases underlying llama.cpp sampler memory. - if self.grammar_sampler: + if getattr(self, "grammar_sampler", None): self.grammar_sampler.close() self.grammar_sampler = None # Free the sampler chain and all attached C samplers. - if self.sampler_chain: + if getattr(self, "sampler_chain", None): self.sampler_chain.close() self.sampler_chain = None @@ -1922,7 +2367,7 @@ def close(self): self._cur_p = None # Clear token history deque to drop references. - if hasattr(self, "prev"): + if getattr(self, "prev", None) is not None: self.prev.clear() self.prev = None @@ -1934,6 +2379,12 @@ def close(self): self._single_token = None self._single_array = None + # A closed sampling context must not keep the model or configuration + # graph alive merely because the wrapper itself is still referenced. + self.vocab = None + self.model = None + self.params = None + def __del__(self): try: self.close() @@ -2726,8 +3177,8 @@ def add_grammar( c_trigger_tokens, len(trigger_tokens) )) - def add_penalties(self, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float): - self._add_sampler(llama_cpp.llama_sampler_init_penalties(penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) + def add_penalties(self, n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float): + self._add_sampler(llama_cpp.llama_sampler_init_penalties(n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) def add_dry(self, model: LlamaModel, multiplier: float, base: float, allowed_len: int, last_n: int, breakers: List[str]): """DRY (Don't Repeat Yourself) sampler.""" diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index b6a2c8d5a7..8092f86956 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -104,16 +104,19 @@ def __init__( cpu_moe: bool = False, n_cpu_moe: int = 0, split_mode: int = llama_cpp_lib.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, + load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_MMAP, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, - vocab_only: bool = False, - use_mmap: bool = True, + kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, + use_mmap: bool = False, use_direct_io: bool = False, use_mlock: bool = False, + vocab_only: bool = False, check_tensors: bool = False, - use_extra_bufts: bool = False, + use_extra_bufts: bool = True, no_host: bool = False, - kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, + no_alloc: bool = False, + load_mtp: bool = False, # Context Params seed: int = llama_cpp_lib.LLAMA_DEFAULT_SEED, n_ctx: int = 512, @@ -215,15 +218,16 @@ def __init__( n_cpu_moe: Keep the MoE expert weights of the first N layers on CPU. Useful when VRAM is insufficient for MoE models. split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options. + load_mode: How to load the model. See llama_cpp.LLAMA_LOAD_MODE_* for options. main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. + kv_overrides: Key-value overrides for the model. vocab_only: Only load the vocabulary no weights. - use_mmap: Use mmap if possible. - use_mlock: Force the system to keep the model in RAM. check_tensors: validate model tensor data use_extra_bufts: use extra buffer types (used for weight repacking) no_host: bypass host buffer allowing extra buffers to be used - kv_overrides: Key-value overrides for the model. + no_alloc: only load metadata and simulate memory allocations + load_mtp: whether to load MTP layers seed: RNG seed, -1 for random n_ctx: Text context, 0 = from model n_keep: Number of tokens to keep from initial prompt @@ -352,10 +356,19 @@ def __init__( self.model_path = model_path + if (use_mmap or use_direct_io or use_mlock) and verbose: + print( + "Llama.__init__: WARNING: " + "Legacy load options (`use_mmap`, `use_direct_io`, `use_mlock`) " + "are deprecated. Use `load_mode` instead.", + file=sys.stderr, + ) + # Model Params self.model_params = llama_cpp_lib.llama_model_default_params() self.model_params.n_gpu_layers = self._parse_n_gpu_layers(n_gpu_layers) self.model_params.split_mode = split_mode + self.model_params.load_mode = load_mode self.model_params.main_gpu = main_gpu self.tensor_split = tensor_split self._c_tensor_split = None @@ -371,12 +384,11 @@ def __init__( ) # keep a reference to the array so it is not gc'd self.model_params.tensor_split = self._c_tensor_split self.model_params.vocab_only = vocab_only - self.model_params.use_mmap = use_mmap - self.model_params.use_direct_io = use_direct_io - self.model_params.use_mlock = use_mlock self.model_params.check_tensors = check_tensors self.model_params.use_extra_bufts = use_extra_bufts self.model_params.no_host = no_host + self.model_params.no_alloc = no_alloc + self.model_params.load_mtp = load_mtp # Logic of cpu_moe, n_cpu_moe # Reference from llama.cpp/tools/llama-bench/llama-bench.cpp @@ -721,7 +733,6 @@ def __init__( self.chat_handler = llama_multimodal.GenericMTMDChatHandler( chat_format = self.metadata.get("tokenizer.chat_template", None), mmproj_path = mmproj_path, - verbose = self.verbose, chat_template_name=chat_template_name, **chat_handler_kwargs ) @@ -1083,6 +1094,37 @@ def abort(self) -> None: print(f"Llama.abort: Abort signal received. Terminating generation...", file=sys.stderr) self._abort_event.set() + def _validate_eval_tokens( + self, + tokens: Sequence[int], + ) -> None: + """Validate token ids before passing them to llama_decode. + + This mirrors llama.cpp server-side token validation and prevents invalid + token ids from reaching the native decode path, where they may cause hard + crashes instead of Python exceptions. + """ + if not tokens: + return + + for i, tok in enumerate(tokens): + if not isinstance(tok, int): + raise ValueError( + f"Llama.eval: invalid token type at index {i}: " + f"{type(tok).__name__}" + ) + + if tok < 0: + raise ValueError( + f"Llama.eval: invalid negative token id at index {i}: {tok}" + ) + + if tok >= self._n_vocab: + raise ValueError( + f"Llama.eval: token out of vocab at index {i}: " + f"{tok} >= n_vocab({self._n_vocab})" + ) + def eval( self, tokens: Sequence[int], @@ -1107,6 +1149,11 @@ def eval( if n_eval == 0: return + # Validate token ids before any context shifting, batch construction, or + # native llama_decode call. Invalid ids may otherwise reach the C/C++ backend + # and cause hard crashes instead of Python exceptions. + self._validate_eval_tokens(tokens) + # Context Shift: Prevent OOM by discarding older tokens when context limit is reached. if self.n_tokens + n_eval > self._n_ctx: # 0. Check if the memory supports shifting @@ -1266,9 +1313,11 @@ def eval( current_batch_size //= 2 except Exception as e: + min_pos = min(current_batch_size, 128) + preview = chunk[:min_pos] # Catch fatal backend failures (e.g., Code -2, -3) raise RuntimeError(f"Llama.eval(decode): Fatal Decode Error at Pos {self.n_tokens}, " - f"Batch size {current_batch_size}: {str(e)}") from e + f"Batch size {current_batch_size}, chunk[:{min_pos}]={preview}: {str(e)}") from e if not success: raise RuntimeError("Llama.eval(decode): Failed completely even with batch size 1.") @@ -1913,21 +1962,25 @@ def adapter(token_data_array: llama_cpp_lib.llama_token_data_array): ) def create_embedding( - self, input: Union[str, List[str]], model: Optional[str] = None + self, + input: Union[str, List[str]], + model: Optional[str] = None, + normalize: Union[bool, int] = False, + truncate: bool = True, ) -> CreateEmbeddingResponse: - """Embed a string. + """Create an OpenAI-compatible embedding response. Args: - input: The utf-8 encoded string to embed. + input: A string or list of strings to embed. + model: Model name reported in the response. + normalize: ``False`` disables normalization, ``True`` uses L2 + normalization, and integer values select a llama.cpp + normalization mode. + truncate: Truncate inputs to the available context/batch capacity. Returns: - An embedding object. + An OpenAI-compatible embedding response. """ - warnings.warn( - "The `create_embedding` method in `Llama` class is deprecated. " - "Please migrate to `LlamaEmbedding.create_embedding` for better efficiency.", - DeprecationWarning, - ) model_name: str = model if model is not None else self.model_path input = input if isinstance(input, list) else [input] @@ -1935,7 +1988,12 @@ def create_embedding( # get numeric embeddings embeds: Union[List[List[float]], List[List[List[float]]]] total_tokens: int - embeds, total_tokens = self.embed(input, return_count=True) # type: ignore + embeds, total_tokens = self.embed( # type: ignore + input, + normalize=normalize, + truncate=truncate, + return_count=True, + ) # convert to CreateEmbeddingResponse data: List[Embedding] = [ @@ -1959,130 +2017,209 @@ def create_embedding( def embed( self, - input: Union[str, List[str]], - normalize: bool = False, + input: Union[str, List[str], List[List[int]]], + normalize: Union[bool, int] = False, truncate: bool = True, + separator: Optional[str] = None, return_count: bool = False, ): - """Embed a string. + """Embed strings or pre-tokenized inputs. Args: - input: The utf-8 encoded string to embed. + input: A string, a list of strings, or a list of token-id lists. + normalize: ``False``/``-1`` disables normalization, ``True`` uses + L2 normalization. Integer modes follow llama.cpp's embedding + example: 0=max-absolute (scaled to 32760), 1=L1, 2=L2, and + values greater than 2 use the corresponding p-norm. + truncate: Truncate inputs that exceed the context/batch capacity. + separator: Split a single string into multiple inputs. + return_count: Return ``(embeddings, token_count)``. Returns: - A list of embeddings + Sequence embeddings, token-level embeddings for pooling type NONE, + or scalar/vector scores for pooling type RANK. """ - warnings.warn( - "The `embed` method in `Llama` class is deprecated and will be removed in future versions. " - "Please use the `LlamaEmbedding` class from `llama_embedding` module for optimized performance and reranking support.", - DeprecationWarning, - ) + if self.context_params.embeddings is False: + raise RuntimeError( + "Llama model must be created with embeddings=True to call this method" + ) - n_embd = self.n_embd() + ctx = self._ctx.ctx n_batch = self.n_batch + n_ctx = self._n_ctx + n_seq_max = self.context_params.n_seq_max - # get pooling information pooling_type = self.pooling_type() - logits_all = pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_NONE + is_rank = pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_RANK + is_none = pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_NONE - if self.context_params.embeddings is False: - raise RuntimeError( - "Llama model must be created with embeddings=True to call this method" - ) + out_dim = ( + llama_cpp_lib.llama_model_n_cls_out(self._model.model) + if is_rank + else self.n_embd() + ) + + # Preserve the historical bool API while accepting llama.cpp's integer + # normalization modes used by LlamaEmbedding. + if isinstance(normalize, bool): + normalize_mode = 2 if normalize else -1 + elif isinstance(normalize, int): + normalize_mode = normalize + else: + raise TypeError("normalize must be a bool or int") + + def normalize_vector(vector: Sequence[float]) -> List[float]: + values = list(vector) + if normalize_mode == -1 or is_rank: + return values + + array = np.asarray(values, dtype=np.float32) + if normalize_mode == 0: + norm = float(np.max(np.abs(array))) if array.size else 0.0 + scale = 32760.0 + elif normalize_mode == 1: + norm = float(np.sum(np.abs(array))) + scale = 1.0 + elif normalize_mode == 2: + norm = float(np.linalg.norm(array)) + scale = 1.0 + elif normalize_mode > 2: + norm = float( + np.sum(np.abs(array) ** normalize_mode) + ** (1.0 / normalize_mode) + ) + scale = 1.0 + else: + return values + + if norm == 0.0: + return values + return ((array / norm) * scale).tolist() if self.verbose: - llama_cpp_lib.llama_perf_context_reset(self._ctx.ctx) + llama_cpp_lib.llama_perf_context_reset(ctx) if isinstance(input, str): - inputs = [input] + inputs: List[Union[str, List[int]]] = ( + input.split(separator) if separator is not None else [input] + ) + is_single = separator is None else: inputs = input + is_single = False - # reset batch self._batch.reset() + llama_cpp_lib.llama_memory_clear( + llama_cpp_lib.llama_get_memory(ctx), True + ) - # decode and fetch embeddings - data: Union[List[List[float]], List[List[List[float]]]] = [] + data: List[Any] = [] + seq_sizes: List[int] = [] + total_tokens = 0 + + def decode_batch() -> None: + nonlocal seq_sizes + if not seq_sizes: + return - def decode_batch(seq_sizes: List[int]): - llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(self._ctx.ctx), True) self._ctx.decode(self._batch) + + if is_none: + token_index = 0 + for size in seq_sizes: + token_embeddings: List[List[float]] = [] + for _ in range(size): + ptr = llama_cpp_lib.llama_get_embeddings_ith( + ctx, token_index + ) + token_embeddings.append( + [0.0] * out_dim + if ptr is None + else normalize_vector(ptr[:out_dim]) + ) + token_index += 1 + data.append(token_embeddings) + else: + for seq_id in range(len(seq_sizes)): + ptr = llama_cpp_lib.llama_get_embeddings_seq(ctx, seq_id) + if ptr is None: + embedding = [0.0] * out_dim + else: + embedding = list(ptr[:out_dim]) + + if is_rank: + data.append( + embedding[0] if len(embedding) == 1 else embedding + ) + else: + data.append(normalize_vector(embedding)) + self._batch.reset() + llama_cpp_lib.llama_memory_clear( + llama_cpp_lib.llama_get_memory(ctx), True + ) + seq_sizes = [] - # store embeddings - if pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_NONE: - pos: int = 0 - for i, size in enumerate(seq_sizes): - ptr = llama_cpp_lib.llama_get_embeddings(self._ctx.ctx) - embedding: List[List[float]] = [ - ptr[pos + j * n_embd : pos + (j + 1) * n_embd] - for j in range(size) - ] - if normalize: - embedding = [ - internals.normalize_embedding(e) for e in embedding - ] - data.append(embedding) - pos += size + for item in inputs: + if isinstance(item, str): + tokens = self.tokenize(item.encode("utf-8")) + elif isinstance(item, list) and ( + not item or isinstance(item[0], int) + ): + tokens = item else: - for i in range(len(seq_sizes)): - ptr = llama_cpp_lib.llama_get_embeddings_seq(self._ctx.ctx, i) - embedding: List[float] = ptr[:n_embd] - if normalize: - embedding = internals.normalize_embedding(embedding) - data.append(embedding) - - # init state - total_tokens = 0 - s_batch = [] - t_batch = 0 - p_batch = 0 + raise ValueError("Input item must be str or List[int]") - # accumulate batches and encode - for text in inputs: - tokens = self.tokenize(text.encode("utf-8")) - if truncate: - tokens = tokens[:n_batch] + max_tokens = min(n_ctx, n_batch) + if truncate and len(tokens) > max_tokens: + tokens = tokens[:max_tokens] n_tokens = len(tokens) total_tokens += n_tokens - # check for overrun if n_tokens > n_batch: raise ValueError( f"Requested tokens ({n_tokens}) exceed batch size of {n_batch}" ) - # time to eval batch - if t_batch + n_tokens > n_batch: - decode_batch(s_batch) - s_batch = [] - t_batch = 0 - p_batch = 0 + if n_tokens == 0: + # Keep result ordering stable when an empty pre-tokenized input + # follows sequences that are still waiting to be decoded. + decode_batch() + data.append(0.0 if is_rank else []) + continue - # add to batch - self._batch.add_sequence(tokens, p_batch, logits_all) + if ( + self._batch.n_tokens() + n_tokens > n_batch + or len(seq_sizes) >= n_seq_max + ): + decode_batch() - # update batch stats - s_batch.append(n_tokens) - t_batch += n_tokens - p_batch += 1 + seq_id = len(seq_sizes) + logits_array = ( + [True] * n_tokens + if is_none + else [False] * (n_tokens - 1) + [True] + ) + self._batch.add_sequence( + token_array=tokens, + pos_array=list(range(n_tokens)), + seq_ids=[seq_id], + logits_array=logits_array, + ) + seq_sizes.append(n_tokens) - # hanlde last batch - decode_batch(s_batch) + decode_batch() if self.verbose: - llama_cpp_lib.llama_perf_context_print(self._ctx.ctx) - - output = data[0] if isinstance(input, str) else data + llama_cpp_lib.llama_perf_context_print(ctx) - llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(self._ctx.ctx), True) + output = data[0] if is_single else data self.reset() if return_count: return output, total_tokens - else: - return output + return output def _create_completion( self, @@ -3320,16 +3457,16 @@ def __getstate__(self): cpu_moe=self.cpu_moe, n_cpu_moe=self.n_cpu_moe, split_mode=self.model_params.split_mode, + load_mode=self.model_params.load_mode, main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, + kv_overrides=self.kv_overrides, vocab_only=self.model_params.vocab_only, - use_mmap=self.model_params.use_mmap, - use_direct_io=self.model_params.use_direct_io, - use_mlock=self.model_params.use_mlock, check_tensors=self.model_params.check_tensors, use_extra_bufts=self.model_params.use_extra_bufts, no_host=self.model_params.no_host, - kv_overrides=self.kv_overrides, + no_alloc=self.model_params.no_alloc, + load_mtp=self.model_params.load_mtp, # Context Params seed=self._seed, n_ctx=self.context_params.n_ctx, @@ -3470,6 +3607,10 @@ def n_layer(self) -> int: """Return the n_layer value.""" return self._model.n_layer() + def n_layer_nextn(self) -> int: + """Return the n_layer_nextn value.""" + return self._model.n_layer_nextn() + def n_head(self) -> int: """Return the head size.""" return self._model.n_head() diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1e81d80f65..609e0bb3b5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -10,6 +10,7 @@ ggml_backend_sched_eval_callback, ggml_log_callback, ggml_opt_get_optimizer_params, + ggml_cgraph ) from typing import ( @@ -145,60 +146,63 @@ class llama_vocab_type(enum.IntEnum): # https://github.com/ggml-org/llama.cpp/blob/master/src/llama-vocab.h#L10 # // pre-tokenization types # enum llama_vocab_pre_type { -# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, -# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, -# LLAMA_VOCAB_PRE_TYPE_FALCON = 4, -# LLAMA_VOCAB_PRE_TYPE_MPT = 5, -# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, -# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, -# LLAMA_VOCAB_PRE_TYPE_REFACT = 8, -# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, -# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10, -# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, -# LLAMA_VOCAB_PRE_TYPE_OLMO = 12, -# LLAMA_VOCAB_PRE_TYPE_DBRX = 13, -# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, -# LLAMA_VOCAB_PRE_TYPE_PORO = 15, -# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16, -# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, -# LLAMA_VOCAB_PRE_TYPE_VIKING = 18, -# LLAMA_VOCAB_PRE_TYPE_JAIS = 19, -# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, -# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, -# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, -# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, -# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, -# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, -# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, -# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, -# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, -# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, -# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, -# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, -# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, -# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, -# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35, -# LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36, -# LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37, -# LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38, -# LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39, -# LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40, -# LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41, -# LLAMA_VOCAB_PRE_TYPE_AFMOE = 42, -# LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43, -# LLAMA_VOCAB_PRE_TYPE_YOUTU = 44, -# LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45, -# LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46, -# LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47, -# LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48, -# LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49, -# LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50, -# LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51, -# LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52, -# LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53, +# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, +# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, +# LLAMA_VOCAB_PRE_TYPE_FALCON = 4, +# LLAMA_VOCAB_PRE_TYPE_MPT = 5, +# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, +# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, +# LLAMA_VOCAB_PRE_TYPE_REFACT = 8, +# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, +# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10, +# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, +# LLAMA_VOCAB_PRE_TYPE_OLMO = 12, +# LLAMA_VOCAB_PRE_TYPE_DBRX = 13, +# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, +# LLAMA_VOCAB_PRE_TYPE_PORO = 15, +# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16, +# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, +# LLAMA_VOCAB_PRE_TYPE_VIKING = 18, +# LLAMA_VOCAB_PRE_TYPE_JAIS = 19, +# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, +# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, +# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, +# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, +# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, +# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, +# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, +# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, +# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, +# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, +# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, +# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, +# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, +# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, +# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35, +# LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36, +# LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37, +# LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38, +# LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39, +# LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40, +# LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41, +# LLAMA_VOCAB_PRE_TYPE_AFMOE = 42, +# LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43, +# LLAMA_VOCAB_PRE_TYPE_YOUTU = 44, +# LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45, +# LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46, +# LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47, +# LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48, +# LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49, +# LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50, +# LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51, +# LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52, +# LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53, +# LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54, +# LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55, +# LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56, # }; class llama_vocab_pre_type(enum.IntEnum): LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 @@ -255,6 +259,9 @@ class llama_vocab_pre_type(enum.IntEnum): LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51 LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52 LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53 + LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54 + LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55 + LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56 # // note: these values should be synchronized with ggml_rope @@ -364,6 +371,7 @@ class llama_token_type(enum.IntEnum): # LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors # LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -372,6 +380,9 @@ class llama_ftype(enum.IntEnum): LLAMA_FTYPE_MOSTLY_F16 = 1 LLAMA_FTYPE_MOSTLY_Q4_0 = 2 LLAMA_FTYPE_MOSTLY_Q4_1 = 3 + # LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4 + # LLAMA_FTYPE_MOSTLY_Q4_2 = 5 + # LLAMA_FTYPE_MOSTLY_Q4_3 = 6 LLAMA_FTYPE_MOSTLY_Q8_0 = 7 LLAMA_FTYPE_MOSTLY_Q5_0 = 8 LLAMA_FTYPE_MOSTLY_Q5_1 = 9 @@ -406,15 +417,30 @@ class llama_ftype(enum.IntEnum): LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 LLAMA_FTYPE_MOSTLY_NVFP4 = 39 LLAMA_FTYPE_MOSTLY_Q1_0 = 40 + LLAMA_FTYPE_MOSTLY_Q2_0 = 41 LLAMA_FTYPE_GUESSED = 1024 +# // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" +# LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); +@ctypes_function( + "llama_ftype_name", + [ctypes.c_int], + ctypes.c_char_p, +) +def llama_ftype_name( + ftype: llama_ftype, / +) -> bytes: + """ + Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" + """ + # enum llama_rope_scaling_type { # LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1, # LLAMA_ROPE_SCALING_TYPE_NONE = 0, # LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, # LLAMA_ROPE_SCALING_TYPE_YARN = 2, # LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3, -# LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN, +# LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE, # }; class llama_rope_scaling_type(enum.IntEnum): LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1 @@ -422,7 +448,7 @@ class llama_rope_scaling_type(enum.IntEnum): LLAMA_ROPE_SCALING_TYPE_LINEAR = 1 LLAMA_ROPE_SCALING_TYPE_YARN = 2 LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3 - LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN + LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE # enum llama_pooling_type { # LLAMA_POOLING_TYPE_UNSPECIFIED = -1, @@ -484,6 +510,30 @@ class llama_split_mode(enum.IntEnum): LLAMA_SPLIT_MODE_ROW = 2 LLAMA_SPLIT_MODE_TENSOR = 3 +# enum llama_load_mode { +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available +# }; +class llama_load_mode(enum.IntEnum): + LLAMA_LOAD_MODE_NONE = 0 # no special loading mode + LLAMA_LOAD_MODE_MMAP = 1 # memory map the model + LLAMA_LOAD_MODE_MLOCK = 2 # force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_MMAP_MLOCK = 3 # mmap + force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_DIRECT_IO = 4 # use direct I/O if available + +# LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); +@ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p) +def llama_load_mode_name(load_mode: int) -> bytes: + ... + +# LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str); +@ctypes_function("llama_load_mode_from_str", [ctypes.c_char_p], ctypes.c_int) +def llama_load_mode_from_str(str: ctypes.c_char_p) -> int: + ... + # enum llama_context_type { # LLAMA_CONTEXT_TYPE_DEFAULT = 0, # LLAMA_CONTEXT_TYPE_MTP = 1, @@ -723,17 +773,15 @@ class llama_model_tensor_buft_override(ctypes.Structure): # struct llama_model_params { # // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) # ggml_backend_dev_t * devices; -# + # // NULL-terminated list of buffer types to use for tensors that match a pattern # const struct llama_model_tensor_buft_override * tensor_buft_overrides; -# + # int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers # enum llama_split_mode split_mode; // how to split the model across multiple GPUs +# enum llama_load_mode load_mode; // how to load the model -# // main_gpu interpretation depends on split_mode: -# // LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model -# // LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results -# // LLAMA_SPLIT_MODE_LAYER: ignored +# // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE # int32_t main_gpu; # // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() @@ -750,16 +798,13 @@ class llama_model_tensor_buft_override(ctypes.Structure): # // override key-value pairs of the model meta data # const struct llama_model_kv_override * kv_overrides; - # // Keep the booleans together to avoid misalignment during copy-by-value. # bool vocab_only; // only load the vocabulary, no weights -# bool use_mmap; // use mmap if possible -# bool use_direct_io; // use direct io, takes precedence over use_mmap when supported -# bool use_mlock; // force system to keep model in RAM # bool check_tensors; // validate model tensor data # bool use_extra_bufts; // use extra buffer types (used for weight repacking) # bool no_host; // bypass host buffer allowing extra buffers to be used # bool no_alloc; // only load metadata and simulate memory allocations +# bool load_mtp; // whether to load MTP layers # }; class llama_model_params(ctypes.Structure): """Parameters for llama_model @@ -769,57 +814,54 @@ class llama_model_params(ctypes.Structure): tensor_buft_overrides(llama_model_tensor_buft_override): NULL-terminated list of buffer types to use for tensors that match a pattern n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers split_mode (int): how to split the model across multiple GPUs + load_mode (int): how to load the model main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights - use_mmap (bool): use mmap if possible - use_direct_io(bool): use direct io, takes precedence over use_mmap when supported - use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking) no_host (bool): bypass host buffer allowing extra buffers to be used - no_alloc (bool): only load metadata and simulate memory allocations""" + no_alloc (bool): only load metadata and simulate memory allocations + load_mtp (bool): whether to load MTP layers""" if TYPE_CHECKING: devices: CtypesArray[ctypes.c_void_p] # NOTE: unused tensor_buft_overrides: CtypesPointer[llama_model_tensor_buft_override] n_gpu_layers: int split_mode: int + load_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] vocab_only: bool - use_mmap: bool - use_direct_io: bool - use_mlock: bool check_tensors: bool use_extra_bufts: bool no_host: bool no_alloc: bool + load_mtp: bool _fields_ = [ - ("devices", ctypes.c_void_p), # NOTE: unnused + ("devices", ctypes.POINTER(ctypes.c_void_p)), # NOTE: unnused ("tensor_buft_overrides", ctypes.POINTER(llama_model_tensor_buft_override)), ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), + ("load_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("vocab_only", ctypes.c_bool), - ("use_mmap", ctypes.c_bool), - ("use_direct_io", ctypes.c_bool), - ("use_mlock", ctypes.c_bool), ("check_tensors", ctypes.c_bool), ("use_extra_bufts", ctypes.c_bool), ("no_host", ctypes.c_bool), ("no_alloc", ctypes.c_bool), + ("load_mtp", ctypes.c_bool), ] llama_model_params_p = ctypes.POINTER(llama_model_params) @@ -1746,6 +1788,11 @@ def llama_model_n_layer(model: llama_model_p, /) -> int: ... +# LLAMA_API int32_t llama_model_n_layer_nextn(const struct llama_model * model); +@ctypes_function("llama_model_n_layer_nextn", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_layer_nextn(model: llama_model_p, /) -> int: + ... + # LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); @ctypes_function("llama_model_n_head", [llama_model_p_ctypes], ctypes.c_int32) def llama_model_n_head(model: llama_model_p, /) -> int: @@ -1917,6 +1964,21 @@ def llama_model_desc( ... +# // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 +# LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model); +@ctypes_function( + "llama_model_ftype", + [llama_model_p_ctypes], + ctypes.c_int, +) +def llama_model_ftype( + model: llama_model_p, + /, +) -> int: + """Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0""" + ... + + # // Returns the total size of all the tensors in the model in bytes # LLAMA_API uint64_t llama_model_size(const struct llama_model * model); @ctypes_function("llama_model_size", [llama_model_p_ctypes], ctypes.c_uint64) @@ -3492,6 +3554,26 @@ def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: ... +# // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) +# LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens); +@ctypes_function( + "llama_vocab_get_suppress_tokens", + [ + llama_vocab_p_ctypes, + ctypes.POINTER(ctypes.c_int32), + ], + llama_token_p, +) +def llama_vocab_get_suppress_tokens( + vocab: llama_vocab_p, + n_suppress_tokens: ctypes.POINTER(ctypes.c_int32), # type: ignore +) -> llama_token_p: # type: ignore + """ + model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) + """ + ... + + # LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); @ctypes_function( "llama_vocab_fim_pre", @@ -4577,16 +4659,24 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( +# int32_t n_vocab, # int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) -# float penalty_repeat, // 1.0 = disabled -# float penalty_freq, // 0.0 = disabled -# float penalty_present); // 0.0 = disabled +# float penalty_repeat, // must be > 0.0, 1.0 = disabled +# float penalty_freq, // must be finite, 0.0 = disabled +# float penalty_present); // must be finite, 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", - [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], + [ + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_float, + ctypes.c_float, + ctypes.c_float, + ], llama_sampler_p_ctypes, ) def llama_sampler_init_penalties( + n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, @@ -5030,3 +5120,309 @@ def llama_opt_epoch( callback_eval: ctypes.c_void_p, / ): ... + +############################## +# // llama.cpp/src/llama-ext.h +############################## + +# // this is a staging header for new llama.cpp API +# // breaking changes and C++ are allowed. everything here should be considered WIP +# // try as much as possible to not include this header in the rest of the codebase + +ctypes_function_llama_ext = ctypes_function_for_shared_library(_lib) + +# // Reserve a new compute graph. It is valid until the next call to llama_graph_reserve. +# LLAMA_API struct ggml_cgraph * llama_graph_reserve( +# struct llama_context * ctx, +# uint32_t n_tokens, +# uint32_t n_seqs, +# uint32_t n_outputs); +@ctypes_function_llama_ext( + [ + "llama_graph_reserve", + "?llama_graph_reserve@@YAPEAUggml_cgraph@@PEAUllama_context@@III@Z", + "__Z19llama_graph_reserveP13llama_contextjjj", + "_Z19llama_graph_reserveP13llama_contextjjj", + ], + [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32], + ctypes.POINTER(ggml_cgraph), + required=False, +) +def llama_graph_reserve( + ctx: llama_context_p, + n_tokens: ctypes.c_uint32, + n_seqs: ctypes.c_uint32, + n_outputs: ctypes.c_uint32, +) -> ctypes.POINTER(ggml_cgraph): # type: ignore + """ + Reserve a new compute graph. It is valid until the next call to llama_graph_reserve. + """ + ... + +# // Get the default ggml_type for a given ftype. +# LLAMA_API ggml_type llama_ftype_get_default_type(llama_ftype ftype); +@ctypes_function_llama_ext( + [ + "llama_ftype_get_default_type", + "?llama_ftype_get_default_type@@YA?AW4ggml_type@@W4llama_ftype@@@Z", + "__Z28llama_ftype_get_default_type11llama_ftype", + "_Z28llama_ftype_get_default_type11llama_ftype", + ], + [ctypes.c_int], + int, + required=False, +) +def llama_ftype_get_default_type( + ftype: llama_ftype +) -> int: + """ + Get the default ggml_type for a given ftype. + """ + ... + +# LLAMA_API int32_t llama_model_n_expert (const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_n_expert", + "?llama_model_n_expert@@YAHPEBUllama_model@@@Z", + "__Z20llama_model_n_expertPK11llama_model", + "_Z20llama_model_n_expertPK11llama_model", + ], + [llama_model_p_ctypes], + ctypes.c_int32, + required=False, +) +def llama_model_n_expert( + model: llama_model_p +) -> ctypes.c_int32: + ... + +# LLAMA_API int32_t llama_model_n_devices(const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_n_devices", + "?llama_model_n_devices@@YAHPEBUllama_model@@@Z", + "__Z21llama_model_n_devicesPK11llama_model", + "_Z21llama_model_n_devicesPK11llama_model", + ], + [llama_model_p_ctypes], + ctypes.c_int32, + required=False, +) +def llama_model_n_devices( + model: llama_model_p +) -> ctypes.c_int32: + ... + +# LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int i); +@ctypes_function_llama_ext( + [ + "llama_model_get_device", + "?llama_model_get_device@@YAPEAUggml_backend_device@@PEBUllama_model@@H@Z", + "__Z22llama_model_get_devicePK11llama_modeli", + "_Z22llama_model_get_devicePK11llama_modeli", + ], + [llama_model_p_ctypes, ctypes.c_int], + ctypes.c_void_p, + required=False, +) +def llama_model_get_device( + model: llama_model_p, + i: int, +) -> ctypes.c_void_p: + ... + +# // Set whether the context outputs nextn embeddings or not +# // If masked == true, output the embeddings only for the tokens with batch.logits != 0 +# // If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits +# LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); +@ctypes_function_llama_ext( + [ + "llama_set_embeddings_nextn", + "?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z", + "__Z26llama_set_embeddings_nextnP13llama_contextbb", + "_Z26llama_set_embeddings_nextnP13llama_contextbb", + ], + [llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], + None, + required=False, +) +def llama_set_embeddings_nextn( + ctx: llama_context_p, + value: bool, + masked: bool, +): + """ + Set whether the context outputs nextn embeddings or not + If masked == true, output the embeddings only for the tokens with batch.logits != 0 + If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits + """ + ... + +# // Select which appended NextN block the DECODER_MTP graph runs (offset past +# // the trunk: il = n_layer() + offset). Used by the speculative NextN driver to +# // chain multiple trained NextN heads. Default 0 (first head). +# LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset); +@ctypes_function_llama_ext( + [ + "llama_set_nextn_layer_offset", + "?llama_set_nextn_layer_offset@@YAXPEAUllama_context@@H@Z", + "__Z28llama_set_nextn_layer_offsetP13llama_contexti", + "_Z28llama_set_nextn_layer_offsetP13llama_contexti", + ], + [llama_context_p_ctypes, ctypes.c_int32], + None, + required=False, +) +def llama_set_nextn_layer_offset( + ctx: llama_context_p, + offset: ctypes.c_int32, +): + """ + Select which appended NextN block the DECODER_MTP graph runs (offset past + the trunk: il = n_layer() + offset). Used by the speculative NextN driver to + chain multiple trained NextN heads. Default 0 (first head). + """ + ... + +# // mirrors: +# // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); +# LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); +@ctypes_function_llama_ext( + [ + "llama_get_embeddings_nextn", + "?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z", + "__Z26llama_get_embeddings_nextnP13llama_context", + "_Z26llama_get_embeddings_nextnP13llama_context", + ], + [llama_context_p_ctypes], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_get_embeddings_nextn( + ctx: llama_context_p, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + ... + +# // LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); +# LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i); +@ctypes_function_llama_ext( + [ + "llama_get_embeddings_nextn_ith", + "?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z", + "__Z30llama_get_embeddings_nextn_ithP13llama_contexti", + "_Z30llama_get_embeddings_nextn_ithP13llama_contexti", + ], + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_get_embeddings_nextn_ith( + ctx: llama_context_p, + i: ctypes.c_int32, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + ... + +# // Set whether the context outputs the input embeddings of a specific layer +# LLAMA_API void llama_set_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid, bool value); +@ctypes_function_llama_ext( + [ + "llama_set_embeddings_layer_inp", + "?llama_set_embeddings_layer_inp@@YAXPEAUllama_context@@I_N@Z", + "__Z30llama_set_embeddings_layer_inpP13llama_contextjb", + "_Z30llama_set_embeddings_layer_inpP13llama_contextjb", + ], + [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_bool], + None, + required=False, +) +def llama_set_embeddings_layer_inp( + ctx: llama_context_p, + lid: ctypes.c_uint32, + value: bool, +) -> None: # type: ignore + """ + Set whether the context outputs the input embeddings of a specific layer + """ + ... + +# // mirrors: +# // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); +# LLAMA_API float * llama_get_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid); +@ctypes_function_llama_ext( + [ + "llama_get_embeddings_layer_inp", + "?llama_get_embeddings_layer_inp@@YAPEAMPEAUllama_context@@I@Z", + "__Z30llama_get_embeddings_layer_inpP13llama_contextj", + "_Z30llama_get_embeddings_layer_inpP13llama_contextj", + ], + [llama_context_p_ctypes, ctypes.c_uint32], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_get_embeddings_layer_inp( + ctx: llama_context_p, + lid: ctypes.c_uint32, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + ... + +# LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); +@ctypes_function_llama_ext( + [ + "llama_get_ctx_other", + "?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z", + "__Z19llama_get_ctx_otherP13llama_context", + "_Z19llama_get_ctx_otherP13llama_context", + ], + [llama_context_p_ctypes], + llama_context_p_ctypes, + required=False, +) +def llama_get_ctx_other( + ctx: llama_context_p, +) -> llama_context_p: + ... + +# // model/context data extraction + +# // returns pointer to the target-model layer indices +# LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_target_layer_ids", + "?llama_model_target_layer_ids@@YAPEBHPEBUllama_model@@@Z", + "__Z28llama_model_target_layer_idsPK11llama_model", + "_Z28llama_model_target_layer_idsPK11llama_model", + ], + [llama_model_p_ctypes], + ctypes.POINTER(ctypes.c_int32), + required=False, +) +def llama_model_target_layer_ids( + model: llama_model_p +) -> ctypes.POINTER(ctypes.c_int32): # type: ignore + """ + returns pointer to the target-model layer indices + """ + ... + +# // returns the number of extracted layers from target model +# LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_target_layer_ids_n", + "?llama_model_target_layer_ids_n@@YAIPEBUllama_model@@@Z", + "__Z30llama_model_target_layer_ids_nPK11llama_model", + "_Z30llama_model_target_layer_ids_nPK11llama_model", + ], + [llama_model_p_ctypes], + ctypes.c_uint32, + required=False, +) +def llama_model_target_layer_ids_n( + model: llama_model_p +) -> int: + """ + returns the number of extracted layers from target model + """ + ... diff --git a/llama_cpp/llama_embedding.py b/llama_cpp/llama_embedding.py index 0c1df339ce..baa4b9f066 100644 --- a/llama_cpp/llama_embedding.py +++ b/llama_cpp/llama_embedding.py @@ -1,6 +1,6 @@ import numpy as np from typing import Union, List, Optional, Dict, Any, Tuple -import llama_cpp.llama_cpp as llama_cpp +import llama_cpp.llama_cpp as llama_cpp_lib from .llama_types import Embedding from .llama import Llama # Pooling types from .llama_cpp @@ -128,7 +128,7 @@ def embed( ctx = self._ctx.ctx n_batch = self.n_batch n_ctx = self._n_ctx - n_ubatch = self.context_params.n_ubatch + n_seq_max = self.context_params.n_seq_max # Determine if it is in Rerank mode try: @@ -137,11 +137,9 @@ def embed( pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED is_rank = (pooling_type == LLAMA_POOLING_TYPE_RANK) is_none = (pooling_type == LLAMA_POOLING_TYPE_NONE) # Token-level embedding - logits_all = True if is_none else False - # Determine the output dimension if is_rank: - out_dim = llama_cpp.llama_model_n_cls_out(self._model.model) + out_dim = llama_cpp_lib.llama_model_n_cls_out(self._model.model) else: out_dim = self.n_embd() @@ -166,9 +164,9 @@ def embed( # Reset Context and Batch if self.verbose: - llama_cpp.llama_perf_context_reset(ctx) + llama_cpp_lib.llama_perf_context_reset(ctx) self._batch.reset() - llama_cpp.llama_memory_clear(llama_cpp.llama_get_memory(ctx), True) + llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(ctx), True) # Initialize State Variables results: List[Any] = [] @@ -190,7 +188,7 @@ def _decode_batch(): doc_tokens_embd = [] for _ in range(seq_len): # Get the vector of the i-th token - ptr = llama_cpp.llama_get_embeddings_ith(ctx, curr_token_idx) + ptr = llama_cpp_lib.llama_get_embeddings_ith(ctx, curr_token_idx) if ptr is None: # Fallback: append zero vector or skip (here we zero-pad to keep shape) doc_tokens_embd.append([0.0] * out_dim) @@ -207,7 +205,7 @@ def _decode_batch(): else: for i in range(len(batch_seq_lens)): # Obtain the vector of the i-th sequence. - ptr = llama_cpp.llama_get_embeddings_seq(ctx, i) + ptr = llama_cpp_lib.llama_get_embeddings_seq(ctx, i) data = ptr[:out_dim] if not is_rank: @@ -219,7 +217,7 @@ def _decode_batch(): results.append(data) self._batch.reset() - llama_cpp.llama_memory_clear(llama_cpp.llama_get_memory(ctx), True) + llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(ctx), True) batch_seq_lens = [] # Main Streaming Loop @@ -247,7 +245,10 @@ def _decode_batch(): continue # Check Batch Capacity - if (self._batch.n_tokens() + n_tokens > n_batch) or (idx_in_batch >= n_ubatch): + if ( + self._batch.n_tokens() + n_tokens > n_batch + or idx_in_batch >= n_seq_max + ): _decode_batch() idx_in_batch = 0 @@ -272,7 +273,7 @@ def _decode_batch(): _decode_batch() if self.verbose: - llama_cpp.llama_perf_context_print(ctx) + llama_cpp_lib.llama_perf_context_print(ctx) final_result = results[0] if is_single else results diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index f1b320b772..cc159924fb 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -102,6 +102,7 @@ def __init__( image_max_tokens: int = -1, chat_template_override: Optional[str] = None, batch_max_tokens: int = 1024, + extra_template_arguments: Optional[Dict[str, Any]] = None, **kwargs ): @@ -148,19 +149,34 @@ def __init__( import llama_cpp.mtmd_cpp as mtmd_cpp self._mtmd_cpp = mtmd_cpp self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None - self.extra_template_arguments: dict[str, Any] = {} + + if extra_template_arguments is not None and not isinstance(extra_template_arguments, dict): + raise TypeError( + f"{self.log_prefix}(__init__): `extra_template_arguments` must be a dict." + ) + + # Preserve subclass attributes + if not hasattr(self, "chat_format"): + self.chat_format = None + + self.chat_format_override = chat_template_override + self.extra_template_arguments: dict[str, Any] = dict(extra_template_arguments or {}) self.is_support_vision = False self.is_support_audio = False self.is_support_video = False + self.chat_template = None + self._chat_format_parser_tags = [] + self._template_initialized = False + # Pre-compile Jinja template - if (not hasattr(self, "chat_format") or self.chat_format is None) and chat_template_override is None: - self.chat_format = self.CHAT_FORMAT - elif chat_template_override is not None: - self.chat_format = chat_template_override + if self.chat_format is None: + if self.chat_format_override is not None: + self.chat_format = self.chat_format_override + else: + self.chat_format = self.CHAT_FORMAT - self._chat_format_parser_tags = [] self._change_chat_template(self.chat_format) self._exit_stack = ExitStack() @@ -243,11 +259,15 @@ def close(self) -> None: if getattr(self, "mtmd_ctx", None) is not None: try: self._mtmd_cpp.mtmd_free(self.mtmd_ctx) + self.mtmd_ctx = None except Exception: pass - self.mtmd_ctx = None - self.mctx_params = None - self.chat_template = None + self.mctx_params = None + self.chat_format = None + self.chat_template = None + self.chat_template_override = None + self._template_initialized = False + self._chat_format_parser_tags = [] if getattr(self, "_exit_stack", None) is not None and hasattr(self._exit_stack, "close"): self._exit_stack.close() @@ -515,6 +535,252 @@ def _is_audio_chunk(self, chunk_type: int) -> bool: == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO ) + def _render_mtmd_prompt( + self, + messages: List[llama_types.ChatCompletionRequestMessage], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + add_generation_prompt: bool = True, + ) -> str: + """ + Render the chat template into plain prompt text. + + This stage only renders the Jinja template. It does not normalize media + placeholders or replace media URLs with the MTMD runtime marker. + """ + return self.chat_template.render( + messages=messages, + add_generation_prompt=add_generation_prompt, + eos_token=self.mtmd_eos_token, + bos_token=self.mtmd_bos_token, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + **getattr(self, "extra_template_arguments", {}), + ) + + def _replace_media_placeholders( + self, + text: str, + media_items: List[Dict[str, str]], + ) -> str: + """ + Normalize rendered media placeholders and media URLs into the MTMD runtime marker. + + llama.cpp MTMD tokenization recognizes the canonical media marker, usually + `<__media__>`. Model chat templates may render media as model-specific tags + such as ``, `<|image|>`, `[IMG]`, `<|image_pad|>`, or as the original + URL/data URI. This stage converts those rendered forms into the canonical + MTMD marker and validates that the final marker count matches the number of + media payloads. + """ + media_marker = self.media_marker + if not media_marker: + raise ValueError( + f"{self.log_prefix}(_replace_media_placeholders): media marker must not be empty." + ) + + # 1. Replace known template-specific media tags first. + # + # This handles templates that render placeholders such as: + # , <|image|>, [IMG], <|image_pad|>, <|media_pad|>, etc. + for tag in self._chat_format_parser_tags: + if tag in text: + text = text.replace(tag, media_marker) + + # 2. Replace rendered media URLs/data URIs. + # + # This handles templates that directly render the original image/audio/video + # URL or data URI instead of a symbolic placeholder. + for item in media_items: + url = item.get("url", "") + if url and url in text: + text = text.replace(url, media_marker, 1) + + # 3. Validate after all normalization is complete. + marker_count = text.count(media_marker) + media_count = len(media_items) + + if marker_count != media_count: + raise ValueError( + f"{self.log_prefix}(_replace_media_placeholders): media marker mismatch\n" + f"- marker_count={marker_count}\n" + f"- media_count={media_count}\n" + f"- media_marker={media_marker!r}\n" + "Each media item must render to exactly one MTMD media marker. " + "Check whether the chat template rendered both a media tag and the " + "original URL/data URI, or failed to render a media placeholder." + ) + + return text + + def _render_and_replace_media( + self, + messages: List[llama_types.ChatCompletionRequestMessage], + media_items: List[Dict[str, str]], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + add_generation_prompt: bool = True, + ) -> str: + """ + Render chat messages and normalize rendered media placeholders into MTMD markers. + """ + text = self._render_mtmd_prompt( + messages=messages, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + add_generation_prompt=add_generation_prompt, + ) + + return self._replace_media_placeholders( + text=text, + media_items=media_items, + ) + + def _validate_mtmd_inputs( + self, + *, + text: str, + bitmaps: Optional[List[Any]] = None, + ) -> None: + """ + Validate Python-side MTMD tokenizer inputs before calling mtmd_tokenize. + + This mirrors the most important checks in llama.cpp mtmd_tokenizer: + - mtmd context must be initialized + - rendered text must be a string + - media marker must not be empty + - media marker count must match bitmap count + - bitmap entries must not be None + + Pure text input is valid: + bitmaps is None or [] + marker_count == 0 + """ + if self.mtmd_ctx is None: + raise ValueError( + f"{self.log_prefix}(_validate_mtmd_inputs): mtmd context not initialized." + ) + + if not isinstance(text, str): + raise TypeError( + f"{self.log_prefix}(_validate_mtmd_inputs): text must be str, " + f"got {type(text).__name__}." + ) + + if not self.media_marker: + raise ValueError( + f"{self.log_prefix}(_validate_mtmd_inputs): media marker must not be empty." + ) + + if bitmaps is None: + bitmaps = [] + + marker_count = text.count(self.media_marker) + bitmap_count = len(bitmaps) + + if marker_count != bitmap_count: + raise ValueError( + f"{self.log_prefix}(_validate_mtmd_inputs): media marker mismatch\n" + f"- marker_count={marker_count}\n" + f"- bitmap_count={bitmap_count}\n" + f"- media_marker={self.media_marker!r}\n" + "The rendered prompt must contain exactly one media marker per decoded media input." + ) + + for i, bitmap in enumerate(bitmaps): + if bitmap is None: + raise ValueError( + f"{self.log_prefix}(_validate_mtmd_inputs): bitmap[{i}] is None." + ) + + def _mtmd_tokenize( + self, + llama: "llama_core.Llama", + text: str, + bitmaps: Optional[List[Any]] = None, + chunks: Optional[Any] = None, + ) -> Any: + """ + Perform MTMD hybrid tokenization. + + This function isolates the llama.cpp mtmd_tokenize call + so that prompt construction logic is decoupled from runtime execution. + + It guarantees: + - stable interface for future async/batch decoding + - isolated error handling for tokenizer failures + - clean separation between prompt building and C++ binding + - strict Python-side marker/bitmap validation before native tokenization + + Pure text input is valid: + bitmaps is None or [] + marker_count == 0 + """ + if bitmaps is None: + bitmaps = [] + + self._validate_mtmd_inputs( + text=text, + bitmaps=bitmaps, + ) + + if chunks is None: + chunks = self._mtmd_cpp.mtmd_input_chunks_init() + if chunks is None: + raise ValueError( + f"{self.log_prefix}(_mtmd_tokenize): failed to init mtmd_input_chunks" + ) + + input_text = self._mtmd_cpp.mtmd_input_text() + encoded_text = text.encode("utf-8") + input_text.text = ctypes.c_char_p(encoded_text) + input_text.text_len = len(encoded_text) + input_text.add_special = (llama.n_tokens == 0) + input_text.parse_special = True + + n_bitmaps = len(bitmaps) + + if n_bitmaps > 0: + bitmap_array = ( + self._mtmd_cpp.mtmd_bitmap_p_ctypes * n_bitmaps + )(*bitmaps) + else: + bitmap_array = None + + result = self._mtmd_cpp.mtmd_tokenize( + self.mtmd_ctx, + chunks, + ctypes.byref(input_text), + bitmap_array, + n_bitmaps, + ) + + if result != 0: + marker_count = text.count(self.media_marker) + raise ValueError( + f"{self.log_prefix}(_mtmd_tokenize): mtmd_tokenize failed\n" + f"- result={result}\n" + f"- text_len={len(text)}\n" + f"- marker_count={marker_count}\n" + f"- n_bitmaps={n_bitmaps}\n" + f"- supports_vision={self.is_support_vision}\n" + f"- supports_audio={self.is_support_audio}\n" + f"- supports_video={self.is_support_video}\n" + "Possible causes: marker/bitmap mismatch, invalid image/audio data, " + "unsupported vision/audio projector, failed media preprocessing, " + "or text tokenization failure." + ) + + return chunks + def _process_mtmd_prompt( self, llama: llama_core.Llama, @@ -546,34 +812,25 @@ def _process_mtmd_prompt( messages = [{"role": "system", "content": self.DEFAULT_SYSTEM_MESSAGE}] + messages media_items = self._get_media_items(messages) - media_marker = self.media_marker - # 2. Render the chat template and replace actual URLs with C++ media markers - text = self.chat_template.render( + # 2. Render chat template and normalize media placeholders to MTMD markers. + text = self._render_and_replace_media( messages=messages, - add_generation_prompt=add_generation_prompt, - eos_token=self.mtmd_eos_token, - bos_token=self.mtmd_bos_token, + media_items=media_items, functions=functions, function_call=function_call, tools=tools, tool_choice=tool_choice, - **getattr(self, 'extra_template_arguments', {}) + add_generation_prompt=add_generation_prompt, ) - for tag in self._chat_format_parser_tags: - if tag not in text: - continue - - text = text.replace(tag, media_marker) - - # Replace image_url by media_marker in text - for item in media_items: - text = text.replace(item["url"], media_marker) - if self.verbose: - print(f"{self.log_prefix}(_process_mtmd_prompt): Rendered prompt length: {len(text)} chars, Media count: {len(media_items)}.", file=sys.stderr) - print(f"{self.log_prefix}(_process_mtmd_prompt): Rendered prompt: {text}", file=sys.stderr) + print( + f"{self.log_prefix}(_process_mtmd_prompt): " + f"Rendered prompt length: {len(text)} chars, Media count: {len(media_items)}.\n" + f"Rendered prompt: {text}", + file=sys.stderr, + ) # 3. Pre-allocate bitmap array to guarantee chronological order during concurrent decoding bitmaps = [None] * len(media_items) @@ -614,29 +871,13 @@ def _create_bitmap_func(idx: int, item: dict): # If there are no images, set the bitmaps to empty. bitmaps = [] - # 4. Initialize mtmd_input_chunks - input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode('utf-8') - input_text.add_special = (llama.n_tokens == 0) - input_text.parse_special = True - - chunks = self._mtmd_cpp.mtmd_input_chunks_init() - if chunks is None: - raise ValueError(f"{self.log_prefix}(mtmd_input_chunks_init): Failed to initialize mtmd_input_chunks.") - - # 5. Hybrid Tokenization (Text + Media binding) - if len(bitmaps) > 0: - bitmap_array = (self._mtmd_cpp.mtmd_bitmap_p_ctypes * len(bitmaps))(*bitmaps) - result = self._mtmd_cpp.mtmd_tokenize( - self.mtmd_ctx, chunks, ctypes.byref(input_text), bitmap_array, len(bitmaps) - ) - else: - result = self._mtmd_cpp.mtmd_tokenize( - self.mtmd_ctx, chunks, ctypes.byref(input_text), None, 0 - ) - - if result != 0: - raise ValueError(f"{self.log_prefix}(mtmd_tokenize): Unable to tokenize prompt, res = {result}.") + # 4. Hybrid Tokenization (Text + Media) + chunks = self._mtmd_tokenize( + llama=llama, + text=text, + bitmaps=bitmaps, + chunks=None, + ) # Video helper contexts only need to stay alive until mtmd_tokenize() completes. if video_cleanup: @@ -644,7 +885,7 @@ def _create_bitmap_func(idx: int, item: dict): self._mtmd_cpp.mtmd_helper_video_free(video_ctx) video_cleanup.clear() - # 6. Virtual Token Ledger Construction + # 5. Virtual Token Ledger Construction full_prompt_ids = [] chunk_token_spans = [] current_idx = 0 @@ -710,6 +951,15 @@ def _create_bitmap_func(idx: int, item: dict): else: raise TypeError(f"{self.log_prefix}(mtmd_input_chunk_get_type): Invalid chunk type, chunk_type = {chunk_type}.") + if media_items_cur != media_items_count: + raise RuntimeError( + f"{self.log_prefix}(_process_mtmd_prompt): not all media inputs were consumed by MTMD chunks\n" + f"- consumed={media_items_cur}\n" + f"- media_items={media_items_count}\n" + "This usually means the rendered prompt did not produce enough media chunks, " + "or the chat template/media marker normalization is incorrect." + ) + return full_prompt_ids, chunk_token_spans, chunks, bitmap_cleanup except Exception as e: @@ -861,7 +1111,12 @@ def __call__( if tokens_to_eval: if self.verbose: - print(f"{self.log_prefix}(__call__): Evaluating TEXT chunk ({len(tokens_to_eval)} tokens) at pos {llama.n_tokens}...", file=sys.stderr) + print( + f"{self.log_prefix}(__call__): Evaluating TEXT chunk " + f"({len(tokens_to_eval)} tokens) at pos {llama.n_tokens}...", + file=sys.stderr, + ) + # Text evaluation delegates shift and chunking to native llama.eval llama.eval(tokens_to_eval) n_past = llama.n_tokens @@ -1419,8 +1674,18 @@ def _resolve_chat_format(self, llama: llama_core.Llama) -> str: self.chat_format = chat_format return chat_format - def __call__(self, **kwargs): - llama = kwargs["llama"] + def _ensure_chat_template( + self, + llama: llama_core.Llama, + ) -> None: + """ + Resolve and analyze chat template once. + + Chat template metadata is static for a model instance, + so it should not be recomputed for every request. + """ + if self._template_initialized: + return self._resolve_chat_format(llama) @@ -1433,7 +1698,18 @@ def __call__(self, **kwargs): "a model that provides tokenizer.chat_template metadata." ) - self._chat_format_parser_tags = [tag for tag in self.KNOWN_MEDIA_TAGS if tag in self.chat_format] + self._chat_format_parser_tags = [ + tag + for tag in self.KNOWN_MEDIA_TAGS + if tag in self.chat_format + ] + + self._template_initialized = True + + def __call__(self, **kwargs): + llama = kwargs["llama"] + + self._ensure_chat_template(llama) if self.verbose: print(f"{self.log_prefix} - Start processing", file=sys.stderr) @@ -2306,7 +2582,9 @@ class Gemma4ChatHandler(MTMDChatHandler): " }\n" "{%- endmacro -%}\n" "{%- macro format_argument(argument, escape_keys=True) -%}\n" - " {%- if argument is string -%}\n" + " {%- if argument is none -%}\n" + " {{- 'null' -}}\n" + " {%- elif argument is string -%}\n" " {{- '<|\"|>' + argument + '<|\"|>' -}}\n" " {%- elif argument is boolean -%}\n" " {{- 'true' if argument else 'false' -}}\n" @@ -2362,18 +2640,21 @@ class Gemma4ChatHandler(MTMDChatHandler): " {{- '' -}}\n" "{%- endmacro -%}\n" "\n" + "{#- ===== SETUP ===== -#}" "{%- set ns = namespace(prev_message_type=None) -%}\n" "{%- set loop_messages = messages -%}\n" + "{%- set enable_thinking = enable_thinking | default(false) -%}\n" + "{%- set preserve_thinking = preserve_thinking | default(false) -%}\n" "{{- bos_token -}}\n" "{#- Handle System/Tool Definitions Block -#}\n" - "{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}\n" + "{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%}\n" " {{- '<|turn>system\\n' -}}\n" " {#- Inject Thinking token at the very top of the FIRST system turn -#}\n" - " {%- if enable_thinking is defined and enable_thinking -%}\n" + " {%- if enable_thinking -%}\n" " {{- '<|think|>\\n' -}}\n" " {%- set ns.prev_message_type = 'think' -%}\n" " {%- endif -%}\n" - " {%- if messages[0]['role'] in ['system', 'developer'] -%}\n" + " {%- if messages and messages[0]['role'] in ['system', 'developer'] -%}\n" " {%- if messages[0]['content'] is string -%}\n" " {{- messages[0]['content'] | trim -}}\n" " {%- elif messages[0]['content'] is sequence -%}\n" @@ -2407,31 +2688,21 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- if message['role'] != 'tool' -%}\n" " {%- set ns.prev_message_type = None -%}\n" " {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n" - " {#- Detect continuation: suppress duplicate <|turn>model when previous non-tool message was also assistant -#}\n" - " {%- set prev_nt = namespace(role=None, found=false) -%}\n" - " {%- if loop.index0 > 0 -%}\n" - " {%- for j in range(loop.index0 - 1, -1, -1) -%}\n" - " {%- if not prev_nt.found -%}\n" - " {%- if loop_messages[j]['role'] != 'tool' -%}\n" - " {%- set prev_nt.role = loop_messages[j]['role'] -%}\n" - " {%- set prev_nt.found = true -%}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- endif -%}\n" - " {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}\n" + "{#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#}\n" + "{%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%}\n" " {%- if not continue_same_model_turn -%}\n" " {{- '<|turn>' + role + '\\n' }}\n" " {%- endif -%}\n" "\n" " {#- Render reasoning/reasoning_content as thinking channel -#}\n" " {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n" - " {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}\n" - " {{- '<|channel>thought\\n' + thinking_text + '\\n' -}}\n" + " {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%}\n" + " {%- if thinking_text and thinking_gate -%}\n" + " {{- '<|channel>thought\n' + thinking_text + '\n' -}}\n" " {%- endif -%}\n" "\n" " {%- if message.get('tool_calls') -%}\n" - " {%- for tool_call in message['tool_calls'] -%}\n" + " {%- for tool_call in message.get('tool_calls') -%}\n" " {%- set function = tool_call['function'] -%}\n" " {{- '<|tool_call>call:' + function['name'] + '{' -}}\n" " {%- if function['arguments'] is mapping -%}\n" @@ -2441,8 +2712,13 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set ns_args.found_first = true -%}\n" " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n" " {%- endfor -%}\n" - " {%- elif function['arguments'] is string -%}\n" - " {{- function['arguments'] -}}\n" + " {%- elif function['arguments'] is none -%}\n" + " {%- else -%}\n" + " {{- raise_exception(\n" + " \"chat_template: tool_calls[].function.arguments must be a \"\n" + " \"JSON object (mapping), not a string. Deserialize arguments \"\n" + " \"before passing to the template.\"\n" + " ) -}}\n" " {%- endif -%}\n" " {{- '}' -}}\n" " {%- endfor -%}\n" @@ -2452,8 +2728,8 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set ns_tr_out = namespace(flag=false) -%}\n" " {%- if message.get('tool_responses') -%}\n" " {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n" - " {%- for tool_response in message['tool_responses'] -%}\n" - " {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}\n" + " {%- for tool_response in message.get('tool_responses') -%}\n" + " {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}}\n" " {%- set ns_tr_out.flag = true -%}\n" " {%- set ns.prev_message_type = 'tool_response' -%}\n" " {%- endfor -%}\n" @@ -2467,8 +2743,8 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- else -%}\n" " {%- set follow = loop_messages[k] -%}\n" " {#- Resolve tool_call_id to function name -#}\n" - " {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}\n" - " {%- for tc in message['tool_calls'] -%}\n" + " {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%}\n" + " {%- for tc in message.get('tool_calls') -%}\n" " {%- if tc.get('id') == follow.get('tool_call_id') -%}\n" " {%- set ns_tname.name = tc['function']['name'] -%}\n" " {%- endif -%}\n" @@ -2486,9 +2762,14 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endfor -%}\n" " {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n" " {%- for part in tool_body -%}\n" - " {%- if part.get('type') == 'image_url' -%}\n" - " {%- set url_val = part['image_url'] if part['image_url'] is string else part['image_url']['url'] -%}\n" - " {{- '<|image|>' + url_val -}}\n" + " {%- if part.get('type') in ['image', 'image_url'] -%}\n" + " {%- if part.get('type') == 'image_url' -%}\n" + " {%- set url_val = part['image_url'] if part['image_url'] is string else part['image_url']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- elif part.get('type') == 'image' -%}\n" + " {%- set url_val = part['image'] if part['image'] is string else part['image']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- endif -%}\n" " {%- elif part.get('type') in ['audio_url', 'input_audio'] -%}\n" " {%- if part.get('type') == 'audio_url' -%}\n" " {%- set audio_val = part['audio_url'] if part['audio_url'] is string else part['audio_url']['url'] -%}\n" @@ -2497,9 +2778,14 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set audio_val = part['input_audio'] if part['input_audio'] is string else ('data:audio/' + part['input_audio']['format'] + ';base64,' + part['input_audio']['data']) -%}\n" " {{- '<|audio|>' + audio_val -}}\n" " {%- endif -%}\n" - # " {%- elif part.get('type') == 'video_url' -%}\n" - # " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" - # " {{- '<|video|>' + video_val -}}\n" + " {%- elif part.get('type') in ['video', 'video_url'] -%}\n" + " {%- if part.get('type') == 'video_url' -%}\n" + " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- elif part.get('type') == 'video' -%}\n" + " {%- set video_val = part['video'] if part['video'] is string else part['video']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- endif -%}\n" " {%- endif -%}\n" " {%- endfor -%}\n" " {%- else -%}\n" @@ -2512,38 +2798,45 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endif -%}\n" "\n" " {%- set captured_content -%}\n" - " {%- if message['content'] is string -%}\n" + " {%- if message.get('content') is string -%}\n" " {%- if role == 'model' -%}\n" " {{- strip_thinking(message['content']) -}}\n" " {%- else -%}\n" " {{- message['content'] | trim -}}\n" " {%- endif -%}\n" - " {%- elif message['content'] is sequence -%}\n" + " {%- elif message.get('content') is sequence -%}\n" " {%- for item in message['content'] -%}\n" - " {%- if item['type'] == 'text' -%}\n" + " {%- if item.get('type') == 'text' -%}\n" " {%- if role == 'model' -%}\n" " {{- strip_thinking(item['text']) -}}\n" " {%- else -%}\n" " {{- item['text'] | trim -}}\n" " {%- endif -%}\n" - " {%- elif item['type'] == 'image_url' -%}\n" - " {%- set url_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" - " {{- '<|image|>' + url_val -}}\n" - " {%- set ns.prev_message_type = 'image' -%}\n" - " {%- elif item['type'] in ['audio_url', 'input_audio'] -%}\n" - " {%- if item['type'] == 'audio_url' -%}\n" + " {%- elif item.get('type') in ['image', 'image_url'] -%}\n" + " {%- if item.get('type')== 'image_url' -%}\n" + " {%- set url_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- elif item.get('type') == 'image' -%}\n" + " {%- set url_val = item['image'] if item['image'] is string else item['image']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- endif -%}\n" + " {%- elif item.get('type') in ['audio_url', 'input_audio'] -%}\n" + " {%- if item.get('type') == 'audio_url' -%}\n" " {%- set audio_val = item['audio_url'] if item['audio_url'] is string else item['audio_url']['url'] -%}\n" " {{- '<|audio|>' + audio_val -}}\n" - " {%- elif item['type'] == 'input_audio' -%}\n" + " {%- elif item.get('type') == 'input_audio' -%}\n" " {%- set audio_val = item['input_audio'] if item['input_audio'] is string else ('data:audio/' + item['input_audio']['format'] + ';base64,' + item['input_audio']['data']) -%}\n" " {{- '<|audio|>' + audio_val -}}\n" " {%- endif -%}\n" - " {%- set ns.prev_message_type = 'audio' -%}\n" + " {%- elif item.get('type') in ['video', 'video_url'] -%}\n" + " {%- if item.get('type') == 'video_url' -%}\n" + " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- elif item.get('type') == 'video' -%}\n" + " {%- set video_val = part['video'] if part['video'] is string else part['video']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- endif -%}\n" " {%- endif -%}\n" - # " {%- elif item['type'] == 'video_url' -%}\n" - # " {%- set video_val = item['video_url'] if item['video_url'] is string else item['video_url']['url'] -%}\n" - # " {{- '<|video|>' + video_val -}}\n" - # " {%- set ns.prev_message_type = 'video' -%}\n" " {%- endfor -%}\n" " {%- endif -%}\n" " {%- endset -%}\n" @@ -2551,20 +2844,43 @@ class Gemma4ChatHandler(MTMDChatHandler): " {{- captured_content -}}\n" " {%- set has_content = captured_content | trim | length > 0 -%}\n" "\n" + " {#- Forward-scan: find next non-tool message role for continuation detection -#}\n" + " {%- set next_nt = namespace(role=None, found=false) -%}\n" + " {%- for j in range(loop.index0 + 1, loop_messages | length) -%}\n" + " {%- if not next_nt.found -%}\n" + " {%- if loop_messages[j]['role'] != 'tool' -%}\n" + " {%- set next_nt.role = loop_messages[j]['role'] -%}\n" + " {%- set next_nt.found = true -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + + " {%- set continues_into_next = (\n" + " role == 'model'\n" + " and next_nt.role == 'assistant'\n" + " and (not message.get('tool_calls') or ns_tr_out.flag)\n" + " ) -%}\n" + "\n" " {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n" " {{- '<|tool_response>' -}}\n" - " {%- elif not (ns_tr_out.flag and not has_content) -%}\n" + " {%- elif continues_into_next -%}\n" + " {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%}\n" " {{- '\\n' -}}\n" " {%- endif -%}\n" + "\n" + " {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#}\n" + " {%- set ns.prev_non_tool_role = message['role'] -%}\n" " {%- endif -%}\n" "{%- endfor -%}\n" "\n" "{%- if add_generation_prompt -%}\n" " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n" - " {{- '<|turn>model\\n' -}}\n" - " {%- if not enable_thinking | default(false) -%}\n" - " {{- '<|channel>thought\\n' -}}\n" + " {{- '<|turn>model\n' -}}\n" + " {%- if not enable_thinking -%}\n" + " {{- '<|channel>thought\n' -}}\n" " {%- endif -%}\n" + " {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}\n" + " {{- '<|channel>thought\n' -}}\n" " {%- endif -%}\n" "{%- endif -%}\n" ) diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 37b041ee87..56451ea251 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -427,7 +427,7 @@ class ChatCompletionRequestAssistantMessageFunctionCall(TypedDict): class ChatCompletionRequestAssistantMessage(TypedDict): """Messages sent by the model in response to user messages.""" role: Literal["assistant"] - name: Optional[str] + name: NotRequired[Optional[str]] content: NotRequired[Optional[str]] refusal: NotRequired[Optional[str]] tool_calls: NotRequired[ChatCompletionMessageToolCalls] diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 27a1a56d8d..fcfaa86ee7 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -178,12 +178,14 @@ class mtmd_pos_type(enum.IntEnum): # struct mtmd_input_text { # const char * text; +# size_t text_len; # bool add_special; # bool parse_special; # }; class mtmd_input_text(Structure): _fields_ = [ ("text", c_char_p), + ("text_len", c_size_t), ("add_special", c_bool), ("parse_special", c_bool), ] diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 6b3fd1dd15..0d509bbcf9 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -294,12 +294,10 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: # Model Params n_gpu_layers=settings.n_gpu_layers, split_mode=settings.split_mode, + load_mode=settings.load_mode, main_gpu=settings.main_gpu, tensor_split=settings.tensor_split, vocab_only=settings.vocab_only, - use_mmap=settings.use_mmap, - use_direct_io=settings.use_direct_io, - use_mlock=settings.use_mlock, check_tensors=settings.check_tensors, use_extra_bufts=settings.use_extra_bufts, no_host=settings.no_host, diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 350ccc2323..62ce3b5044 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -32,8 +32,12 @@ class ModelSettings(BaseSettings): ) split_mode: int = Field( default=llama_cpp.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, - description="The split mode to use.", + description="how to split the model across multiple GPUs", ) + load_mode: int = Field( + default=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP, + description="how to load the model", + ) main_gpu: int = Field( default=0, ge=0, @@ -46,18 +50,6 @@ class ModelSettings(BaseSettings): vocab_only: bool = Field( default=False, description="Whether to only return the vocabulary." ) - use_mmap: bool = Field( - default=True, - description="Enable mmap to use filesystem cache.", - ) - use_direct_io: bool = Field( - default=False, - description="Use direct io, takes precedence over use_mmap.", - ) - use_mlock: bool = Field( - default=False, - description="Use mlock for force system to keep model in RAM", - ) check_tensors: bool = Field( default=False, description="Validate model tensor data.", diff --git a/tests/test_llama.py b/tests/test_llama.py index bf075845e0..d0053feaba 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -19,6 +19,158 @@ MODEL = "./vendor/llama.cpp/models/ggml-vocab-llama-spm.gguf" +def test_model_init_frees_native_model_when_vocab_lookup_fails(monkeypatch): + native_model_handle = object() + freed_model_handles = [] + + def model_path_exists(_path): + return True + + def load_native_model(_path, _params): + return native_model_handle + + def fail_to_get_model_vocab(_model_handle): + return None + + def record_model_free(model_handle): + freed_model_handles.append(model_handle) + + monkeypatch.setattr(internals.os.path, "exists", model_path_exists) + monkeypatch.setattr( + internals.llama_cpp, + "llama_model_load_from_file", + load_native_model, + ) + monkeypatch.setattr( + internals.llama_cpp, + "llama_model_get_vocab", + fail_to_get_model_vocab, + ) + monkeypatch.setattr( + internals.llama_cpp, + "llama_model_free", + record_model_free, + ) + + with pytest.raises(ValueError, match="Failed to get vocab"): + internals.LlamaModel( + path_model="model.gguf", + params=object(), + verbose=False, + ) + + assert freed_model_handles == [native_model_handle] + + +def test_batch_init_frees_native_batch_when_validation_fails(monkeypatch): + class InvalidMixedNativeBatch: + token = object() + embd = object() + + invalid_mixed_batch = InvalidMixedNativeBatch() + freed_batch_handles = [] + + def allocate_invalid_mixed_batch(_n_tokens, _embd, _n_seq_max): + return invalid_mixed_batch + + def record_batch_free(batch_handle): + freed_batch_handles.append(batch_handle) + + monkeypatch.setattr( + internals.llama_cpp, + "llama_batch_init", + allocate_invalid_mixed_batch, + ) + monkeypatch.setattr( + internals.llama_cpp, + "llama_batch_free", + record_batch_free, + ) + + with pytest.raises(RuntimeError, match="expected batch.token to be NULL"): + internals.LlamaBatch( + n_tokens=1, + embd=1, + n_seq_max=1, + mixed=True, + verbose=False, + ) + + assert freed_batch_handles == [invalid_mixed_batch] + + +def test_context_close_releases_parent_references(): + context = internals.LlamaContext.__new__(internals.LlamaContext) + context.ctx = None + context.model = object() + context.params = object() + context._exit_stack = None + + context.close() + context.close() # Closing an already closed context must be a no-op. + + assert context.model is None + assert context.params is None + + +def test_sampling_context_partial_init_can_close_idempotently(monkeypatch): + closed_resources = [] + + class MinimalModelForSampling: + model = object() + verbose = False + + def n_vocab(self): + return 8 + + class TrackedTokenDataArray: + def __init__(self, *, n_vocab): + assert n_vocab == 8 + + def close(self): + closed_resources.append("token-data") + + class TrackedSamplerChain: + def close(self): + closed_resources.append("sampler-chain") + + def get_sampling_vocab(_model_handle): + return object() + + def fail_sampler_chain_build(_sampling_context): + raise RuntimeError("sampler chain build failed") + + monkeypatch.setattr(internals, "LlamaTokenDataArray", TrackedTokenDataArray) + monkeypatch.setattr(internals, "LlamaSampler", TrackedSamplerChain) + monkeypatch.setattr( + internals.llama_cpp, + "llama_model_get_vocab", + get_sampling_vocab, + ) + monkeypatch.setattr( + internals.LlamaSamplingContext, + "_build_sampler_chain", + fail_sampler_chain_build, + ) + + sampling_context = internals.LlamaSamplingContext.__new__( + internals.LlamaSamplingContext + ) + with pytest.raises(RuntimeError, match="sampler chain build failed"): + sampling_context.__init__( + params=internals.LlamaSamplingParams(), + model=MinimalModelForSampling(), + ) + + sampling_context.close() + sampling_context.close() # Closing an already closed context must be a no-op. + + assert closed_resources == ["sampler-chain", "token-data"] + assert sampling_context.model is None + assert sampling_context.params is None + assert sampling_context.vocab is None + + def test_llama_cpp_version(): assert llama_cpp.__version__ @@ -64,6 +216,32 @@ def test_llama_cpp_tokenization(): assert text == llama.detokenize(tokens) +def test_llama_batch_seq_id_error_guidance(): + """Sequence-capacity errors should explain how to fix parallel batching.""" + batch = internals.LlamaBatch( + n_tokens=2, + embd=0, + n_seq_max=1, + verbose=False, + ) + try: + with pytest.raises(ValueError) as exc_info: + batch.add_sequence( + token_array=[1], + pos_array=[0], + seq_ids=[1], + logits_array=[True], + ) + + message = str(exc_info.value) + assert "n_seq_max=1" in message + assert "valid IDs are 0 through 0" in message + assert "n_seq_max>=2" in message + assert "LlamaEmbedding" in message + finally: + batch.close() + + @pytest.fixture def llama_cpp_model_path(): """Fixture to download a real GGUF model for integration tests.""" @@ -82,9 +260,6 @@ def test_real_model(llama_cpp_model_path): # 1. Setup Model Parameters params = llama_cpp.llama_model_default_params() - params.use_mmap = llama_cpp.llama_supports_mmap() - params.use_direct_io = False - params.use_mlock = llama_cpp.llama_supports_mlock() params.check_tensors = False # 2. Load the Model @@ -365,16 +540,75 @@ def no_e_processor(input_ids, scores): def test_real_llama_embeddings(llama_cpp_model_path): """ - Test Embedding Generation. - Verifies that the model can produce vector embeddings. + Test embedding generation through the specialized LlamaEmbedding class. """ model = LlamaEmbedding( - model_path=llama_cpp_model_path, - n_ctx=32, - n_batch=32, - n_ubatch=32, - pooling_type=LLAMA_POOLING_TYPE_NONE) - # Smoke test for now - embeddings = model.embed("Hello, world!") - assert isinstance(embeddings, list) - assert len(embeddings) > 0 + model_path=llama_cpp_model_path, + n_ctx=32, + n_batch=32, + n_ubatch=32, + pooling_type=LLAMA_POOLING_TYPE_NONE, + ) + try: + # The inherited n_seq_max=1 processes this list as three streaming + # decode batches instead of assigning an invalid seq_id. + embeddings = model.embed(["Hello", "world", "embedding"]) + assert isinstance(embeddings, list) + assert len(embeddings) == 3 + assert all(len(embedding) > 0 for embedding in embeddings) + finally: + model.close() + + +def test_real_llama_base_embedding_api(llama_cpp_model_path): + """ + Test the maintained embedding API on the standard Llama class. + + Covers pre-tokenized batching, normalization, separator-based string + batching, token counts, and the OpenAI-compatible response wrapper. + """ + model = llama_cpp.Llama( + model_path=llama_cpp_model_path, + embeddings=True, + n_ctx=32, + n_batch=32, + n_ubatch=32, + n_seq_max=2, + kv_unified=True, + pooling_type=LLAMA_POOLING_TYPE_NONE, + verbose=False, + ) + + try: + token_inputs = [ + model.tokenize(b"Hello"), + model.tokenize(b"world"), + ] + embeddings, token_count = model.embed( + token_inputs, + normalize=True, + return_count=True, + ) + + assert len(embeddings) == len(token_inputs) + assert token_count == sum(map(len, token_inputs)) + assert len(embeddings[0]) == len(token_inputs[0]) + assert np.linalg.norm(embeddings[0][0]) == pytest.approx(1.0) + + split_embeddings = model.embed( + "Hello\nworld", + separator="\n", + normalize=False, + ) + assert len(split_embeddings) == 2 + + response = model.create_embedding( + ["Hello", "world"], + normalize=2, + ) + assert response["object"] == "list" + assert len(response["data"]) == 2 + assert response["usage"]["prompt_tokens"] > 0 + assert response["usage"]["total_tokens"] == response["usage"]["prompt_tokens"] + finally: + model.close() diff --git a/tests/test_llama_chat_format.py b/tests/test_llama_chat_format.py index f031bf72b7..4860a4c7d6 100644 --- a/tests/test_llama_chat_format.py +++ b/tests/test_llama_chat_format.py @@ -1,89 +1,297 @@ -import json - import jinja2 +import numpy as np +import pytest + +from llama_cpp.llama_chat_format import Jinja2ChatFormatter + +QWEN35_EOS_TOKEN = "<|im_end|>" + +# A compact Qwen3.5-style template keeps these tests independent of model files. +QWEN35_CHAT_TEMPLATE = r""" +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, is_system=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if "image" in item or "image_url" in item or item.type == "image" %} + {%- if is_system %} + {{- raise_exception("System message cannot contain images.") }} + {%- endif %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %} + {{- "Picture " ~ image_count.value ~ ": " }} + {%- endif %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif "video" in item or item.type == "video" %} + {%- if is_system %} + {{- raise_exception("System message cannot contain videos.") }} + {%- endif %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %} + {{- "Video " ~ video_count.value ~ ": " }} + {%- endif %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif "text" in item %} + {{- item.text }} + {%- else %} + {{- raise_exception("Unexpected item type in content.") }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- "" }} + {%- else %} + {{- raise_exception("Unexpected content type.") }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception("No messages provided.") }} +{%- endif %} +{%- if tools %} + {{- "<|im_start|>system\n# Tools\n\n" }} + {%- for tool in tools %} + {{- "\n" ~ (tool | tojson) }} + {%- endfor %} + {{- "\n<|im_end|>\n" }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content( + message.content, message.role == "system" + ) | trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception("System message must be at the beginning.") }} + {%- endif %} + {{- "<|im_start|>system\n" ~ content ~ "<|im_end|>\n" }} + {%- elif message.role == "user" %} + {{- "<|im_start|>user\n" ~ content ~ "<|im_end|>\n" }} + {%- elif message.role == "assistant" %} + {{- "<|im_start|>assistant\n" }} + {%- if message.reasoning_content is string %} + {{- "\n" ~ (message.reasoning_content | trim) + ~ "\n\n\n" }} + {%- endif %} + {{- content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- set call = tool_call.function %} + {{- "\n\n\n\n" }} + {%- for name, value in call.arguments | items %} + {{- "\n" ~ value + ~ "\n\n" }} + {%- endfor %} + {{- "\n" }} + {%- endfor %} + {%- endif %} + {{- "<|im_end|>\n" }} + {%- elif message.role == "tool" %} + {{- "<|im_start|>user\n\n" ~ content + ~ "\n<|im_end|>\n" }} + {%- else %} + {{- raise_exception("Unexpected message role.") }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- "<|im_start|>assistant\n" }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- "\n\n\n\n" }} + {%- else %} + {{- "\n" }} + {%- endif %} +{%- endif %} +""" + + +@pytest.fixture() +def qwen35_formatter() -> Jinja2ChatFormatter: + return Jinja2ChatFormatter( + template=QWEN35_CHAT_TEMPLATE, + eos_token=QWEN35_EOS_TOKEN, + bos_token="", + add_generation_prompt=True, + ) + -from llama_cpp import ( - ChatCompletionRequestUserMessage, +def test_qwen35_basic_conversation(qwen35_formatter: Jinja2ChatFormatter): + response = qwen35_formatter( + messages=[ + {"role": "system", "content": "Be concise."}, + {"role": "user", "content": "Hello"}, + ], + enable_thinking=False, + ) + + assert response.prompt == ( + "<|im_start|>system\n" + "Be concise.<|im_end|>\n" + "<|im_start|>user\n" + "Hello<|im_end|>\n" + "<|im_start|>assistant\n" + "\n\n\n\n" + ) + assert response.stop == [QWEN35_EOS_TOKEN] + assert response.added_special is True + + +@pytest.mark.parametrize( + ("enable_thinking", "expected_suffix"), + [ + (True, "<|im_start|>assistant\n\n"), + (False, "<|im_start|>assistant\n\n\n\n\n"), + ], ) -import llama_cpp.llama_types as llama_types -import llama_cpp.llama_chat_format as llama_chat_format - -from llama_cpp.llama_chat_format import hf_tokenizer_config_to_chat_formatter - -def test_mistral_instruct(): - chat_template = "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}" - chat_formatter = jinja2.Template(chat_template) - messages = [ - llama_types.ChatCompletionRequestUserMessage(role="user", content="Instruction"), - llama_types.ChatCompletionRequestAssistantMessage(role="assistant", content="Model answer"), - llama_types.ChatCompletionRequestUserMessage(role="user", content="Follow-up instruction"), - ] - response = llama_chat_format.format_mistral_instruct( - messages=messages, +def test_qwen35_generation_prompt_thinking_modes( + qwen35_formatter: Jinja2ChatFormatter, + enable_thinking: bool, + expected_suffix: str, +): + response = qwen35_formatter( + messages=[{"role": "user", "content": "Solve this problem."}], + enable_thinking=enable_thinking, ) - prompt = ("" if response.added_special else "") + response.prompt - reference = chat_formatter.render( - messages=messages, - bos_token="", - eos_token="", + + assert response.prompt.endswith(expected_suffix) + + +def test_qwen35_multimodal_content(qwen35_formatter: Jinja2ChatFormatter): + # Qwen3.5 assigns separate sequence numbers to images and videos. + response = qwen35_formatter( + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "image.png"}, + }, + {"type": "text", "text": "Compare this with "}, + {"type": "video", "video": "video.mp4"}, + ], + } + ], + add_vision_id=True, + enable_thinking=False, ) - assert prompt == reference - - -mistral_7b_tokenizer_config = """{ - "add_bos_token": true, - "add_eos_token": false, - "added_tokens_decoder": { - "0": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "1": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "2": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - } - }, - "additional_special_tokens": [], - "bos_token": "", - "clean_up_tokenization_spaces": false, - "eos_token": "", - "legacy": true, - "model_max_length": 1000000000000000019884624838656, - "pad_token": null, - "sp_model_kwargs": {}, - "spaces_between_special_tokens": false, - "tokenizer_class": "LlamaTokenizer", - "unk_token": "", - "use_default_system_prompt": false, - "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}" -}""" - - -def test_hf_tokenizer_config_str_to_chat_formatter(): - tokenizer_config = json.loads(mistral_7b_tokenizer_config) - chat_formatter = hf_tokenizer_config_to_chat_formatter( - tokenizer_config + + assert "Picture 1: <|vision_start|><|image_pad|><|vision_end|>" in response.prompt + assert ( + "Compare this with Video 1: " + "<|vision_start|><|video_pad|><|vision_end|>" in response.prompt ) - chat_formatter_respoonse = chat_formatter( + assert response.prompt.count("<|vision_start|>") == 2 + assert response.prompt.endswith("<|im_start|>assistant\n\n\n\n\n") + + +def test_qwen35_tools_and_tool_history(qwen35_formatter: Jinja2ChatFormatter): + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + response = qwen35_formatter( messages=[ - ChatCompletionRequestUserMessage(role="user", content="Hello, world!"), - ] + {"role": "user", "content": "What is the weather?"}, + { + "role": "assistant", + "content": "I will check.", + "reasoning_content": "A weather lookup is required.", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "London"}, + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call-1", + "content": "Sunny, 28 C", + }, + ], + tools=tools, + enable_thinking=False, + ) + + # Tool calls and their responses use Qwen3.5's XML-like markers. + assert '"description": "Get the current weather for a city"' in response.prompt + assert "\nA weather lookup is required.\n" in response.prompt + assert ( + "\n" + "\n" + "\n" + "London\n" + "\n" + "\n" + "" in response.prompt + ) + assert "\nSunny, 28 C\n" in response.prompt + + +@pytest.mark.parametrize( + ("messages", "error"), + [ + ([], "No messages provided."), + ( + [ + {"role": "user", "content": "Hello"}, + {"role": "system", "content": "Too late"}, + ], + "System message must be at the beginning.", + ), + ( + [ + { + "role": "system", + "content": [ + { + "type": "image_url", + "image_url": {"url": "image.png"}, + } + ], + }, + {"role": "user", "content": "Hello"}, + ], + "System message cannot contain images.", + ), + ], +) +def test_qwen35_rejects_invalid_messages( + qwen35_formatter: Jinja2ChatFormatter, + messages, + error: str, +): + with pytest.raises(jinja2.TemplateError, match=error): + qwen35_formatter(messages=messages) + + +def test_qwen35_stop_token_ids(): + # Verify that model-specific stop token IDs terminate generation. + formatter = Jinja2ChatFormatter( + template=QWEN35_CHAT_TEMPLATE, + eos_token=QWEN35_EOS_TOKEN, + bos_token="", + stop_token_ids=[248044], ) + response = formatter(messages=[{"role": "user", "content": "Hello"}]) + + assert response.stopping_criteria is not None + criterion = response.stopping_criteria[0] + logits = np.empty(0, dtype=np.single) - assert chat_formatter_respoonse.prompt == ("[INST] Hello, world! [/INST]" "") + assert criterion(np.array([], dtype=np.intc), logits) is False + assert criterion(np.array([1, 248044], dtype=np.intc), logits) is True + assert criterion(np.array([1, 2], dtype=np.intc), logits) is False diff --git a/tools/abi/README.md b/tools/abi/README.md new file mode 100644 index 0000000000..ea509f2f7a --- /dev/null +++ b/tools/abi/README.md @@ -0,0 +1,164 @@ +# Cross-platform ABI inspection + +Author: **JamePeng** + +This repository-only tool inspects PE (`.dll`), ELF (`.so` and `.so.*`), and +Mach-O (`.dylib`) exports. Its primary purpose is to collect ctypes symbol +candidates and verify optional `llama_ext` bindings across MSVC, GCC/Clang, +and macOS builds. + +## Boundary and safety + +The tool is intentionally excluded from wheels: + +```toml +wheel.packages = ["llama_cpp"] +``` + +It is not imported by `llama_cpp`, has no installed command, and keeps LIEF +out of project dependencies. Run it only from a trusted source checkout. +LIEF parses native binaries, so do not scan untrusted artifacts. + +Install the maintainer-only dependency: + +```bash +python -m pip install lief +``` + +The tool and its documentation use the same MIT License as this repository. + +## Artifact layout + +Run commands from the repository root. Put builds under +`tools/abi/artifacts`, or replace that argument with an external absolute +directory: + +```text +tools/abi/artifacts/ +├── windows-x86_64/ +│ └── +├── linux-x86_64/ +│ └── +└── macos-arm64/ + └── +``` + +Names are not significant. `--select-symbol llama_decode` identifies the +llama library by content when dependency and backend libraries share the same +directory. + +Artifacts may come from local builds, an installed or extracted wheel, +[project releases](https://github.com/JamePeng/llama-cpp-python/releases), or +[upstream releases](https://github.com/ggml-org/llama.cpp/releases). Record +the source revision, compiler, architecture, and build options. Upstream +artifacts may not contain fork-only `llama_ext` APIs. + +## Scan exports + +```bash +python -m tools.abi scan tools/abi/artifacts --recursive +``` + +The default output is one same-named JSONL file per library: + +```text +tools/abi/output/ +└── 20260728T153012.123456Z/ + ├── llama.dll.jsonl + ├── libllama.so.jsonl + └── libllama.dylib.jsonl +``` + +Useful options: + +```bash +# Select only binaries that export llama_decode. +python -m tools.abi scan tools/abi/artifacts --recursive \ + --select-symbol llama_decode + +# Print instead of writing per-library JSONL. +python -m tools.abi scan tools/abi/artifacts --recursive --format text + +# Write one aggregate file; its filename receives a UTC timestamp. +python -m tools.abi scan tools/abi/artifacts --recursive \ + --format jsonl --output all-symbols.jsonl +``` + +`--prefix` is optional. By default all exports are retained: + +```bash +python -m tools.abi scan tools/abi/artifacts --recursive \ + --prefix llama_ --prefix ggml_ +``` + +## Check optional llama_ext bindings + +This is the primary ABI validation command: + +```bash +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive \ + --source llama_cpp/llama_cpp.py +``` + +It statically reads ctypes decorators without importing `llama_cpp`. The +default `--scope optional` checks declarations marked `required=False` and +returns exit code 1 if any candidate is missing. Other scopes are available: + +```bash +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive --scope required +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive --scope all +``` + +## Compare and create a manifest + +```bash +python -m tools.abi compare tools/abi/artifacts \ + --recursive --select-symbol llama_decode + +python -m tools.abi manifest tools/abi/artifacts \ + --recursive --select-symbol llama_decode \ + --output llama-exports.json +``` + +Cross-platform comparison uses `canonical_name`: + +```text +?llama_graph_reserve@@... MSVC +_Z19llama_graph_reserve... Linux Itanium ABI +__Z19llama_graph_reserve... Mach-O symbol table + ↓ +llama_graph_reserve canonical name +``` + +Records retain `raw_name`, ctypes `lookup_name`, `canonical_name`, ABI, +address, ordinal, library filename, format, architecture, SHA-256, and UTC +generation time. They never contain the artifact's absolute source path. + +Every run receives a timestamp, preventing normal output from overwriting +previous results. Generated artifacts and reports are ignored by Git. + +## Verification + +Unit tests are independent from the project's default test suite: + +```bash +python -m pytest tools/abi/tests/test_scan_dynamic.py -q +``` + +The opt-in integration test requires Windows, Linux, and macOS artifacts: + +```powershell +$env:LLAMA_ABI_ARTIFACTS = "tools/abi/artifacts" +python -m pytest tools/abi/tests/test_platform_artifacts.py -q +``` + +```bash +LLAMA_ABI_ARTIFACTS=tools/abi/artifacts \ +python -m pytest tools/abi/tests/test_platform_artifacts.py -q +``` + +Without configured artifacts, integration tests skip. With +`LLAMA_ABI_ARTIFACTS` set, a missing platform or optional ABI alias fails. diff --git a/tools/abi/__init__.py b/tools/abi/__init__.py new file mode 100644 index 0000000000..0a387e6759 --- /dev/null +++ b/tools/abi/__init__.py @@ -0,0 +1,3 @@ +"""Cross-platform shared-library ABI inspection tools.""" + +__author__ = "JamePeng" diff --git a/tools/abi/__main__.py b/tools/abi/__main__.py new file mode 100644 index 0000000000..acfe21acfa --- /dev/null +++ b/tools/abi/__main__.py @@ -0,0 +1,5 @@ +from .scan_dynamic import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/abi/artifacts/.gitignore b/tools/abi/artifacts/.gitignore new file mode 100644 index 0000000000..dbea0aa373 --- /dev/null +++ b/tools/abi/artifacts/.gitignore @@ -0,0 +1,4 @@ +# Keep downloaded or locally built native libraries out of Git. +* +!.gitignore +!README.md diff --git a/tools/abi/artifacts/README.md b/tools/abi/artifacts/README.md new file mode 100644 index 0000000000..30863ea8c0 --- /dev/null +++ b/tools/abi/artifacts/README.md @@ -0,0 +1,20 @@ +# ABI artifacts + +Maintainer: **JamePeng** + +Place trusted Windows, Linux, and macOS build artifacts here for local ABI +inspection. Filenames do not need to follow a fixed convention. + +```text +tools/abi/artifacts/ +├── windows-x86_64/ +├── linux-x86_64/ +├── macos-arm64/ +└── macos-x86_64/ +``` + +Downloaded and copied content is ignored by both the local and repository +`.gitignore`; only this README and `.gitignore` are tracked. `git add -f` can +still deliberately override ignore rules. + +See `tools/abi/README.md` for commands and artifact provenance requirements. diff --git a/tools/abi/output/.gitignore b/tools/abi/output/.gitignore new file mode 100644 index 0000000000..f12e9f061e --- /dev/null +++ b/tools/abi/output/.gitignore @@ -0,0 +1,4 @@ +# Keep generated ABI reports local. +* +!.gitignore +!README.md diff --git a/tools/abi/output/README.md b/tools/abi/output/README.md new file mode 100644 index 0000000000..3601ac132e --- /dev/null +++ b/tools/abi/output/README.md @@ -0,0 +1,7 @@ +# Local ABI reports + +Each run is stored in a UTC timestamp directory. Reports omit artifact source +paths but may contain binary hashes and non-public symbols. + +Generated content is ignored by both the local and repository `.gitignore`; +only this README and `.gitignore` are tracked. Review reports before sharing. diff --git a/tools/abi/scan_dynamic.py b/tools/abi/scan_dynamic.py new file mode 100644 index 0000000000..8ee11fd8c6 --- /dev/null +++ b/tools/abi/scan_dynamic.py @@ -0,0 +1,863 @@ +"""Inspect and compare exported symbols in PE, ELF, and Mach-O libraries. + +This repository-only maintainer utility supports collection and verification +of cross-platform ctypes symbol candidates, with particular focus on optional +llama_ext APIs. + +LIEF is imported lazily so that ``--help`` remains available when the optional +dependency is not installed. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import re +import sys +from collections import defaultdict +from dataclasses import asdict, dataclass, replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Sequence + +LIBRARY_SUFFIXES = {".dll", ".dylib", ".so"} +__author__ = "JamePeng" + + +class ScanError(RuntimeError): + """Raised when a shared library cannot be inspected.""" + + +@dataclass(frozen=True) +class SymbolRecord: + """One exported symbol and its cross-platform names.""" + + raw_name: str + lookup_name: str + canonical_name: str + abi: str + address: str + ordinal: int | None = None + + +@dataclass(frozen=True) +class BindingDeclaration: + """One ctypes decorator declaration extracted without importing llama_cpp.""" + + python_name: str + candidates: tuple[str, ...] + required: bool + line: int + + +@dataclass(frozen=True) +class LibraryScan: + """Metadata and exported symbols for one binary architecture.""" + + library: str + format: str + platform: str + architecture: str + sha256: str + symbols: tuple[SymbolRecord, ...] + + +def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def generation_timestamp() -> str: + """Return a sortable, collision-resistant UTC generation timestamp.""" + + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + + +def _enum_name(value: Any) -> str: + text = str(value) + return text.rsplit(".", 1)[-1] + + +def get_format(binary: Any) -> str: + value = str(binary.format).upper() + if "MACHO" in value: + return "Mach-O" + if "ELF" in value: + return "ELF" + if "PE" in value: + return "PE" + return str(binary.format) + + +def get_platform(binary_format: str) -> str: + return { + "PE": "windows", + "ELF": "linux", + "Mach-O": "darwin", + }.get(binary_format, "unknown") + + +def get_architecture(binary: Any, binary_format: str) -> str: + header = binary.header + if binary_format == "PE": + return _enum_name(header.machine) + if binary_format == "ELF": + return _enum_name(header.machine_type) + if binary_format == "Mach-O": + return _enum_name(header.cpu_type) + return "unknown" + + +def normalize_symbol_name(raw_name: str, binary_format: str) -> str: + """Return the name used by ctypes/dlsym and cross-platform comparison. + + Mach-O symbol tables prefix external C names with an underscore. dlsym and + ctypes callers use the source-level name without that platform prefix. + """ + + if binary_format == "Mach-O" and raw_name.startswith("_"): + return raw_name[1:] + return raw_name + + +def detect_abi(normalized_name: str) -> str: + if normalized_name.startswith("?"): + return "msvc-cxxabi" + if normalized_name.startswith("_Z"): + return "itanium-cxxabi" + return "unmangled" + + +def canonicalize_symbol_name(normalized_name: str, abi: str) -> str: + """Recover a source-level name from simple global C++ mangling. + + llama_ext functions are global functions, so their MSVC and Itanium + spellings can be mapped without a full ABI demangler. Namespaced, + overloaded, and templated symbols remain mangled to avoid false matches. + """ + + if abi == "msvc-cxxabi": + match = re.match(r"^\?([^@?$]+)@@", normalized_name) + if match: + return match.group(1) + + if abi == "itanium-cxxabi": + match = re.match(r"^_Z(\d+)", normalized_name) + if match: + length = int(match.group(1)) + start = match.end() + candidate = normalized_name[start : start + length] + if len(candidate) == length: + return candidate + + return normalized_name + + +def _symbol_address(symbol: Any) -> str: + value = getattr(symbol, "address", None) + if value is None: + value = getattr(symbol, "value", 0) + return hex(int(value)) + + +def _exported_symbols(binary: Any, binary_format: str) -> Iterable[Any]: + if binary_format == "PE": + if not binary.has_exports: + return () + return binary.get_export().entries + + # LIEF's exported_symbols filters undefined ELF imports and non-exported + # Mach-O symbols, unlike dynamic_symbols/symbols. + return binary.exported_symbols + + +def _iter_binaries(parsed: Any) -> list[Any]: + # A universal Mach-O may contain several architecture slices. + if type(parsed).__name__ == "FatBinary": + return list(parsed) + return [parsed] + + +def scan_library( + path: str | Path, +) -> list[LibraryScan]: + """Inspect one library, returning one result per architecture slice.""" + + try: + import lief + except ImportError as exc: + raise ScanError( + "LIEF is required for ABI inspection. Install it with: pip install lief" + ) from exc + + library_path = Path(path).expanduser().resolve() + if not library_path.is_file(): + raise ScanError(f"Not a file: {library_path.name}") + + try: + parsed = lief.parse(str(library_path)) + except Exception as exc: + detail = str(exc).replace(str(library_path), library_path.name) + raise ScanError(f"Failed to parse {library_path.name}: {detail}") from exc + + if parsed is None: + raise ScanError(f"LIEF did not recognize {library_path.name}") + + digest = sha256_file(library_path) + results: list[LibraryScan] = [] + + for binary in _iter_binaries(parsed): + binary_format = get_format(binary) + records: list[SymbolRecord] = [] + + for symbol in _exported_symbols(binary, binary_format): + raw_name = getattr(symbol, "name", None) + if not raw_name: + # PE supports ordinal-only exports. They cannot be matched to + # Python bindings by name, so keep a stable synthetic label. + ordinal = getattr(symbol, "ordinal", None) + if ordinal is None: + continue + raw_name = f"#{ordinal}" + + lookup_name = normalize_symbol_name(raw_name, binary_format) + abi = detect_abi(lookup_name) + canonical_name = canonicalize_symbol_name(lookup_name, abi) + + records.append( + SymbolRecord( + raw_name=raw_name, + lookup_name=lookup_name, + canonical_name=canonical_name, + abi=abi, + address=_symbol_address(symbol), + ordinal=getattr(symbol, "ordinal", None), + ) + ) + + records.sort(key=lambda item: (item.canonical_name, item.raw_name)) + results.append( + LibraryScan( + library=library_path.name, + format=binary_format, + platform=get_platform(binary_format), + architecture=get_architecture(binary, binary_format), + sha256=digest, + symbols=tuple(records), + ) + ) + + return results + + +def select_scans_by_symbols( + scans: Sequence[LibraryScan], + required_symbols: Sequence[str], +) -> list[LibraryScan]: + """Select binaries by exported canonical names, independent of filenames.""" + + if not required_symbols: + return list(scans) + selected = [] + for scan in scans: + exported = {symbol.canonical_name for symbol in scan.symbols} + if all(name in exported for name in required_symbols): + selected.append(scan) + return selected + + +def filter_scan_symbols( + scans: Sequence[LibraryScan], + prefixes: Sequence[str], +) -> list[LibraryScan]: + if not prefixes: + return list(scans) + return [ + replace( + scan, + symbols=tuple( + symbol + for symbol in scan.symbols + if any(symbol.canonical_name.startswith(prefix) for prefix in prefixes) + ), + ) + for scan in scans + ] + + +def extract_ctypes_bindings(source: str | Path) -> list[BindingDeclaration]: + """Extract literal ctypes decorator candidates without importing the module.""" + + source_path = Path(source) + try: + tree = ast.parse( + source_path.read_text(encoding="utf-8"), + filename=str(source_path), + ) + except (OSError, SyntaxError) as exc: + raise ScanError(f"Failed to parse binding source {source_path}: {exc}") from exc + + declarations: list[BindingDeclaration] = [] + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for decorator in node.decorator_list: + if not isinstance(decorator, ast.Call) or not decorator.args: + continue + decorator_name = "" + if isinstance(decorator.func, ast.Name): + decorator_name = decorator.func.id + elif isinstance(decorator.func, ast.Attribute): + decorator_name = decorator.func.attr + if not decorator_name.startswith("ctypes_function"): + continue + + try: + names = ast.literal_eval(decorator.args[0]) + except (ValueError, TypeError): + continue + if isinstance(names, str): + candidates = (names,) + elif isinstance(names, (list, tuple)) and all( + isinstance(name, str) for name in names + ): + candidates = tuple(names) + else: + continue + + required = True + for keyword in decorator.keywords: + if keyword.arg == "required": + try: + required = bool(ast.literal_eval(keyword.value)) + except (ValueError, TypeError): + pass + + declarations.append( + BindingDeclaration( + python_name=node.name, + candidates=candidates, + required=required, + line=node.lineno, + ) + ) + + return sorted(declarations, key=lambda item: item.line) + + +def check_bindings( + scan: LibraryScan, + declarations: Sequence[BindingDeclaration], +) -> dict[str, Any]: + """Check which ctypes candidate would be selected for one library.""" + + exported = {symbol.lookup_name for symbol in scan.symbols} + available = [] + missing_required = [] + missing_optional = [] + + for declaration in declarations: + selected = next( + (name for name in declaration.candidates if name in exported), + None, + ) + item = { + "python_name": declaration.python_name, + "required": declaration.required, + "line": declaration.line, + "candidates": list(declaration.candidates), + "selected": selected, + } + if selected is not None: + available.append(item) + elif declaration.required: + missing_required.append(item) + else: + missing_optional.append(item) + + return { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "declaration_count": len(declarations), + "available_count": len(available), + "available": available, + "missing_required": missing_required, + "missing_optional": missing_optional, + } + + +def compare_scans(scans: Sequence[LibraryScan]) -> dict[str, Any]: + if len(scans) < 2: + raise ValueError("At least two library scans are required for comparison") + + symbol_sets = [{symbol.canonical_name for symbol in scan.symbols} for scan in scans] + common = set.intersection(*symbol_sets) + libraries = [] + + for index, scan in enumerate(scans): + others = set.union(*(symbol_sets[i] for i in range(len(scans)) if i != index)) + libraries.append( + { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "symbol_count": len(symbol_sets[index]), + "only_here": sorted(symbol_sets[index] - others), + "missing_here": sorted(others - symbol_sets[index]), + } + ) + + return { + "common_count": len(common), + "common": sorted(common), + "libraries": libraries, + } + + +def build_manifest( + scans: Sequence[LibraryScan], + *, + generated_at: str | None = None, +) -> dict[str, Any]: + generated_at = generated_at or generation_timestamp() + symbols: dict[str, list[dict[str, Any]]] = defaultdict(list) + libraries = [] + + for scan in scans: + libraries.append(_scan_metadata(scan)) + for symbol in scan.symbols: + symbols[symbol.canonical_name].append( + { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "raw_name": symbol.raw_name, + "lookup_name": symbol.lookup_name, + "abi": symbol.abi, + "address": symbol.address, + "ordinal": symbol.ordinal, + } + ) + + return { + "schema_version": 1, + "generated_at": generated_at, + "libraries": libraries, + "symbols": dict(sorted(symbols.items())), + } + + +def collect_library_paths( + inputs: Sequence[str], + *, + recursive: bool = False, +) -> list[Path]: + def is_shared_library(path: Path) -> bool: + name = path.name.lower() + return path.suffix.lower() in LIBRARY_SUFFIXES or ".so." in name + + paths: list[Path] = [] + for value in inputs: + path = Path(value).expanduser() + if path.is_dir(): + candidates = path.rglob("*") if recursive else path.iterdir() + paths.extend( + candidate + for candidate in candidates + if candidate.is_file() and is_shared_library(candidate) + ) + else: + paths.append(path) + return sorted(set(paths), key=lambda item: str(item).lower()) + + +def _scan_paths( + paths: Sequence[Path], +) -> tuple[list[LibraryScan], list[str]]: + scans: list[LibraryScan] = [] + errors: list[str] = [] + for path in paths: + try: + scans.extend(scan_library(path)) + except ScanError as exc: + errors.append(str(exc)) + except Exception as exc: + detail = str(exc).replace(str(path.resolve()), path.name) + errors.append(f"{path.name}: {detail}") + return scans, errors + + +def _timestamped_output_path(output: str | Path, timestamp: str) -> Path: + path = Path(output) + return path.with_name(f"{path.stem}.{timestamp}{path.suffix}") + + +def _write_output( + text: str, + output: str | None, + *, + timestamp: str, +) -> None: + if output: + output_path = _timestamped_output_path(output, timestamp) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(text + "\n", encoding="utf-8") + print(f"saved: {output_path}") + else: + print(text) + + +def _scan_metadata(scan: LibraryScan) -> dict[str, Any]: + return {key: value for key, value in asdict(scan).items() if key != "symbols"} + + +def _jsonl_rows(scan: LibraryScan, generated_at: str) -> list[str]: + metadata = _scan_metadata(scan) + return [ + json.dumps( + { + "generated_at": generated_at, + **metadata, + **asdict(symbol), + }, + ensure_ascii=False, + ) + for symbol in scan.symbols + ] + + +def write_library_jsonl( + scans: Sequence[LibraryScan], + output_dir: str | Path = "tools/abi/output", + *, + timestamp: str | None = None, +) -> list[Path]: + """Write one same-named JSONL per library under a timestamped run directory.""" + + timestamp = timestamp or generation_timestamp() + destination = Path(output_dir) / timestamp + destination.mkdir(parents=True, exist_ok=True) + grouped: dict[str, list[LibraryScan]] = defaultdict(list) + for scan in scans: + grouped[scan.library].append(scan) + + written = [] + for library, library_scans in sorted(grouped.items()): + output_path = destination / f"{library}.jsonl" + rows = [ + row + for library_scan in library_scans + for row in _jsonl_rows(library_scan, timestamp) + ] + output_path.write_text( + "\n".join(rows) + ("\n" if rows else ""), + encoding="utf-8", + ) + written.append(output_path) + return written + + +def _scan_text(scans: Sequence[LibraryScan], errors: Sequence[str]) -> str: + lines: list[str] = [] + for scan in scans: + lines.append( + f"{scan.library} [{scan.format}/{scan.architecture}]: " + f"{len(scan.symbols)} exported symbol(s)" + ) + for symbol in scan.symbols: + raw_suffix = ( + f" (raw: {symbol.raw_name})" + if symbol.raw_name != symbol.canonical_name + else "" + ) + lines.append( + f" {symbol.canonical_name} [{symbol.abi}]" + f" @ {symbol.address}{raw_suffix}" + ) + for error in errors: + lines.append(f"ERROR: {error}") + return "\n".join(lines) + + +def _compare_text(comparison: dict[str, Any]) -> str: + lines = [f"Common canonical symbols: {comparison['common_count']}"] + for library in comparison["libraries"]: + lines.extend( + [ + "", + ( + f"{library['library']} " + f"[{library['platform']}/{library['architecture']}]: " + f"{library['symbol_count']} symbol(s)" + ), + f" Only here: {len(library['only_here'])}", + ] + ) + lines.extend(f" {name}" for name in library["only_here"]) + lines.append(f" Missing here: {len(library['missing_here'])}") + lines.extend(f" {name}" for name in library["missing_here"]) + return "\n".join(lines) + + +def _bindings_text( + results: Sequence[dict[str, Any]], + scope: str, +) -> str: + lines: list[str] = [] + for result in results: + lines.append( + f"{result['library']} " + f"[{result['platform']}/{result['architecture']}]: " + f"{result['available_count']}/{result['declaration_count']} " + "binding(s) available in selected scope" + ) + if scope in {"required", "all"}: + lines.append(f" Missing required: {len(result['missing_required'])}") + lines.extend( + f" {item['python_name']} (line {item['line']})" + for item in result["missing_required"] + ) + if scope in {"optional", "all"}: + lines.append(f" Missing optional: {len(result['missing_optional'])}") + lines.extend( + f" {item['python_name']} (line {item['line']})" + for item in result["missing_optional"] + ) + return "\n".join(lines) + + +def create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Inspect and compare PE, ELF, and Mach-O exported symbols." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + def add_common(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument("paths", nargs="+", help="Library files or directories") + subparser.add_argument( + "--prefix", + action="append", + default=[], + help=( + "Optional canonical-name filter; may be repeated " + "(default: keep all exports)" + ), + ) + subparser.add_argument( + "--select-symbol", + action="append", + default=[], + help=( + "Select libraries exporting this canonical symbol; may be " + "repeated and does not depend on the library filename" + ), + ) + subparser.add_argument( + "--recursive", + action="store_true", + help="Recursively search directory inputs", + ) + subparser.add_argument("-o", "--output", help="Write output to this file") + + scan_parser = subparsers.add_parser("scan", help="List exported symbols") + add_common(scan_parser) + scan_parser.add_argument( + "--format", + choices=("text", "json", "jsonl"), + default="jsonl", + help="Output format", + ) + scan_parser.add_argument( + "--output-dir", + default="tools/abi/output", + help=( + "Directory for default per-library JSONL files " + "(default: tools/abi/output)" + ), + ) + + compare_parser = subparsers.add_parser( + "compare", help="Compare canonical symbol names across libraries" + ) + add_common(compare_parser) + compare_parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="Output format", + ) + + manifest_parser = subparsers.add_parser( + "manifest", help="Create a cross-platform symbol manifest" + ) + add_common(manifest_parser) + + bindings_parser = subparsers.add_parser( + "check-bindings", + help="Check literal ctypes decorator candidates against libraries", + ) + add_common(bindings_parser) + bindings_parser.add_argument( + "--source", + default="llama_cpp/llama_cpp.py", + help="Python binding source to inspect without importing it", + ) + bindings_parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="Output format", + ) + bindings_parser.add_argument( + "--scope", + choices=("optional", "required", "all"), + default="optional", + help=("Binding declarations to check " "(default: optional llama_ext APIs)"), + ) + + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = create_parser().parse_args(argv) + paths = collect_library_paths(args.paths, recursive=args.recursive) + if not paths: + print("No shared libraries found.", file=sys.stderr) + return 2 + + # Scan all exports first. Selection must not depend on --prefix, because a + # caller may use an anchor outside the displayed prefix set. + scans, errors = _scan_paths(paths) + if not scans: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + if args.select_symbol: + scans = select_scans_by_symbols(scans, args.select_symbol) + if not scans: + print( + "No library exports all requested selection symbols: " + + ", ".join(args.select_symbol), + file=sys.stderr, + ) + return 2 + + scans = filter_scan_symbols(scans, args.prefix) + if args.command in {"compare", "manifest"} and args.prefix: + # A package lib directory normally contains ggml and accelerator + # backends. Empty prefix matches are not comparison targets. + scans = [scan for scan in scans if scan.symbols] + timestamp = generation_timestamp() + + validation_failed = False + + if args.command == "scan": + if args.format == "text": + output = _scan_text(scans, errors) + elif args.format == "json": + output = json.dumps( + { + "generated_at": timestamp, + "libraries": [asdict(scan) for scan in scans], + "errors": errors, + }, + ensure_ascii=False, + indent=2, + ) + else: + output = "\n".join( + row for scan in scans for row in _jsonl_rows(scan, timestamp) + ) + + if args.format == "jsonl" and args.output is None: + try: + written = write_library_jsonl( + scans, + args.output_dir, + timestamp=timestamp, + ) + except OSError as exc: + print(f"ERROR: failed to write JSONL output: {exc}", file=sys.stderr) + return 1 + for path in written: + print(f"saved: {path}") + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors else 0 + elif args.command == "compare": + if len(scans) < 2: + print("Comparison requires at least two libraries.", file=sys.stderr) + return 2 + comparison = compare_scans(scans) + output = ( + _compare_text(comparison) + if args.format == "text" + else json.dumps(comparison, ensure_ascii=False, indent=2) + ) + elif args.command == "manifest": + output = json.dumps( + build_manifest(scans, generated_at=timestamp), + ensure_ascii=False, + indent=2, + ) + else: + try: + declarations = extract_ctypes_bindings(args.source) + except ScanError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + if args.scope == "optional": + declarations = [ + declaration for declaration in declarations if not declaration.required + ] + elif args.scope == "required": + declarations = [ + declaration for declaration in declarations if declaration.required + ] + if not declarations: + print( + f"No {args.scope} ctypes binding declarations found in " + f"{args.source}.", + file=sys.stderr, + ) + return 2 + binding_results = [check_bindings(scan, declarations) for scan in scans] + if not args.select_symbol and binding_results: + # A package directory may contain arbitrarily named dependency and + # backend libraries. The library ctypes would want is the one with + # the greatest declaration coverage, regardless of filename. + best_count = max(result["available_count"] for result in binding_results) + binding_results = [ + result + for result in binding_results + if result["available_count"] == best_count + ] + output = ( + _bindings_text(binding_results, args.scope) + if args.format == "text" + else json.dumps(binding_results, ensure_ascii=False, indent=2) + ) + validation_failed = any( + result["missing_required"] or result["missing_optional"] + for result in binding_results + ) + + try: + _write_output(output, args.output, timestamp=timestamp) + except OSError as exc: + print(f"ERROR: failed to write output: {exc}", file=sys.stderr) + return 1 + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors or validation_failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/abi/tests/test_platform_artifacts.py b/tools/abi/tests/test_platform_artifacts.py new file mode 100644 index 0000000000..b5f88dcc3f --- /dev/null +++ b/tools/abi/tests/test_platform_artifacts.py @@ -0,0 +1,85 @@ +"""Opt-in integration tests for real Windows, Linux, and macOS artifacts.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from tools.abi.scan_dynamic import ( + check_bindings, + collect_library_paths, + extract_ctypes_bindings, + scan_library, + select_scans_by_symbols, +) + +ARTIFACTS_ENV = "LLAMA_ABI_ARTIFACTS" +DEFAULT_ARTIFACTS = Path("tools/abi/artifacts") +REQUIRED_PLATFORMS = {"windows", "linux", "darwin"} +STABLE_LLAMA_SYMBOLS = { + "llama_decode", + "llama_model_load_from_file", +} +BINDING_SOURCE = Path("llama_cpp/llama_cpp.py") + + +@pytest.fixture(scope="module") +def platform_scans(): + configured = os.environ.get(ARTIFACTS_ENV) + artifacts = Path(configured) if configured else DEFAULT_ARTIFACTS + paths = collect_library_paths([str(artifacts)], recursive=True) + + if not paths and not configured: + pytest.skip( + "No ABI artifacts installed. Set LLAMA_ABI_ARTIFACTS to run " + "the Windows/Linux/macOS integration test." + ) + + assert paths, f"No shared libraries found under {artifacts}" + scans = [scan for path in paths for scan in scan_library(path)] + scans = select_scans_by_symbols(scans, ["llama_decode"]) + by_platform = {scan.platform: scan for scan in scans} + assert REQUIRED_PLATFORMS <= set(by_platform), ( + "The ABI artifact set must contain llama libraries for Windows, " + f"Linux, and macOS. Found: {sorted(by_platform)}" + ) + return by_platform + + +def test_windows_linux_and_macos_llama_exports(platform_scans): + common = set.intersection( + *( + {symbol.canonical_name for symbol in platform_scans[platform].symbols} + for platform in sorted(REQUIRED_PLATFORMS) + ) + ) + assert STABLE_LLAMA_SYMBOLS <= common + + +def test_macos_macho_lookup_name_removes_symbol_table_prefix(platform_scans): + decode = next( + symbol + for symbol in platform_scans["darwin"].symbols + if symbol.canonical_name == "llama_decode" + ) + assert decode.raw_name == "_llama_decode" + assert decode.lookup_name == "llama_decode" + + +def test_optional_llama_ext_abi_aliases_on_all_platforms(platform_scans): + optional = [ + declaration + for declaration in extract_ctypes_bindings(BINDING_SOURCE) + if not declaration.required + ] + assert optional, "No optional llama_ext ctypes bindings were found" + + for platform in sorted(REQUIRED_PLATFORMS): + result = check_bindings(platform_scans[platform], optional) + assert ( + result["missing_optional"] == [] + ), f"{platform} is missing optional llama_ext ABI aliases: " + ", ".join( + item["python_name"] for item in result["missing_optional"] + ) diff --git a/tools/abi/tests/test_scan_dynamic.py b/tools/abi/tests/test_scan_dynamic.py new file mode 100644 index 0000000000..ebcf75fd47 --- /dev/null +++ b/tools/abi/tests/test_scan_dynamic.py @@ -0,0 +1,194 @@ +import json + +import tools.abi.scan_dynamic as abi_tool + +from tools.abi.scan_dynamic import ( + BindingDeclaration, + SymbolRecord, + LibraryScan, + canonicalize_symbol_name, + check_bindings, + collect_library_paths, + compare_scans, + detect_abi, + extract_ctypes_bindings, + normalize_symbol_name, + select_scans_by_symbols, + write_library_jsonl, +) + + +def _scan(library: str, platform: str, names: list[str]) -> LibraryScan: + records = tuple( + SymbolRecord( + raw_name=name, + lookup_name=name, + canonical_name=name, + abi="unmangled", + address="0x0", + ) + for name in names + ) + return LibraryScan( + library=library, + format="test", + platform=platform, + architecture="test", + sha256="test", + symbols=records, + ) + + +def test_normalizes_macho_external_prefix(): + assert normalize_symbol_name("_llama_decode", "Mach-O") == "llama_decode" + assert normalize_symbol_name("__ZN5llama", "Mach-O") == "_ZN5llama" + assert normalize_symbol_name("llama_decode", "ELF") == "llama_decode" + assert normalize_symbol_name("llama_decode", "PE") == "llama_decode" + + +def test_detects_abi_after_platform_normalization(): + assert detect_abi("?function@@YAXXZ") == "msvc-cxxabi" + assert detect_abi("_ZN5llama") == "itanium-cxxabi" + assert detect_abi("llama_decode") == "unmangled" + + +def test_canonicalizes_simple_global_cpp_names(): + assert ( + canonicalize_symbol_name( + "?llama_graph_reserve@@YAXXZ", + "msvc-cxxabi", + ) + == "llama_graph_reserve" + ) + assert ( + canonicalize_symbol_name( + "_Z19llama_graph_reserveP13llama_contextjjj", + "itanium-cxxabi", + ) + == "llama_graph_reserve" + ) + nested = "_ZN5llama6detail3fooEv" + assert canonicalize_symbol_name(nested, "itanium-cxxabi") == nested + + +def test_compares_canonical_names(): + comparison = compare_scans( + [ + _scan("libllama.so", "linux", ["llama_decode"]), + _scan( + "llama.dll", + "windows", + ["llama_decode", "llama_windows_only"], + ), + ] + ) + + assert comparison["common"] == ["llama_decode"] + assert comparison["libraries"][0]["missing_here"] == ["llama_windows_only"] + assert comparison["libraries"][1]["only_here"] == ["llama_windows_only"] + + +def test_collects_versioned_elf_library(tmp_path): + library = tmp_path / "libllama.so.1" + library.touch() + + assert collect_library_paths([str(tmp_path)]) == [library] + + +def test_extracts_and_checks_literal_binding_aliases(tmp_path): + source = tmp_path / "bindings.py" + source.write_text( + """ +@ctypes_function( + ["llama_ext", "?llama_ext@@YAXXZ", "_Z9llama_extv"], + [], + None, + required=False, +) +def llama_ext(): + pass +""", + encoding="utf-8", + ) + declarations = extract_ctypes_bindings(source) + scan = _scan("libllama.so", "linux", ["_Z9llama_extv"]) + result = check_bindings(scan, declarations) + + assert declarations == [ + BindingDeclaration( + python_name="llama_ext", + candidates=( + "llama_ext", + "?llama_ext@@YAXXZ", + "_Z9llama_extv", + ), + required=False, + line=8, + ) + ] + assert result["available"][0]["selected"] == "_Z9llama_extv" + assert result["missing_optional"] == [] + + +def test_selects_library_by_symbol_not_filename(): + scans = [ + _scan("custom-backend-name.dll", "windows", ["ggml_backend_init"]), + _scan("renamed-native-output.bin", "windows", ["llama_decode"]), + ] + + selected = select_scans_by_symbols(scans, ["llama_decode"]) + + assert [scan.library for scan in selected] == ["renamed-native-output.bin"] + + +def test_writes_jsonl_named_after_dynamic_library(tmp_path): + output_dir = tmp_path / "output" + scans = [ + _scan("libllama.so", "linux", ["llama_decode"]), + _scan("llama.dll", "windows", ["llama_decode"]), + ] + + timestamp = "20260728T120000.123456Z" + written = write_library_jsonl( + scans, + output_dir, + timestamp=timestamp, + ) + + assert [path.name for path in written] == [ + "libllama.so.jsonl", + "llama.dll.jsonl", + ] + run_dir = output_dir / timestamp + row = json.loads((run_dir / "llama.dll.jsonl").read_text("utf-8")) + assert row["library"] == "llama.dll" + assert row["canonical_name"] == "llama_decode" + assert row["generated_at"] == timestamp + assert "path" not in row + + +def test_check_bindings_cli_fails_when_optional_api_is_missing( + tmp_path, + monkeypatch, +): + library = tmp_path / "renamed.dll" + library.touch() + source = tmp_path / "bindings.py" + source.write_text( + """ +@ctypes_function(["llama_ext", "_Z9llama_extv"], [], None, required=False) +def llama_ext(): + pass +""", + encoding="utf-8", + ) + scan = _scan("renamed.dll", "windows", ["llama_decode"]) + monkeypatch.setattr( + abi_tool, + "_scan_paths", + lambda paths: ([scan], []), + ) + + exit_code = abi_tool.main(["check-bindings", str(library), "--source", str(source)]) + + assert exit_code == 1 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index bddfd2b113..0ec1cfd056 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit bddfd2b1137cd6e51fbb939081caf50e9f496a66 +Subproject commit 0ec1cfd056fb97dbe2a354cca74e5a5091e48674