diff --git a/.gitignore b/.gitignore index fad7f43313..b5d60bf894 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,15 @@ local_settings.py models/ docker/open_llama/*.bin +# Repository-only ABI tool inputs and generated reports. +# Keep only the directory instructions and local ignore rules tracked. +/tools/abi/artifacts/* +!/tools/abi/artifacts/.gitignore +!/tools/abi/artifacts/README.md +/tools/abi/output/* +!/tools/abi/output/.gitignore +!/tools/abi/output/README.md + # C extensions (llama_cpp bindings) llama_cpp/*.so llama_cpp/*.dylib @@ -208,4 +217,4 @@ docs/_build/ # Installer logs pip-log.txt -pip-delete-this-directory.txt \ No newline at end of file +pip-delete-this-directory.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 69fc02b25c..075b978847 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,158 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.45] Reactivated Built-in Embeddings, Modern Model Loading, and Stronger Cross-Platform Reliability + +- fix(ctypes): correct llama-ext binding signatures + - use uint32_t for layer IDs + - fix void return type for embedding extraction control + - return target layer count as uint32_t + +- feat(llama): expose additional model loading options + - add `no_alloc` and `load_mtp` parameters + - enable `extra buffer types` by default + +- feat(llama): support llama_model_params `load_mode` + - Update model loading configuration to use the new `load_mode` field from + llama_model_params and align with the latest llama.cpp API changes. + - Remove deprecated internal handling of legacy loading flags and keep + backward compatibility by warning users when `use_mmap`, `use_direct_io`, + or `use_mlock` are still used. + - This prepares the Python bindings for the updated llama.cpp model loading + interface while providing a smoother migration path for existing users. + - docs: document `load_mode` migration + - Replace references to the legacy model loading flags with load_mode, document all supported loading modes for the Python API and server, and update the performance tuning example. + +- feat(tools): add cross-platform ABI inspection utility + - Inspect `PE`, `ELF`, and `Mach-O` exports and normalize platform-specific symbol names. + - Validate optional `llama_ext` ctypes aliases across Windows, Linux, and macOS builds. Keep artifacts and timestamped privacy-safe reports local to the repository. + - More information see here: [Cross-platform ABI inspection](https://github.com/JamePeng/llama-cpp-python/tree/main/tools/abi) + +- fix(ctypes): support GCC/Clang mangled symbols for optional llama_ext APIs + - Add missing `_Z` Itanium C++ ABI symbol variants to ctypes function + lookup lists. This improves compatibility with Linux and macOS builds + where C++ symbols are exported using GCC/Clang name mangling. + - Issue report from **@ckcfcc** (https://github.com/JamePeng/llama-cpp-python/issues/159) + +- fix(loader): guard `HIP_PATH` and `VULKAN_SDK` dirs with os.path.exists +os.add_dll_directory() raises FileNotFoundError [WinError 3] when the +directory does not exist, so a stale `HIP_PATH` or `VULKAN_SDK` left behind by +an uninstalled SDK makes "import llama_cpp" fail outright on Windows.(by **@emptyngton**) + + The CUDA_PATH branch above already guards each candidate directory with + os.path.exists(); this applies the same pattern to the HIP and Vulkan + branches. Valid directories are still added individually, so a partially + removed SDK contributes whichever of bin/lib remain instead of raising. + +- fix(_internals): clean up native resources on initialization failures + - Register native model and batch ownership immediately after allocation + so later validation failures cannot leak llama.cpp resources. + Free a loaded model when vocab lookup fails, and route mixed-batch setup + failures through idempotent cleanup. + - Initialize sampling-context resource fields before fallible setup and + make partial teardown safe to repeat. This prevents missing attributes + from interrupting cleanup when sampler-chain construction fails. + - Clear model, vocabulary, and sampling parameter references after native + context and sampler resources have been released. This prevents closed + wrapper objects from unnecessarily keeping models and related Python + objects alive. + - Add failure-injection tests that verify model and batch handles are freed + exactly once and partially initialized sampling contexts release their resources + idempotently.Extend lifecycle tests to verify that parent references are cleared + and that repeated close calls remain safe. + +- test(chat-format): modernize coverage with Qwen3.5-style templates + - Replace the legacy Mistral-focused chat format tests with self-contained + Qwen3.5-style Jinja template coverage: + - verify ChatML system, user, and assistant message rendering + - cover enabled and disabled thinking generation prompts + - test image and video placeholders with vision identifiers + - validate tool definitions, tool calls, and tool response history + - add clear error coverage for invalid message structures + - verify model-specific stop token criteria + - keep the tests independent of tokenizer files and model weights + +- docs(readme): replace the new logo with fork project branding + - Add the new llama-cpp-python logo asset under docs and update the README + header to reference the repository-local image. + - the new logo which combined llama, C++, and Project branding remains readable. + +- docs(embedding): add end-to-end embeddings and reranking guide + - Create a schema-compliant feature guide covering sentence embeddings, + token-level vectors, reranking workflows, pooling modes, normalization, + streaming batch configuration, return shapes, and output formats. + - Add complete examples for the standard Llama API, LlamaEmbedding, + pre-tokenized inputs, cosine-similarity output, and cross-encoder + reranking. + - Document common configuration problems, implementation limitations, and + the embedding and reranking model families currently listed as supported + by the project. + - Expose the new feature guide through the Wiki index. + +- docs(llama): expand embedding parameters and API guidance + - Add a role overview and reorganize constructor options into focused, + readable parameter groups. + - Document embedding, pooling, attention, KV cache, sequence capacity, and + recurrent-state settings with their defaults and runtime behavior. + - Expand the embed() and create_embedding() sections with normalization + modes, return shapes, batching semantics, pooling recommendations, + OpenAI compatibility notes, and resource-safe examples. + - Fix the YAML frontmatter and improve Markdown spacing for cleaner Wiki + rendering. + +- docs(embedding): document maintained APIs and sequence batch capacity + - Replace the deprecated Llama embedding guidance with current embed() and + create_embedding() usage. + - Document the roles of n_batch, n_ubatch, and n_seq_max, including + parallel batching examples, resource considerations, common sequence ID + errors, and the required configuration changes. + - Clarify that LlamaEmbedding remains a convenience interface for + embedding-oriented defaults and reranking workflows. + +- docs(example): refresh the built-in embedding usage example + - Fix the Llama constructor option from embedding=True to embeddings=True + and demonstrate L2-normalized output through create_embedding(). + +- test(embedding): cover built-in and streaming embedding workflows + - Add coverage for actionable LlamaBatch sequence-capacity errors and the + maintained embedding APIs on the standard Llama class. + - Verify pre-tokenized batches, normalization, separator-based inputs, + token accounting, OpenAI-compatible responses, and LlamaEmbedding + streaming behavior with n_seq_max=1. + - Explicitly close embedding models after integration tests to release + native context and model resources. + +- fix(embedding): respect n_seq_max when streaming embedding batches + - Use the configured sequence capacity instead of n_ubatch when deciding + when to decode the current LlamaEmbedding batch. + - This prevents invalid sequence IDs for multi-document inputs and allows + the default n_seq_max=1 configuration to process documents sequentially + without failing. + +- refactor(batch): improve sequence capacity validation guidance + - Make LlamaBatch sequence validation errors explain the configured + n_seq_max value, valid sequence ID range, and minimum capacity required + for parallel batching. + - Handle negative sequence IDs separately and provide actionable setup + guidance for Llama, LlamaEmbedding, and direct LlamaBatch users. + - Remove the unused normalize_embedding helper now that normalization is + handled by the embedding pipeline. + +- feat(embedding): modernize the built-in Llama embedding API + - Replace the legacy embedding path with sequence-aware streaming batch + processing based on the current LlamaBatch interface. + - Support string, batched string, and pre-tokenized inputs, token-level and + rank pooling outputs, separator-based splitting, token accounting, and + llama.cpp-compatible normalization modes. + - Restore embed() and create_embedding() as maintained Llama APIs while + preserving the existing boolean normalization behavior. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/876a4321163249c43ca4e986818fab5ab081f282](https://github.com/ggml-org/llama.cpp/commit/876a4321163249c43ca4e986818fab5ab081f282) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260801 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/ebf6099b81cf67cfb5eec569466367c9fa04e9d4...aafc6fb74ebfba6a044510f80b5e9ad277109c12 + ## [0.3.44] Improved Windows DLL(OpenMP) Loading Reliability for GGML Backends - fix(ggml): preload bundled OpenMP runtime before loading ggml-base diff --git a/docs/server.md b/docs/server.md index cd6f86c513..3bdc0c7e69 100644 --- a/docs/server.md +++ b/docs/server.md @@ -37,6 +37,26 @@ CLI arguments and environment variables are available for all of the fields defi Additionally the server supports configuration check out the [configuration section](#configuration-and-multi-model-support) for more information and examples. +#### Model loading mode + +Use `load_mode` to select how the server loads model data. The corresponding +CLI option is `--load_mode`, the environment variable is `LOAD_MODE`, and a +multi-model JSON configuration can set `"load_mode"` for each model. +`use_mmap`, `use_direct_io`, and `use_mlock` are no longer server settings. + +| Value | Mode | Description | +|---:|---|---| +| `0` | `LLAMA_LOAD_MODE_NONE` | Use no special model-loading mode. | +| `1` | `LLAMA_LOAD_MODE_MMAP` | Memory-map the model. This is the default. | +| `2` | `LLAMA_LOAD_MODE_MLOCK` | Keep the loaded model in RAM rather than allowing it to be swapped or compressed. | +| `3` | `LLAMA_LOAD_MODE_MMAP_MLOCK` | Memory-map the model and keep its mapped pages in RAM. | +| `4` | `LLAMA_LOAD_MODE_DIRECT_IO` | Use direct I/O when it is available. | + +For example, start the server with memory mapping plus memory locking: + +```bash +python3 -m llama_cpp.server --model --load_mode 3 +``` ## Guides diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 305624c8fc..00add6ea4c 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -3,7 +3,7 @@ title: Llama Class module_name: llama_cpp.llama source_file: llama_cpp/llama.py class_name: Llama -last_updated: 2026-07-26 +last_updated: 2026-07-29 version_target: "latest" --- @@ -36,13 +36,47 @@ Initialize the model and context. Note that model loading will immediately alloc | `cpu_moe` | `bool` | `False` | Whether to keep all MoE weights on CPU | | `n_cpu_moe` | `int` | `0` | Number of first N MoE layers to keep on CPU (compatible with `cpu_moe`) | | `split_mode` | `int` | `LLAMA_SPLIT_MODE_LAYER` | Model GPU split mode:
• `LLAMA_SPLIT_MODE_NONE`: single GPU
• `LLAMA_SPLIT_MODE_ROW`: row-level split
• `LLAMA_SPLIT_MODE_LAYER`: layer-level split | +| `load_mode` | `int` (`llama_load_mode`) | `LLAMA_LOAD_MODE_MMAP` | How model data is loaded. Select one of the `LLAMA_LOAD_MODE_*` values described below. | | `main_gpu` | `int` | `0` | The primary GPU to use for intermediate results or the entire model. | | `tensor_split` | `List[float]` | `None` | Proportional split of tensors across GPUs (max `LLAMA_MAX_DEVICES`). | -| `use_mmap` | `bool` | `True` | Whether to use memory mapping (mmap) if possible. | -| `use_mlock` | `bool` | `False` | Force the system to keep the model in RAM, preventing swapping. | | `kv_overrides` | `Dict` | `None` | Key-value overrides for the model metadata (supports bool, int, float, str). | | `numa` | `Union[bool, int]` | `False` | NUMA strategy (e.g., `GGML_NUMA_STRATEGY_DISTRIBUTE`). | +#### Model Load Modes + +`load_mode` replaces the legacy `use_mmap`, `use_direct_io`, and `use_mlock` +arguments. It accepts a member of `llama_cpp.llama_load_mode`: + +| Value | Integer | Description | +| :--- | :---: | :--- | +| `LLAMA_LOAD_MODE_NONE` | `0` | Use no special model-loading mode. | +| `LLAMA_LOAD_MODE_MMAP` | `1` | Memory-map the model. This is the default. | +| `LLAMA_LOAD_MODE_MLOCK` | `2` | Keep the loaded model in RAM rather than allowing it to be swapped or compressed. | +| `LLAMA_LOAD_MODE_MMAP_MLOCK` | `3` | Memory-map the model and keep its mapped pages in RAM. | +| `LLAMA_LOAD_MODE_DIRECT_IO` | `4` | Use direct I/O when it is available. | + +```python +import llama_cpp + +llm = llama_cpp.Llama( + model_path="models/model.gguf", + load_mode=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP_MLOCK, +) +``` + +The legacy loading arguments are retained only for call compatibility. They no +longer configure the underlying model parameters and may emit a deprecation +warning; set `load_mode` explicitly instead. Use the following migration +mapping: + +| Legacy configuration | Replacement | +| :--- | :--- | +| `use_mmap=False, use_mlock=False` | `load_mode=LLAMA_LOAD_MODE_NONE` | +| `use_mmap=True, use_mlock=False` | `load_mode=LLAMA_LOAD_MODE_MMAP` | +| `use_mmap=False, use_mlock=True` | `load_mode=LLAMA_LOAD_MODE_MLOCK` | +| `use_mmap=True, use_mlock=True` | `load_mode=LLAMA_LOAD_MODE_MMAP_MLOCK` | +| `use_direct_io=True` | `load_mode=LLAMA_LOAD_MODE_DIRECT_IO` | + ### Context & Batch Parameters | Parameter | Type | Default | Description | diff --git a/examples/low_level_api/common.py b/examples/low_level_api/common.py index 8adb2923cc..601f5cebdf 100644 --- a/examples/low_level_api/common.py +++ b/examples/low_level_api/common.py @@ -60,9 +60,6 @@ class GptParams: instruct: bool = False perplexity: bool = False - use_mmap: bool = True - use_direct_io: bool = False - use_mlock: bool = False mem_test: bool = False verbose_prompt: bool = False diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py index 1f4f5b3e79..96c4121f4f 100644 --- a/examples/low_level_api/low_level_api_chat_cpp.py +++ b/examples/low_level_api/low_level_api_chat_cpp.py @@ -76,9 +76,6 @@ def __init__(self, params: GptParams) -> None: self.lparams.n_parts = self.params.n_parts self.lparams.seed = self.params.seed self.lparams.memory_f16 = self.params.memory_f16 - self.lparams.use_mlock = self.params.use_mlock - self.lparams.use_mmap = self.params.use_mmap - self.lparams.use_direct_io = self.params.use_direct_io self.model = llama_cpp.llama_load_model_from_file( self.params.model.encode("utf8"), self.lparams diff --git a/examples/notebooks/PerformanceTuning.ipynb b/examples/notebooks/PerformanceTuning.ipynb index ba74e4a41f..43772a5b7e 100644 --- a/examples/notebooks/PerformanceTuning.ipynb +++ b/examples/notebooks/PerformanceTuning.ipynb @@ -24,7 +24,13 @@ "# Hyperparameters\n", "space = [\n", " Categorical([True, False], name=\"f16_kv\"),\n", - " Categorical([True, False], name=\"use_mlock\"),\n", + " Categorical(\n", + " [\n", + " llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP,\n", + " llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP_MLOCK,\n", + " ],\n", + " name=\"load_mode\",\n", + " ),\n", " Integer(1, multiprocessing.cpu_count(), name=\"n_threads\"),\n", " Integer(1, 2048, name=\"n_batch\"),\n", "]\n", @@ -46,13 +52,13 @@ "@use_named_args(space)\n", "def objective(**params):\n", " f16_kv = params[\"f16_kv\"]\n", - " use_mlock = params[\"use_mlock\"]\n", + " load_mode = params[\"load_mode\"]\n", " n_threads = params[\"n_threads\"]\n", " n_batch = params[\"n_batch\"]\n", " llm = llama_cpp.Llama(\n", " model_path=MODEL_PATH,\n", " f16_kv=f16_kv,\n", - " use_mlock=use_mlock,\n", + " load_mode=load_mode,\n", " n_threads=n_threads,\n", " n_batch=n_batch,\n", " )\n", diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 10e452d5f6..b359355f9e 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.44" +__version__ = "0.3.45" diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index a9a2c02e50..3634720681 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -118,13 +118,19 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list # Add HIP runtime DLL directories when HIP backend is available. if "HIP_PATH" in os.environ: - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) + hip_path = os.environ["HIP_PATH"] + for sub_dir in ["bin", "lib"]: + full_path = os.path.join(hip_path, sub_dir) + if os.path.exists(full_path): + os.add_dll_directory(full_path) # Add Vulkan SDK DLL directories when Vulkan backend is enabled. if "VULKAN_SDK" in os.environ: - os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Bin")) - os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Lib")) + vulkan_sdk = os.environ["VULKAN_SDK"] + for sub_dir in ["Bin", "Lib"]: + full_path = os.path.join(vulkan_sdk, sub_dir) + if os.path.exists(full_path): + os.add_dll_directory(full_path) # Add package-provided library directories. # diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index 9a7dac517b..ee1a101870 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -35,6 +35,12 @@ def _preload_openmp_runtime(): if not _version_at_least("0.3.39"): return + # Some ComfyUI environments include complex software packages and may also contain + # additional OpenMP libraries (such as `libiomp5md.dll`); + # the best approach is to delete the conflicting libraries + # (i.e., OpenMP dynamic libraries that are not the VC143 version). + os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" + libomp_path = (pathlib.Path(__file__).parent / "lib" / "libomp140.x86_64.dll") if not libomp_path.exists(): @@ -54,7 +60,7 @@ def _preload_openmp_runtime(): libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) libggml_base_paths = [ libggml_base_path / "lib", - libggml_base_path / "bin", + # libggml_base_path / "bin", # The `bin` path is no longer used as a search path for dynamic ggml libraries. ] # Load bundled OpenMP runtime before ggml-base on Windows. diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 9b37ebcc74..7cbd87e4b5 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -868,6 +868,63 @@ def get_embeddings_seq(self, seq_id: int): self._assert_ctx() return llama_cpp.llama_get_embeddings_seq(self.ctx, seq_id) + def set_embeddings_nextn(self, enabled: bool, masked: bool) -> None: + """ + Set whether the context outputs nextn embeddings or not + If masked == true, output the embeddings only for the tokens with batch.logits != 0 + If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits + """ + self._assert_ctx() + llama_cpp.llama_set_embeddings_nextn(self.ctx, enabled, masked) + + def get_embeddings_nextn(self): + self._assert_ctx() + embeddings = llama_cpp.llama_get_embeddings_nextn(self.ctx) + if not embeddings: + raise RuntimeError("LlamaContext.get_embeddings_nextn: output is unavailable") + return embeddings + + def get_embeddings_nextn_ith(self, i: int): + self._assert_ctx() + embeddings = llama_cpp.llama_get_embeddings_nextn_ith(self.ctx, i) + if not embeddings: + raise RuntimeError( + f"LlamaContext.get_embeddings_nextn_ith: invalid output index {i}" + ) + return embeddings + + def set_embeddings_layer_inp(self, layer_id: int, enabled: bool) -> None: + self._assert_ctx() + if layer_id < 0: + raise ValueError("layer_id must be non-negative") + llama_cpp.llama_set_embeddings_layer_inp(self.ctx, layer_id, enabled) + + def get_embeddings_layer_inp(self, layer_id: int): + self._assert_ctx() + if layer_id < 0: + raise ValueError("layer_id must be non-negative") + embeddings = llama_cpp.llama_get_embeddings_layer_inp(self.ctx, layer_id) + if not embeddings: + raise RuntimeError( + f"LlamaContext.get_embeddings_layer_inp: layer {layer_id} output is unavailable" + ) + return embeddings + + def set_nextn_layer_offset(self, offset: int) -> None: + """ + Select which appended NextN block the DECODER_MTP graph runs (offset past + the trunk: il = n_layer() + offset). Used by the speculative NextN driver to + chain multiple trained NextN heads. Default 0 (first head). + """ + self._assert_ctx() + if offset < 0: + raise ValueError("NextN layer offset must be non-negative") + llama_cpp.llama_set_nextn_layer_offset(self.ctx, offset) + + def get_ctx_other(self): + self._assert_ctx() + return llama_cpp.llama_get_ctx_other(self.ctx) + def reset_timings(self): llama_cpp.llama_perf_context_reset(self.ctx) @@ -2007,6 +2064,7 @@ def _build_sampler_chain(self): # Note: In some implementations, penalties come before other samplers if CommonSamplerType.PENALTIES in p.samplers: s.add_penalties( + self.n_vocab, p.penalty_last_n, p.penalty_repeat, p.penalty_freq, @@ -3119,8 +3177,8 @@ def add_grammar( c_trigger_tokens, len(trigger_tokens) )) - def add_penalties(self, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float): - self._add_sampler(llama_cpp.llama_sampler_init_penalties(penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) + def add_penalties(self, n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float): + self._add_sampler(llama_cpp.llama_sampler_init_penalties(n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) def add_dry(self, model: LlamaModel, multiplier: float, base: float, allowed_len: int, last_n: int, breakers: List[str]): """DRY (Don't Repeat Yourself) sampler.""" diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index f733d7afb9..8092f86956 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -104,16 +104,19 @@ def __init__( cpu_moe: bool = False, n_cpu_moe: int = 0, split_mode: int = llama_cpp_lib.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, + load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_MMAP, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, - vocab_only: bool = False, - use_mmap: bool = True, + kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, + use_mmap: bool = False, use_direct_io: bool = False, use_mlock: bool = False, + vocab_only: bool = False, check_tensors: bool = False, - use_extra_bufts: bool = False, + use_extra_bufts: bool = True, no_host: bool = False, - kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, + no_alloc: bool = False, + load_mtp: bool = False, # Context Params seed: int = llama_cpp_lib.LLAMA_DEFAULT_SEED, n_ctx: int = 512, @@ -215,15 +218,16 @@ def __init__( n_cpu_moe: Keep the MoE expert weights of the first N layers on CPU. Useful when VRAM is insufficient for MoE models. split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options. + load_mode: How to load the model. See llama_cpp.LLAMA_LOAD_MODE_* for options. main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. + kv_overrides: Key-value overrides for the model. vocab_only: Only load the vocabulary no weights. - use_mmap: Use mmap if possible. - use_mlock: Force the system to keep the model in RAM. check_tensors: validate model tensor data use_extra_bufts: use extra buffer types (used for weight repacking) no_host: bypass host buffer allowing extra buffers to be used - kv_overrides: Key-value overrides for the model. + no_alloc: only load metadata and simulate memory allocations + load_mtp: whether to load MTP layers seed: RNG seed, -1 for random n_ctx: Text context, 0 = from model n_keep: Number of tokens to keep from initial prompt @@ -352,10 +356,19 @@ def __init__( self.model_path = model_path + if (use_mmap or use_direct_io or use_mlock) and verbose: + print( + "Llama.__init__: WARNING: " + "Legacy load options (`use_mmap`, `use_direct_io`, `use_mlock`) " + "are deprecated. Use `load_mode` instead.", + file=sys.stderr, + ) + # Model Params self.model_params = llama_cpp_lib.llama_model_default_params() self.model_params.n_gpu_layers = self._parse_n_gpu_layers(n_gpu_layers) self.model_params.split_mode = split_mode + self.model_params.load_mode = load_mode self.model_params.main_gpu = main_gpu self.tensor_split = tensor_split self._c_tensor_split = None @@ -371,12 +384,11 @@ def __init__( ) # keep a reference to the array so it is not gc'd self.model_params.tensor_split = self._c_tensor_split self.model_params.vocab_only = vocab_only - self.model_params.use_mmap = use_mmap - self.model_params.use_direct_io = use_direct_io - self.model_params.use_mlock = use_mlock self.model_params.check_tensors = check_tensors self.model_params.use_extra_bufts = use_extra_bufts self.model_params.no_host = no_host + self.model_params.no_alloc = no_alloc + self.model_params.load_mtp = load_mtp # Logic of cpu_moe, n_cpu_moe # Reference from llama.cpp/tools/llama-bench/llama-bench.cpp @@ -3445,16 +3457,16 @@ def __getstate__(self): cpu_moe=self.cpu_moe, n_cpu_moe=self.n_cpu_moe, split_mode=self.model_params.split_mode, + load_mode=self.model_params.load_mode, main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, + kv_overrides=self.kv_overrides, vocab_only=self.model_params.vocab_only, - use_mmap=self.model_params.use_mmap, - use_direct_io=self.model_params.use_direct_io, - use_mlock=self.model_params.use_mlock, check_tensors=self.model_params.check_tensors, use_extra_bufts=self.model_params.use_extra_bufts, no_host=self.model_params.no_host, - kv_overrides=self.kv_overrides, + no_alloc=self.model_params.no_alloc, + load_mtp=self.model_params.load_mtp, # Context Params seed=self._seed, n_ctx=self.context_params.n_ctx, diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 2f402cd74b..609e0bb3b5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -146,60 +146,63 @@ class llama_vocab_type(enum.IntEnum): # https://github.com/ggml-org/llama.cpp/blob/master/src/llama-vocab.h#L10 # // pre-tokenization types # enum llama_vocab_pre_type { -# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, -# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, -# LLAMA_VOCAB_PRE_TYPE_FALCON = 4, -# LLAMA_VOCAB_PRE_TYPE_MPT = 5, -# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, -# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, -# LLAMA_VOCAB_PRE_TYPE_REFACT = 8, -# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, -# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10, -# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, -# LLAMA_VOCAB_PRE_TYPE_OLMO = 12, -# LLAMA_VOCAB_PRE_TYPE_DBRX = 13, -# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, -# LLAMA_VOCAB_PRE_TYPE_PORO = 15, -# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16, -# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, -# LLAMA_VOCAB_PRE_TYPE_VIKING = 18, -# LLAMA_VOCAB_PRE_TYPE_JAIS = 19, -# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, -# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, -# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, -# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, -# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, -# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, -# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, -# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, -# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, -# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, -# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, -# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, -# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, -# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, -# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35, -# LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36, -# LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37, -# LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38, -# LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39, -# LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40, -# LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41, -# LLAMA_VOCAB_PRE_TYPE_AFMOE = 42, -# LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43, -# LLAMA_VOCAB_PRE_TYPE_YOUTU = 44, -# LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45, -# LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46, -# LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47, -# LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48, -# LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49, -# LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50, -# LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51, -# LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52, -# LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53, +# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, +# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, +# LLAMA_VOCAB_PRE_TYPE_FALCON = 4, +# LLAMA_VOCAB_PRE_TYPE_MPT = 5, +# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, +# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, +# LLAMA_VOCAB_PRE_TYPE_REFACT = 8, +# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, +# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10, +# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, +# LLAMA_VOCAB_PRE_TYPE_OLMO = 12, +# LLAMA_VOCAB_PRE_TYPE_DBRX = 13, +# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, +# LLAMA_VOCAB_PRE_TYPE_PORO = 15, +# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16, +# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, +# LLAMA_VOCAB_PRE_TYPE_VIKING = 18, +# LLAMA_VOCAB_PRE_TYPE_JAIS = 19, +# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, +# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, +# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, +# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, +# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, +# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, +# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, +# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, +# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, +# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, +# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, +# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, +# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, +# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, +# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35, +# LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36, +# LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37, +# LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38, +# LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39, +# LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40, +# LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41, +# LLAMA_VOCAB_PRE_TYPE_AFMOE = 42, +# LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43, +# LLAMA_VOCAB_PRE_TYPE_YOUTU = 44, +# LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45, +# LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46, +# LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47, +# LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48, +# LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49, +# LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50, +# LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51, +# LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52, +# LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53, +# LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54, +# LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55, +# LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56, # }; class llama_vocab_pre_type(enum.IntEnum): LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 @@ -256,6 +259,9 @@ class llama_vocab_pre_type(enum.IntEnum): LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51 LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52 LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53 + LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54 + LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55 + LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56 # // note: these values should be synchronized with ggml_rope @@ -504,6 +510,30 @@ class llama_split_mode(enum.IntEnum): LLAMA_SPLIT_MODE_ROW = 2 LLAMA_SPLIT_MODE_TENSOR = 3 +# enum llama_load_mode { +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available +# }; +class llama_load_mode(enum.IntEnum): + LLAMA_LOAD_MODE_NONE = 0 # no special loading mode + LLAMA_LOAD_MODE_MMAP = 1 # memory map the model + LLAMA_LOAD_MODE_MLOCK = 2 # force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_MMAP_MLOCK = 3 # mmap + force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_DIRECT_IO = 4 # use direct I/O if available + +# LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); +@ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p) +def llama_load_mode_name(load_mode: int) -> bytes: + ... + +# LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str); +@ctypes_function("llama_load_mode_from_str", [ctypes.c_char_p], ctypes.c_int) +def llama_load_mode_from_str(str: ctypes.c_char_p) -> int: + ... + # enum llama_context_type { # LLAMA_CONTEXT_TYPE_DEFAULT = 0, # LLAMA_CONTEXT_TYPE_MTP = 1, @@ -743,17 +773,15 @@ class llama_model_tensor_buft_override(ctypes.Structure): # struct llama_model_params { # // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) # ggml_backend_dev_t * devices; -# + # // NULL-terminated list of buffer types to use for tensors that match a pattern # const struct llama_model_tensor_buft_override * tensor_buft_overrides; -# + # int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers # enum llama_split_mode split_mode; // how to split the model across multiple GPUs +# enum llama_load_mode load_mode; // how to load the model -# // main_gpu interpretation depends on split_mode: -# // LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model -# // LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results -# // LLAMA_SPLIT_MODE_LAYER: ignored +# // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE # int32_t main_gpu; # // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() @@ -770,16 +798,13 @@ class llama_model_tensor_buft_override(ctypes.Structure): # // override key-value pairs of the model meta data # const struct llama_model_kv_override * kv_overrides; - # // Keep the booleans together to avoid misalignment during copy-by-value. # bool vocab_only; // only load the vocabulary, no weights -# bool use_mmap; // use mmap if possible -# bool use_direct_io; // use direct io, takes precedence over use_mmap when supported -# bool use_mlock; // force system to keep model in RAM # bool check_tensors; // validate model tensor data # bool use_extra_bufts; // use extra buffer types (used for weight repacking) # bool no_host; // bypass host buffer allowing extra buffers to be used # bool no_alloc; // only load metadata and simulate memory allocations +# bool load_mtp; // whether to load MTP layers # }; class llama_model_params(ctypes.Structure): """Parameters for llama_model @@ -789,57 +814,54 @@ class llama_model_params(ctypes.Structure): tensor_buft_overrides(llama_model_tensor_buft_override): NULL-terminated list of buffer types to use for tensors that match a pattern n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers split_mode (int): how to split the model across multiple GPUs + load_mode (int): how to load the model main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights - use_mmap (bool): use mmap if possible - use_direct_io(bool): use direct io, takes precedence over use_mmap when supported - use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking) no_host (bool): bypass host buffer allowing extra buffers to be used - no_alloc (bool): only load metadata and simulate memory allocations""" + no_alloc (bool): only load metadata and simulate memory allocations + load_mtp (bool): whether to load MTP layers""" if TYPE_CHECKING: devices: CtypesArray[ctypes.c_void_p] # NOTE: unused tensor_buft_overrides: CtypesPointer[llama_model_tensor_buft_override] n_gpu_layers: int split_mode: int + load_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] vocab_only: bool - use_mmap: bool - use_direct_io: bool - use_mlock: bool check_tensors: bool use_extra_bufts: bool no_host: bool no_alloc: bool + load_mtp: bool _fields_ = [ - ("devices", ctypes.c_void_p), # NOTE: unnused + ("devices", ctypes.POINTER(ctypes.c_void_p)), # NOTE: unnused ("tensor_buft_overrides", ctypes.POINTER(llama_model_tensor_buft_override)), ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), + ("load_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("vocab_only", ctypes.c_bool), - ("use_mmap", ctypes.c_bool), - ("use_direct_io", ctypes.c_bool), - ("use_mlock", ctypes.c_bool), ("check_tensors", ctypes.c_bool), ("use_extra_bufts", ctypes.c_bool), ("no_host", ctypes.c_bool), ("no_alloc", ctypes.c_bool), + ("load_mtp", ctypes.c_bool), ] llama_model_params_p = ctypes.POINTER(llama_model_params) @@ -3532,6 +3554,26 @@ def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: ... +# // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) +# LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens); +@ctypes_function( + "llama_vocab_get_suppress_tokens", + [ + llama_vocab_p_ctypes, + ctypes.POINTER(ctypes.c_int32), + ], + llama_token_p, +) +def llama_vocab_get_suppress_tokens( + vocab: llama_vocab_p, + n_suppress_tokens: ctypes.POINTER(ctypes.c_int32), # type: ignore +) -> llama_token_p: # type: ignore + """ + model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) + """ + ... + + # LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); @ctypes_function( "llama_vocab_fim_pre", @@ -4617,16 +4659,24 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( +# int32_t n_vocab, # int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) -# float penalty_repeat, // 1.0 = disabled -# float penalty_freq, // 0.0 = disabled -# float penalty_present); // 0.0 = disabled +# float penalty_repeat, // must be > 0.0, 1.0 = disabled +# float penalty_freq, // must be finite, 0.0 = disabled +# float penalty_present); // must be finite, 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", - [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], + [ + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_float, + ctypes.c_float, + ctypes.c_float, + ], llama_sampler_p_ctypes, ) def llama_sampler_init_penalties( + n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, @@ -5092,6 +5142,7 @@ def llama_opt_epoch( "llama_graph_reserve", "?llama_graph_reserve@@YAPEAUggml_cgraph@@PEAUllama_context@@III@Z", "__Z19llama_graph_reserveP13llama_contextjjj", + "_Z19llama_graph_reserveP13llama_contextjjj", ], [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32], ctypes.POINTER(ggml_cgraph), @@ -5115,6 +5166,7 @@ def llama_graph_reserve( "llama_ftype_get_default_type", "?llama_ftype_get_default_type@@YA?AW4ggml_type@@W4llama_ftype@@@Z", "__Z28llama_ftype_get_default_type11llama_ftype", + "_Z28llama_ftype_get_default_type11llama_ftype", ], [ctypes.c_int], int, @@ -5134,6 +5186,7 @@ def llama_ftype_get_default_type( "llama_model_n_expert", "?llama_model_n_expert@@YAHPEBUllama_model@@@Z", "__Z20llama_model_n_expertPK11llama_model", + "_Z20llama_model_n_expertPK11llama_model", ], [llama_model_p_ctypes], ctypes.c_int32, @@ -5150,6 +5203,7 @@ def llama_model_n_expert( "llama_model_n_devices", "?llama_model_n_devices@@YAHPEBUllama_model@@@Z", "__Z21llama_model_n_devicesPK11llama_model", + "_Z21llama_model_n_devicesPK11llama_model", ], [llama_model_p_ctypes], ctypes.c_int32, @@ -5166,6 +5220,7 @@ def llama_model_n_devices( "llama_model_get_device", "?llama_model_get_device@@YAPEAUggml_backend_device@@PEBUllama_model@@H@Z", "__Z22llama_model_get_devicePK11llama_modeli", + "_Z22llama_model_get_devicePK11llama_modeli", ], [llama_model_p_ctypes, ctypes.c_int], ctypes.c_void_p, @@ -5186,6 +5241,7 @@ def llama_model_get_device( "llama_set_embeddings_nextn", "?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z", "__Z26llama_set_embeddings_nextnP13llama_contextbb", + "_Z26llama_set_embeddings_nextnP13llama_contextbb", ], [llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], None, @@ -5212,6 +5268,7 @@ def llama_set_embeddings_nextn( "llama_set_nextn_layer_offset", "?llama_set_nextn_layer_offset@@YAXPEAUllama_context@@H@Z", "__Z28llama_set_nextn_layer_offsetP13llama_contexti", + "_Z28llama_set_nextn_layer_offsetP13llama_contexti", ], [llama_context_p_ctypes, ctypes.c_int32], None, @@ -5236,6 +5293,7 @@ def llama_set_nextn_layer_offset( "llama_get_embeddings_nextn", "?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z", "__Z26llama_get_embeddings_nextnP13llama_context", + "_Z26llama_get_embeddings_nextnP13llama_context", ], [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float), @@ -5253,6 +5311,7 @@ def llama_get_embeddings_nextn( "llama_get_embeddings_nextn_ith", "?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z", "__Z30llama_get_embeddings_nextn_ithP13llama_contexti", + "_Z30llama_get_embeddings_nextn_ithP13llama_contexti", ], [llama_context_p_ctypes, ctypes.c_int32], ctypes.POINTER(ctypes.c_float), @@ -5271,16 +5330,17 @@ def llama_get_embeddings_nextn_ith( "llama_set_embeddings_layer_inp", "?llama_set_embeddings_layer_inp@@YAXPEAUllama_context@@I_N@Z", "__Z30llama_set_embeddings_layer_inpP13llama_contextjb", + "_Z30llama_set_embeddings_layer_inpP13llama_contextjb", ], - [llama_context_p_ctypes, ctypes.c_int32, ctypes.c_bool], - ctypes.POINTER(ctypes.c_float), + [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_bool], + None, required=False, ) def llama_set_embeddings_layer_inp( ctx: llama_context_p, - lid: ctypes.c_int32, + lid: ctypes.c_uint32, value: bool, -) -> ctypes.POINTER(ctypes.c_float): # type: ignore +) -> None: # type: ignore """ Set whether the context outputs the input embeddings of a specific layer """ @@ -5294,14 +5354,15 @@ def llama_set_embeddings_layer_inp( "llama_get_embeddings_layer_inp", "?llama_get_embeddings_layer_inp@@YAPEAMPEAUllama_context@@I@Z", "__Z30llama_get_embeddings_layer_inpP13llama_contextj", + "_Z30llama_get_embeddings_layer_inpP13llama_contextj", ], - [llama_context_p_ctypes, ctypes.c_int32], + [llama_context_p_ctypes, ctypes.c_uint32], ctypes.POINTER(ctypes.c_float), required=False, ) def llama_get_embeddings_layer_inp( ctx: llama_context_p, - lid: ctypes.c_int32, + lid: ctypes.c_uint32, ) -> ctypes.POINTER(ctypes.c_float): # type: ignore ... @@ -5311,6 +5372,7 @@ def llama_get_embeddings_layer_inp( "llama_get_ctx_other", "?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z", "__Z19llama_get_ctx_otherP13llama_context", + "_Z19llama_get_ctx_otherP13llama_context", ], [llama_context_p_ctypes], llama_context_p_ctypes, @@ -5330,6 +5392,7 @@ def llama_get_ctx_other( "llama_model_target_layer_ids", "?llama_model_target_layer_ids@@YAPEBHPEBUllama_model@@@Z", "__Z28llama_model_target_layer_idsPK11llama_model", + "_Z28llama_model_target_layer_idsPK11llama_model", ], [llama_model_p_ctypes], ctypes.POINTER(ctypes.c_int32), @@ -5349,15 +5412,16 @@ def llama_model_target_layer_ids( [ "llama_model_target_layer_ids_n", "?llama_model_target_layer_ids_n@@YAIPEBUllama_model@@@Z", - "__Z30llama_model_target_layer_ids_nPK11llama_model" + "__Z30llama_model_target_layer_ids_nPK11llama_model", + "_Z30llama_model_target_layer_ids_nPK11llama_model", ], [llama_model_p_ctypes], - ctypes.POINTER(ctypes.c_uint32), + ctypes.c_uint32, required=False, ) def llama_model_target_layer_ids_n( model: llama_model_p -) -> ctypes.POINTER(ctypes.c_uint32): # type: ignore +) -> int: """ returns the number of extracted layers from target model """ diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 6b3fd1dd15..0d509bbcf9 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -294,12 +294,10 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: # Model Params n_gpu_layers=settings.n_gpu_layers, split_mode=settings.split_mode, + load_mode=settings.load_mode, main_gpu=settings.main_gpu, tensor_split=settings.tensor_split, vocab_only=settings.vocab_only, - use_mmap=settings.use_mmap, - use_direct_io=settings.use_direct_io, - use_mlock=settings.use_mlock, check_tensors=settings.check_tensors, use_extra_bufts=settings.use_extra_bufts, no_host=settings.no_host, diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 350ccc2323..62ce3b5044 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -32,8 +32,12 @@ class ModelSettings(BaseSettings): ) split_mode: int = Field( default=llama_cpp.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, - description="The split mode to use.", + description="how to split the model across multiple GPUs", ) + load_mode: int = Field( + default=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP, + description="how to load the model", + ) main_gpu: int = Field( default=0, ge=0, @@ -46,18 +50,6 @@ class ModelSettings(BaseSettings): vocab_only: bool = Field( default=False, description="Whether to only return the vocabulary." ) - use_mmap: bool = Field( - default=True, - description="Enable mmap to use filesystem cache.", - ) - use_direct_io: bool = Field( - default=False, - description="Use direct io, takes precedence over use_mmap.", - ) - use_mlock: bool = Field( - default=False, - description="Use mlock for force system to keep model in RAM", - ) check_tensors: bool = Field( default=False, description="Validate model tensor data.", diff --git a/tests/test_llama.py b/tests/test_llama.py index b233ea5266..d0053feaba 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -260,9 +260,6 @@ def test_real_model(llama_cpp_model_path): # 1. Setup Model Parameters params = llama_cpp.llama_model_default_params() - params.use_mmap = llama_cpp.llama_supports_mmap() - params.use_direct_io = False - params.use_mlock = llama_cpp.llama_supports_mlock() params.check_tensors = False # 2. Load the Model diff --git a/tools/abi/README.md b/tools/abi/README.md new file mode 100644 index 0000000000..ea509f2f7a --- /dev/null +++ b/tools/abi/README.md @@ -0,0 +1,164 @@ +# Cross-platform ABI inspection + +Author: **JamePeng** + +This repository-only tool inspects PE (`.dll`), ELF (`.so` and `.so.*`), and +Mach-O (`.dylib`) exports. Its primary purpose is to collect ctypes symbol +candidates and verify optional `llama_ext` bindings across MSVC, GCC/Clang, +and macOS builds. + +## Boundary and safety + +The tool is intentionally excluded from wheels: + +```toml +wheel.packages = ["llama_cpp"] +``` + +It is not imported by `llama_cpp`, has no installed command, and keeps LIEF +out of project dependencies. Run it only from a trusted source checkout. +LIEF parses native binaries, so do not scan untrusted artifacts. + +Install the maintainer-only dependency: + +```bash +python -m pip install lief +``` + +The tool and its documentation use the same MIT License as this repository. + +## Artifact layout + +Run commands from the repository root. Put builds under +`tools/abi/artifacts`, or replace that argument with an external absolute +directory: + +```text +tools/abi/artifacts/ +├── windows-x86_64/ +│ └── +├── linux-x86_64/ +│ └── +└── macos-arm64/ + └── +``` + +Names are not significant. `--select-symbol llama_decode` identifies the +llama library by content when dependency and backend libraries share the same +directory. + +Artifacts may come from local builds, an installed or extracted wheel, +[project releases](https://github.com/JamePeng/llama-cpp-python/releases), or +[upstream releases](https://github.com/ggml-org/llama.cpp/releases). Record +the source revision, compiler, architecture, and build options. Upstream +artifacts may not contain fork-only `llama_ext` APIs. + +## Scan exports + +```bash +python -m tools.abi scan tools/abi/artifacts --recursive +``` + +The default output is one same-named JSONL file per library: + +```text +tools/abi/output/ +└── 20260728T153012.123456Z/ + ├── llama.dll.jsonl + ├── libllama.so.jsonl + └── libllama.dylib.jsonl +``` + +Useful options: + +```bash +# Select only binaries that export llama_decode. +python -m tools.abi scan tools/abi/artifacts --recursive \ + --select-symbol llama_decode + +# Print instead of writing per-library JSONL. +python -m tools.abi scan tools/abi/artifacts --recursive --format text + +# Write one aggregate file; its filename receives a UTC timestamp. +python -m tools.abi scan tools/abi/artifacts --recursive \ + --format jsonl --output all-symbols.jsonl +``` + +`--prefix` is optional. By default all exports are retained: + +```bash +python -m tools.abi scan tools/abi/artifacts --recursive \ + --prefix llama_ --prefix ggml_ +``` + +## Check optional llama_ext bindings + +This is the primary ABI validation command: + +```bash +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive \ + --source llama_cpp/llama_cpp.py +``` + +It statically reads ctypes decorators without importing `llama_cpp`. The +default `--scope optional` checks declarations marked `required=False` and +returns exit code 1 if any candidate is missing. Other scopes are available: + +```bash +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive --scope required +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive --scope all +``` + +## Compare and create a manifest + +```bash +python -m tools.abi compare tools/abi/artifacts \ + --recursive --select-symbol llama_decode + +python -m tools.abi manifest tools/abi/artifacts \ + --recursive --select-symbol llama_decode \ + --output llama-exports.json +``` + +Cross-platform comparison uses `canonical_name`: + +```text +?llama_graph_reserve@@... MSVC +_Z19llama_graph_reserve... Linux Itanium ABI +__Z19llama_graph_reserve... Mach-O symbol table + ↓ +llama_graph_reserve canonical name +``` + +Records retain `raw_name`, ctypes `lookup_name`, `canonical_name`, ABI, +address, ordinal, library filename, format, architecture, SHA-256, and UTC +generation time. They never contain the artifact's absolute source path. + +Every run receives a timestamp, preventing normal output from overwriting +previous results. Generated artifacts and reports are ignored by Git. + +## Verification + +Unit tests are independent from the project's default test suite: + +```bash +python -m pytest tools/abi/tests/test_scan_dynamic.py -q +``` + +The opt-in integration test requires Windows, Linux, and macOS artifacts: + +```powershell +$env:LLAMA_ABI_ARTIFACTS = "tools/abi/artifacts" +python -m pytest tools/abi/tests/test_platform_artifacts.py -q +``` + +```bash +LLAMA_ABI_ARTIFACTS=tools/abi/artifacts \ +python -m pytest tools/abi/tests/test_platform_artifacts.py -q +``` + +Without configured artifacts, integration tests skip. With +`LLAMA_ABI_ARTIFACTS` set, a missing platform or optional ABI alias fails. diff --git a/tools/abi/__init__.py b/tools/abi/__init__.py new file mode 100644 index 0000000000..0a387e6759 --- /dev/null +++ b/tools/abi/__init__.py @@ -0,0 +1,3 @@ +"""Cross-platform shared-library ABI inspection tools.""" + +__author__ = "JamePeng" diff --git a/tools/abi/__main__.py b/tools/abi/__main__.py new file mode 100644 index 0000000000..acfe21acfa --- /dev/null +++ b/tools/abi/__main__.py @@ -0,0 +1,5 @@ +from .scan_dynamic import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/abi/artifacts/.gitignore b/tools/abi/artifacts/.gitignore new file mode 100644 index 0000000000..dbea0aa373 --- /dev/null +++ b/tools/abi/artifacts/.gitignore @@ -0,0 +1,4 @@ +# Keep downloaded or locally built native libraries out of Git. +* +!.gitignore +!README.md diff --git a/tools/abi/artifacts/README.md b/tools/abi/artifacts/README.md new file mode 100644 index 0000000000..30863ea8c0 --- /dev/null +++ b/tools/abi/artifacts/README.md @@ -0,0 +1,20 @@ +# ABI artifacts + +Maintainer: **JamePeng** + +Place trusted Windows, Linux, and macOS build artifacts here for local ABI +inspection. Filenames do not need to follow a fixed convention. + +```text +tools/abi/artifacts/ +├── windows-x86_64/ +├── linux-x86_64/ +├── macos-arm64/ +└── macos-x86_64/ +``` + +Downloaded and copied content is ignored by both the local and repository +`.gitignore`; only this README and `.gitignore` are tracked. `git add -f` can +still deliberately override ignore rules. + +See `tools/abi/README.md` for commands and artifact provenance requirements. diff --git a/tools/abi/output/.gitignore b/tools/abi/output/.gitignore new file mode 100644 index 0000000000..f12e9f061e --- /dev/null +++ b/tools/abi/output/.gitignore @@ -0,0 +1,4 @@ +# Keep generated ABI reports local. +* +!.gitignore +!README.md diff --git a/tools/abi/output/README.md b/tools/abi/output/README.md new file mode 100644 index 0000000000..3601ac132e --- /dev/null +++ b/tools/abi/output/README.md @@ -0,0 +1,7 @@ +# Local ABI reports + +Each run is stored in a UTC timestamp directory. Reports omit artifact source +paths but may contain binary hashes and non-public symbols. + +Generated content is ignored by both the local and repository `.gitignore`; +only this README and `.gitignore` are tracked. Review reports before sharing. diff --git a/tools/abi/scan_dynamic.py b/tools/abi/scan_dynamic.py new file mode 100644 index 0000000000..8ee11fd8c6 --- /dev/null +++ b/tools/abi/scan_dynamic.py @@ -0,0 +1,863 @@ +"""Inspect and compare exported symbols in PE, ELF, and Mach-O libraries. + +This repository-only maintainer utility supports collection and verification +of cross-platform ctypes symbol candidates, with particular focus on optional +llama_ext APIs. + +LIEF is imported lazily so that ``--help`` remains available when the optional +dependency is not installed. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import re +import sys +from collections import defaultdict +from dataclasses import asdict, dataclass, replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Sequence + +LIBRARY_SUFFIXES = {".dll", ".dylib", ".so"} +__author__ = "JamePeng" + + +class ScanError(RuntimeError): + """Raised when a shared library cannot be inspected.""" + + +@dataclass(frozen=True) +class SymbolRecord: + """One exported symbol and its cross-platform names.""" + + raw_name: str + lookup_name: str + canonical_name: str + abi: str + address: str + ordinal: int | None = None + + +@dataclass(frozen=True) +class BindingDeclaration: + """One ctypes decorator declaration extracted without importing llama_cpp.""" + + python_name: str + candidates: tuple[str, ...] + required: bool + line: int + + +@dataclass(frozen=True) +class LibraryScan: + """Metadata and exported symbols for one binary architecture.""" + + library: str + format: str + platform: str + architecture: str + sha256: str + symbols: tuple[SymbolRecord, ...] + + +def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def generation_timestamp() -> str: + """Return a sortable, collision-resistant UTC generation timestamp.""" + + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + + +def _enum_name(value: Any) -> str: + text = str(value) + return text.rsplit(".", 1)[-1] + + +def get_format(binary: Any) -> str: + value = str(binary.format).upper() + if "MACHO" in value: + return "Mach-O" + if "ELF" in value: + return "ELF" + if "PE" in value: + return "PE" + return str(binary.format) + + +def get_platform(binary_format: str) -> str: + return { + "PE": "windows", + "ELF": "linux", + "Mach-O": "darwin", + }.get(binary_format, "unknown") + + +def get_architecture(binary: Any, binary_format: str) -> str: + header = binary.header + if binary_format == "PE": + return _enum_name(header.machine) + if binary_format == "ELF": + return _enum_name(header.machine_type) + if binary_format == "Mach-O": + return _enum_name(header.cpu_type) + return "unknown" + + +def normalize_symbol_name(raw_name: str, binary_format: str) -> str: + """Return the name used by ctypes/dlsym and cross-platform comparison. + + Mach-O symbol tables prefix external C names with an underscore. dlsym and + ctypes callers use the source-level name without that platform prefix. + """ + + if binary_format == "Mach-O" and raw_name.startswith("_"): + return raw_name[1:] + return raw_name + + +def detect_abi(normalized_name: str) -> str: + if normalized_name.startswith("?"): + return "msvc-cxxabi" + if normalized_name.startswith("_Z"): + return "itanium-cxxabi" + return "unmangled" + + +def canonicalize_symbol_name(normalized_name: str, abi: str) -> str: + """Recover a source-level name from simple global C++ mangling. + + llama_ext functions are global functions, so their MSVC and Itanium + spellings can be mapped without a full ABI demangler. Namespaced, + overloaded, and templated symbols remain mangled to avoid false matches. + """ + + if abi == "msvc-cxxabi": + match = re.match(r"^\?([^@?$]+)@@", normalized_name) + if match: + return match.group(1) + + if abi == "itanium-cxxabi": + match = re.match(r"^_Z(\d+)", normalized_name) + if match: + length = int(match.group(1)) + start = match.end() + candidate = normalized_name[start : start + length] + if len(candidate) == length: + return candidate + + return normalized_name + + +def _symbol_address(symbol: Any) -> str: + value = getattr(symbol, "address", None) + if value is None: + value = getattr(symbol, "value", 0) + return hex(int(value)) + + +def _exported_symbols(binary: Any, binary_format: str) -> Iterable[Any]: + if binary_format == "PE": + if not binary.has_exports: + return () + return binary.get_export().entries + + # LIEF's exported_symbols filters undefined ELF imports and non-exported + # Mach-O symbols, unlike dynamic_symbols/symbols. + return binary.exported_symbols + + +def _iter_binaries(parsed: Any) -> list[Any]: + # A universal Mach-O may contain several architecture slices. + if type(parsed).__name__ == "FatBinary": + return list(parsed) + return [parsed] + + +def scan_library( + path: str | Path, +) -> list[LibraryScan]: + """Inspect one library, returning one result per architecture slice.""" + + try: + import lief + except ImportError as exc: + raise ScanError( + "LIEF is required for ABI inspection. Install it with: pip install lief" + ) from exc + + library_path = Path(path).expanduser().resolve() + if not library_path.is_file(): + raise ScanError(f"Not a file: {library_path.name}") + + try: + parsed = lief.parse(str(library_path)) + except Exception as exc: + detail = str(exc).replace(str(library_path), library_path.name) + raise ScanError(f"Failed to parse {library_path.name}: {detail}") from exc + + if parsed is None: + raise ScanError(f"LIEF did not recognize {library_path.name}") + + digest = sha256_file(library_path) + results: list[LibraryScan] = [] + + for binary in _iter_binaries(parsed): + binary_format = get_format(binary) + records: list[SymbolRecord] = [] + + for symbol in _exported_symbols(binary, binary_format): + raw_name = getattr(symbol, "name", None) + if not raw_name: + # PE supports ordinal-only exports. They cannot be matched to + # Python bindings by name, so keep a stable synthetic label. + ordinal = getattr(symbol, "ordinal", None) + if ordinal is None: + continue + raw_name = f"#{ordinal}" + + lookup_name = normalize_symbol_name(raw_name, binary_format) + abi = detect_abi(lookup_name) + canonical_name = canonicalize_symbol_name(lookup_name, abi) + + records.append( + SymbolRecord( + raw_name=raw_name, + lookup_name=lookup_name, + canonical_name=canonical_name, + abi=abi, + address=_symbol_address(symbol), + ordinal=getattr(symbol, "ordinal", None), + ) + ) + + records.sort(key=lambda item: (item.canonical_name, item.raw_name)) + results.append( + LibraryScan( + library=library_path.name, + format=binary_format, + platform=get_platform(binary_format), + architecture=get_architecture(binary, binary_format), + sha256=digest, + symbols=tuple(records), + ) + ) + + return results + + +def select_scans_by_symbols( + scans: Sequence[LibraryScan], + required_symbols: Sequence[str], +) -> list[LibraryScan]: + """Select binaries by exported canonical names, independent of filenames.""" + + if not required_symbols: + return list(scans) + selected = [] + for scan in scans: + exported = {symbol.canonical_name for symbol in scan.symbols} + if all(name in exported for name in required_symbols): + selected.append(scan) + return selected + + +def filter_scan_symbols( + scans: Sequence[LibraryScan], + prefixes: Sequence[str], +) -> list[LibraryScan]: + if not prefixes: + return list(scans) + return [ + replace( + scan, + symbols=tuple( + symbol + for symbol in scan.symbols + if any(symbol.canonical_name.startswith(prefix) for prefix in prefixes) + ), + ) + for scan in scans + ] + + +def extract_ctypes_bindings(source: str | Path) -> list[BindingDeclaration]: + """Extract literal ctypes decorator candidates without importing the module.""" + + source_path = Path(source) + try: + tree = ast.parse( + source_path.read_text(encoding="utf-8"), + filename=str(source_path), + ) + except (OSError, SyntaxError) as exc: + raise ScanError(f"Failed to parse binding source {source_path}: {exc}") from exc + + declarations: list[BindingDeclaration] = [] + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for decorator in node.decorator_list: + if not isinstance(decorator, ast.Call) or not decorator.args: + continue + decorator_name = "" + if isinstance(decorator.func, ast.Name): + decorator_name = decorator.func.id + elif isinstance(decorator.func, ast.Attribute): + decorator_name = decorator.func.attr + if not decorator_name.startswith("ctypes_function"): + continue + + try: + names = ast.literal_eval(decorator.args[0]) + except (ValueError, TypeError): + continue + if isinstance(names, str): + candidates = (names,) + elif isinstance(names, (list, tuple)) and all( + isinstance(name, str) for name in names + ): + candidates = tuple(names) + else: + continue + + required = True + for keyword in decorator.keywords: + if keyword.arg == "required": + try: + required = bool(ast.literal_eval(keyword.value)) + except (ValueError, TypeError): + pass + + declarations.append( + BindingDeclaration( + python_name=node.name, + candidates=candidates, + required=required, + line=node.lineno, + ) + ) + + return sorted(declarations, key=lambda item: item.line) + + +def check_bindings( + scan: LibraryScan, + declarations: Sequence[BindingDeclaration], +) -> dict[str, Any]: + """Check which ctypes candidate would be selected for one library.""" + + exported = {symbol.lookup_name for symbol in scan.symbols} + available = [] + missing_required = [] + missing_optional = [] + + for declaration in declarations: + selected = next( + (name for name in declaration.candidates if name in exported), + None, + ) + item = { + "python_name": declaration.python_name, + "required": declaration.required, + "line": declaration.line, + "candidates": list(declaration.candidates), + "selected": selected, + } + if selected is not None: + available.append(item) + elif declaration.required: + missing_required.append(item) + else: + missing_optional.append(item) + + return { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "declaration_count": len(declarations), + "available_count": len(available), + "available": available, + "missing_required": missing_required, + "missing_optional": missing_optional, + } + + +def compare_scans(scans: Sequence[LibraryScan]) -> dict[str, Any]: + if len(scans) < 2: + raise ValueError("At least two library scans are required for comparison") + + symbol_sets = [{symbol.canonical_name for symbol in scan.symbols} for scan in scans] + common = set.intersection(*symbol_sets) + libraries = [] + + for index, scan in enumerate(scans): + others = set.union(*(symbol_sets[i] for i in range(len(scans)) if i != index)) + libraries.append( + { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "symbol_count": len(symbol_sets[index]), + "only_here": sorted(symbol_sets[index] - others), + "missing_here": sorted(others - symbol_sets[index]), + } + ) + + return { + "common_count": len(common), + "common": sorted(common), + "libraries": libraries, + } + + +def build_manifest( + scans: Sequence[LibraryScan], + *, + generated_at: str | None = None, +) -> dict[str, Any]: + generated_at = generated_at or generation_timestamp() + symbols: dict[str, list[dict[str, Any]]] = defaultdict(list) + libraries = [] + + for scan in scans: + libraries.append(_scan_metadata(scan)) + for symbol in scan.symbols: + symbols[symbol.canonical_name].append( + { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "raw_name": symbol.raw_name, + "lookup_name": symbol.lookup_name, + "abi": symbol.abi, + "address": symbol.address, + "ordinal": symbol.ordinal, + } + ) + + return { + "schema_version": 1, + "generated_at": generated_at, + "libraries": libraries, + "symbols": dict(sorted(symbols.items())), + } + + +def collect_library_paths( + inputs: Sequence[str], + *, + recursive: bool = False, +) -> list[Path]: + def is_shared_library(path: Path) -> bool: + name = path.name.lower() + return path.suffix.lower() in LIBRARY_SUFFIXES or ".so." in name + + paths: list[Path] = [] + for value in inputs: + path = Path(value).expanduser() + if path.is_dir(): + candidates = path.rglob("*") if recursive else path.iterdir() + paths.extend( + candidate + for candidate in candidates + if candidate.is_file() and is_shared_library(candidate) + ) + else: + paths.append(path) + return sorted(set(paths), key=lambda item: str(item).lower()) + + +def _scan_paths( + paths: Sequence[Path], +) -> tuple[list[LibraryScan], list[str]]: + scans: list[LibraryScan] = [] + errors: list[str] = [] + for path in paths: + try: + scans.extend(scan_library(path)) + except ScanError as exc: + errors.append(str(exc)) + except Exception as exc: + detail = str(exc).replace(str(path.resolve()), path.name) + errors.append(f"{path.name}: {detail}") + return scans, errors + + +def _timestamped_output_path(output: str | Path, timestamp: str) -> Path: + path = Path(output) + return path.with_name(f"{path.stem}.{timestamp}{path.suffix}") + + +def _write_output( + text: str, + output: str | None, + *, + timestamp: str, +) -> None: + if output: + output_path = _timestamped_output_path(output, timestamp) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(text + "\n", encoding="utf-8") + print(f"saved: {output_path}") + else: + print(text) + + +def _scan_metadata(scan: LibraryScan) -> dict[str, Any]: + return {key: value for key, value in asdict(scan).items() if key != "symbols"} + + +def _jsonl_rows(scan: LibraryScan, generated_at: str) -> list[str]: + metadata = _scan_metadata(scan) + return [ + json.dumps( + { + "generated_at": generated_at, + **metadata, + **asdict(symbol), + }, + ensure_ascii=False, + ) + for symbol in scan.symbols + ] + + +def write_library_jsonl( + scans: Sequence[LibraryScan], + output_dir: str | Path = "tools/abi/output", + *, + timestamp: str | None = None, +) -> list[Path]: + """Write one same-named JSONL per library under a timestamped run directory.""" + + timestamp = timestamp or generation_timestamp() + destination = Path(output_dir) / timestamp + destination.mkdir(parents=True, exist_ok=True) + grouped: dict[str, list[LibraryScan]] = defaultdict(list) + for scan in scans: + grouped[scan.library].append(scan) + + written = [] + for library, library_scans in sorted(grouped.items()): + output_path = destination / f"{library}.jsonl" + rows = [ + row + for library_scan in library_scans + for row in _jsonl_rows(library_scan, timestamp) + ] + output_path.write_text( + "\n".join(rows) + ("\n" if rows else ""), + encoding="utf-8", + ) + written.append(output_path) + return written + + +def _scan_text(scans: Sequence[LibraryScan], errors: Sequence[str]) -> str: + lines: list[str] = [] + for scan in scans: + lines.append( + f"{scan.library} [{scan.format}/{scan.architecture}]: " + f"{len(scan.symbols)} exported symbol(s)" + ) + for symbol in scan.symbols: + raw_suffix = ( + f" (raw: {symbol.raw_name})" + if symbol.raw_name != symbol.canonical_name + else "" + ) + lines.append( + f" {symbol.canonical_name} [{symbol.abi}]" + f" @ {symbol.address}{raw_suffix}" + ) + for error in errors: + lines.append(f"ERROR: {error}") + return "\n".join(lines) + + +def _compare_text(comparison: dict[str, Any]) -> str: + lines = [f"Common canonical symbols: {comparison['common_count']}"] + for library in comparison["libraries"]: + lines.extend( + [ + "", + ( + f"{library['library']} " + f"[{library['platform']}/{library['architecture']}]: " + f"{library['symbol_count']} symbol(s)" + ), + f" Only here: {len(library['only_here'])}", + ] + ) + lines.extend(f" {name}" for name in library["only_here"]) + lines.append(f" Missing here: {len(library['missing_here'])}") + lines.extend(f" {name}" for name in library["missing_here"]) + return "\n".join(lines) + + +def _bindings_text( + results: Sequence[dict[str, Any]], + scope: str, +) -> str: + lines: list[str] = [] + for result in results: + lines.append( + f"{result['library']} " + f"[{result['platform']}/{result['architecture']}]: " + f"{result['available_count']}/{result['declaration_count']} " + "binding(s) available in selected scope" + ) + if scope in {"required", "all"}: + lines.append(f" Missing required: {len(result['missing_required'])}") + lines.extend( + f" {item['python_name']} (line {item['line']})" + for item in result["missing_required"] + ) + if scope in {"optional", "all"}: + lines.append(f" Missing optional: {len(result['missing_optional'])}") + lines.extend( + f" {item['python_name']} (line {item['line']})" + for item in result["missing_optional"] + ) + return "\n".join(lines) + + +def create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Inspect and compare PE, ELF, and Mach-O exported symbols." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + def add_common(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument("paths", nargs="+", help="Library files or directories") + subparser.add_argument( + "--prefix", + action="append", + default=[], + help=( + "Optional canonical-name filter; may be repeated " + "(default: keep all exports)" + ), + ) + subparser.add_argument( + "--select-symbol", + action="append", + default=[], + help=( + "Select libraries exporting this canonical symbol; may be " + "repeated and does not depend on the library filename" + ), + ) + subparser.add_argument( + "--recursive", + action="store_true", + help="Recursively search directory inputs", + ) + subparser.add_argument("-o", "--output", help="Write output to this file") + + scan_parser = subparsers.add_parser("scan", help="List exported symbols") + add_common(scan_parser) + scan_parser.add_argument( + "--format", + choices=("text", "json", "jsonl"), + default="jsonl", + help="Output format", + ) + scan_parser.add_argument( + "--output-dir", + default="tools/abi/output", + help=( + "Directory for default per-library JSONL files " + "(default: tools/abi/output)" + ), + ) + + compare_parser = subparsers.add_parser( + "compare", help="Compare canonical symbol names across libraries" + ) + add_common(compare_parser) + compare_parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="Output format", + ) + + manifest_parser = subparsers.add_parser( + "manifest", help="Create a cross-platform symbol manifest" + ) + add_common(manifest_parser) + + bindings_parser = subparsers.add_parser( + "check-bindings", + help="Check literal ctypes decorator candidates against libraries", + ) + add_common(bindings_parser) + bindings_parser.add_argument( + "--source", + default="llama_cpp/llama_cpp.py", + help="Python binding source to inspect without importing it", + ) + bindings_parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="Output format", + ) + bindings_parser.add_argument( + "--scope", + choices=("optional", "required", "all"), + default="optional", + help=("Binding declarations to check " "(default: optional llama_ext APIs)"), + ) + + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = create_parser().parse_args(argv) + paths = collect_library_paths(args.paths, recursive=args.recursive) + if not paths: + print("No shared libraries found.", file=sys.stderr) + return 2 + + # Scan all exports first. Selection must not depend on --prefix, because a + # caller may use an anchor outside the displayed prefix set. + scans, errors = _scan_paths(paths) + if not scans: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + if args.select_symbol: + scans = select_scans_by_symbols(scans, args.select_symbol) + if not scans: + print( + "No library exports all requested selection symbols: " + + ", ".join(args.select_symbol), + file=sys.stderr, + ) + return 2 + + scans = filter_scan_symbols(scans, args.prefix) + if args.command in {"compare", "manifest"} and args.prefix: + # A package lib directory normally contains ggml and accelerator + # backends. Empty prefix matches are not comparison targets. + scans = [scan for scan in scans if scan.symbols] + timestamp = generation_timestamp() + + validation_failed = False + + if args.command == "scan": + if args.format == "text": + output = _scan_text(scans, errors) + elif args.format == "json": + output = json.dumps( + { + "generated_at": timestamp, + "libraries": [asdict(scan) for scan in scans], + "errors": errors, + }, + ensure_ascii=False, + indent=2, + ) + else: + output = "\n".join( + row for scan in scans for row in _jsonl_rows(scan, timestamp) + ) + + if args.format == "jsonl" and args.output is None: + try: + written = write_library_jsonl( + scans, + args.output_dir, + timestamp=timestamp, + ) + except OSError as exc: + print(f"ERROR: failed to write JSONL output: {exc}", file=sys.stderr) + return 1 + for path in written: + print(f"saved: {path}") + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors else 0 + elif args.command == "compare": + if len(scans) < 2: + print("Comparison requires at least two libraries.", file=sys.stderr) + return 2 + comparison = compare_scans(scans) + output = ( + _compare_text(comparison) + if args.format == "text" + else json.dumps(comparison, ensure_ascii=False, indent=2) + ) + elif args.command == "manifest": + output = json.dumps( + build_manifest(scans, generated_at=timestamp), + ensure_ascii=False, + indent=2, + ) + else: + try: + declarations = extract_ctypes_bindings(args.source) + except ScanError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + if args.scope == "optional": + declarations = [ + declaration for declaration in declarations if not declaration.required + ] + elif args.scope == "required": + declarations = [ + declaration for declaration in declarations if declaration.required + ] + if not declarations: + print( + f"No {args.scope} ctypes binding declarations found in " + f"{args.source}.", + file=sys.stderr, + ) + return 2 + binding_results = [check_bindings(scan, declarations) for scan in scans] + if not args.select_symbol and binding_results: + # A package directory may contain arbitrarily named dependency and + # backend libraries. The library ctypes would want is the one with + # the greatest declaration coverage, regardless of filename. + best_count = max(result["available_count"] for result in binding_results) + binding_results = [ + result + for result in binding_results + if result["available_count"] == best_count + ] + output = ( + _bindings_text(binding_results, args.scope) + if args.format == "text" + else json.dumps(binding_results, ensure_ascii=False, indent=2) + ) + validation_failed = any( + result["missing_required"] or result["missing_optional"] + for result in binding_results + ) + + try: + _write_output(output, args.output, timestamp=timestamp) + except OSError as exc: + print(f"ERROR: failed to write output: {exc}", file=sys.stderr) + return 1 + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors or validation_failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/abi/tests/test_platform_artifacts.py b/tools/abi/tests/test_platform_artifacts.py new file mode 100644 index 0000000000..b5f88dcc3f --- /dev/null +++ b/tools/abi/tests/test_platform_artifacts.py @@ -0,0 +1,85 @@ +"""Opt-in integration tests for real Windows, Linux, and macOS artifacts.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from tools.abi.scan_dynamic import ( + check_bindings, + collect_library_paths, + extract_ctypes_bindings, + scan_library, + select_scans_by_symbols, +) + +ARTIFACTS_ENV = "LLAMA_ABI_ARTIFACTS" +DEFAULT_ARTIFACTS = Path("tools/abi/artifacts") +REQUIRED_PLATFORMS = {"windows", "linux", "darwin"} +STABLE_LLAMA_SYMBOLS = { + "llama_decode", + "llama_model_load_from_file", +} +BINDING_SOURCE = Path("llama_cpp/llama_cpp.py") + + +@pytest.fixture(scope="module") +def platform_scans(): + configured = os.environ.get(ARTIFACTS_ENV) + artifacts = Path(configured) if configured else DEFAULT_ARTIFACTS + paths = collect_library_paths([str(artifacts)], recursive=True) + + if not paths and not configured: + pytest.skip( + "No ABI artifacts installed. Set LLAMA_ABI_ARTIFACTS to run " + "the Windows/Linux/macOS integration test." + ) + + assert paths, f"No shared libraries found under {artifacts}" + scans = [scan for path in paths for scan in scan_library(path)] + scans = select_scans_by_symbols(scans, ["llama_decode"]) + by_platform = {scan.platform: scan for scan in scans} + assert REQUIRED_PLATFORMS <= set(by_platform), ( + "The ABI artifact set must contain llama libraries for Windows, " + f"Linux, and macOS. Found: {sorted(by_platform)}" + ) + return by_platform + + +def test_windows_linux_and_macos_llama_exports(platform_scans): + common = set.intersection( + *( + {symbol.canonical_name for symbol in platform_scans[platform].symbols} + for platform in sorted(REQUIRED_PLATFORMS) + ) + ) + assert STABLE_LLAMA_SYMBOLS <= common + + +def test_macos_macho_lookup_name_removes_symbol_table_prefix(platform_scans): + decode = next( + symbol + for symbol in platform_scans["darwin"].symbols + if symbol.canonical_name == "llama_decode" + ) + assert decode.raw_name == "_llama_decode" + assert decode.lookup_name == "llama_decode" + + +def test_optional_llama_ext_abi_aliases_on_all_platforms(platform_scans): + optional = [ + declaration + for declaration in extract_ctypes_bindings(BINDING_SOURCE) + if not declaration.required + ] + assert optional, "No optional llama_ext ctypes bindings were found" + + for platform in sorted(REQUIRED_PLATFORMS): + result = check_bindings(platform_scans[platform], optional) + assert ( + result["missing_optional"] == [] + ), f"{platform} is missing optional llama_ext ABI aliases: " + ", ".join( + item["python_name"] for item in result["missing_optional"] + ) diff --git a/tools/abi/tests/test_scan_dynamic.py b/tools/abi/tests/test_scan_dynamic.py new file mode 100644 index 0000000000..ebcf75fd47 --- /dev/null +++ b/tools/abi/tests/test_scan_dynamic.py @@ -0,0 +1,194 @@ +import json + +import tools.abi.scan_dynamic as abi_tool + +from tools.abi.scan_dynamic import ( + BindingDeclaration, + SymbolRecord, + LibraryScan, + canonicalize_symbol_name, + check_bindings, + collect_library_paths, + compare_scans, + detect_abi, + extract_ctypes_bindings, + normalize_symbol_name, + select_scans_by_symbols, + write_library_jsonl, +) + + +def _scan(library: str, platform: str, names: list[str]) -> LibraryScan: + records = tuple( + SymbolRecord( + raw_name=name, + lookup_name=name, + canonical_name=name, + abi="unmangled", + address="0x0", + ) + for name in names + ) + return LibraryScan( + library=library, + format="test", + platform=platform, + architecture="test", + sha256="test", + symbols=records, + ) + + +def test_normalizes_macho_external_prefix(): + assert normalize_symbol_name("_llama_decode", "Mach-O") == "llama_decode" + assert normalize_symbol_name("__ZN5llama", "Mach-O") == "_ZN5llama" + assert normalize_symbol_name("llama_decode", "ELF") == "llama_decode" + assert normalize_symbol_name("llama_decode", "PE") == "llama_decode" + + +def test_detects_abi_after_platform_normalization(): + assert detect_abi("?function@@YAXXZ") == "msvc-cxxabi" + assert detect_abi("_ZN5llama") == "itanium-cxxabi" + assert detect_abi("llama_decode") == "unmangled" + + +def test_canonicalizes_simple_global_cpp_names(): + assert ( + canonicalize_symbol_name( + "?llama_graph_reserve@@YAXXZ", + "msvc-cxxabi", + ) + == "llama_graph_reserve" + ) + assert ( + canonicalize_symbol_name( + "_Z19llama_graph_reserveP13llama_contextjjj", + "itanium-cxxabi", + ) + == "llama_graph_reserve" + ) + nested = "_ZN5llama6detail3fooEv" + assert canonicalize_symbol_name(nested, "itanium-cxxabi") == nested + + +def test_compares_canonical_names(): + comparison = compare_scans( + [ + _scan("libllama.so", "linux", ["llama_decode"]), + _scan( + "llama.dll", + "windows", + ["llama_decode", "llama_windows_only"], + ), + ] + ) + + assert comparison["common"] == ["llama_decode"] + assert comparison["libraries"][0]["missing_here"] == ["llama_windows_only"] + assert comparison["libraries"][1]["only_here"] == ["llama_windows_only"] + + +def test_collects_versioned_elf_library(tmp_path): + library = tmp_path / "libllama.so.1" + library.touch() + + assert collect_library_paths([str(tmp_path)]) == [library] + + +def test_extracts_and_checks_literal_binding_aliases(tmp_path): + source = tmp_path / "bindings.py" + source.write_text( + """ +@ctypes_function( + ["llama_ext", "?llama_ext@@YAXXZ", "_Z9llama_extv"], + [], + None, + required=False, +) +def llama_ext(): + pass +""", + encoding="utf-8", + ) + declarations = extract_ctypes_bindings(source) + scan = _scan("libllama.so", "linux", ["_Z9llama_extv"]) + result = check_bindings(scan, declarations) + + assert declarations == [ + BindingDeclaration( + python_name="llama_ext", + candidates=( + "llama_ext", + "?llama_ext@@YAXXZ", + "_Z9llama_extv", + ), + required=False, + line=8, + ) + ] + assert result["available"][0]["selected"] == "_Z9llama_extv" + assert result["missing_optional"] == [] + + +def test_selects_library_by_symbol_not_filename(): + scans = [ + _scan("custom-backend-name.dll", "windows", ["ggml_backend_init"]), + _scan("renamed-native-output.bin", "windows", ["llama_decode"]), + ] + + selected = select_scans_by_symbols(scans, ["llama_decode"]) + + assert [scan.library for scan in selected] == ["renamed-native-output.bin"] + + +def test_writes_jsonl_named_after_dynamic_library(tmp_path): + output_dir = tmp_path / "output" + scans = [ + _scan("libllama.so", "linux", ["llama_decode"]), + _scan("llama.dll", "windows", ["llama_decode"]), + ] + + timestamp = "20260728T120000.123456Z" + written = write_library_jsonl( + scans, + output_dir, + timestamp=timestamp, + ) + + assert [path.name for path in written] == [ + "libllama.so.jsonl", + "llama.dll.jsonl", + ] + run_dir = output_dir / timestamp + row = json.loads((run_dir / "llama.dll.jsonl").read_text("utf-8")) + assert row["library"] == "llama.dll" + assert row["canonical_name"] == "llama_decode" + assert row["generated_at"] == timestamp + assert "path" not in row + + +def test_check_bindings_cli_fails_when_optional_api_is_missing( + tmp_path, + monkeypatch, +): + library = tmp_path / "renamed.dll" + library.touch() + source = tmp_path / "bindings.py" + source.write_text( + """ +@ctypes_function(["llama_ext", "_Z9llama_extv"], [], None, required=False) +def llama_ext(): + pass +""", + encoding="utf-8", + ) + scan = _scan("renamed.dll", "windows", ["llama_decode"]) + monkeypatch.setattr( + abi_tool, + "_scan_paths", + lambda paths: ([scan], []), + ) + + exit_code = abi_tool.main(["check-bindings", str(library), "--source", str(source)]) + + assert exit_code == 1 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8bb909374d..1c3c9674de 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8bb909374d04d40621340aee5ba2245860027fdc +Subproject commit 1c3c9674de4d455f1e571bed808252af54932767