From cd9f058ef6b904012921f213e5c7a4f1b73f2ef5 Mon Sep 17 00:00:00 2001 From: Yingbei Date: Tue, 9 Jan 2024 15:21:49 -0800 Subject: [PATCH 1/3] allow default grammar --- llama_cpp/server/app.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index c54e4eb5ce..0380adfda4 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -49,6 +49,9 @@ _server_settings: Optional[ServerSettings] = None +DEFAULT_GRAMMAR_FILE = os.getenv("DEFAULT_GRAMMAR_FILE", "./json_grammar.gbnf") +with open(DEFAULT_GRAMMAR_FILE) as f: + default_grammar_text = f.read() def set_server_settings(server_settings: ServerSettings): global _server_settings @@ -237,6 +240,7 @@ async def create_completion( if body.grammar is not None: kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) + iterator_or_completion: Union[ llama_cpp.CreateCompletionResponse, Iterator[llama_cpp.CreateCompletionStreamResponse], @@ -303,7 +307,8 @@ async def create_chat_completion( if body.grammar is not None: kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) - + else: + kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(default_grammar_text) iterator_or_completion: Union[ llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) From 0a91cc3c2b8a30d480ddf56e87555824a0dd8dfc Mon Sep 17 00:00:00 2001 From: Yingbei Date: Tue, 9 Jan 2024 20:12:28 -0800 Subject: [PATCH 2/3] minor fix --- llama_cpp/server/app.py | 10 ++++++---- vendor/llama.cpp | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 0380adfda4..b08e510463 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -49,9 +49,10 @@ _server_settings: Optional[ServerSettings] = None -DEFAULT_GRAMMAR_FILE = os.getenv("DEFAULT_GRAMMAR_FILE", "./json_grammar.gbnf") -with open(DEFAULT_GRAMMAR_FILE) as f: - default_grammar_text = f.read() +DEFAULT_GRAMMAR_FILE = os.getenv("GRAMMAR_FILE", "") +if DEFAULT_GRAMMAR_FILE != "": + with open(DEFAULT_GRAMMAR_FILE) as f: + default_grammar_text = f.read() def set_server_settings(server_settings: ServerSettings): global _server_settings @@ -308,7 +309,8 @@ async def create_chat_completion( if body.grammar is not None: kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) else: - kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(default_grammar_text) + if DEFAULT_GRAMMAR_FILE: + kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(default_grammar_text) iterator_or_completion: Union[ llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6efb8eb30e..1fc2f265ff 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6efb8eb30e7025b168f3fda3ff83b9b386428ad6 +Subproject commit 1fc2f265ff9377a37fd2c61eae9cd813a3491bea From e7177489f765cd8160077022fa96e07317cdc1a9 Mon Sep 17 00:00:00 2001 From: Yingbei Date: Thu, 11 Jan 2024 19:32:49 -0800 Subject: [PATCH 3/3] logic to reuse to address grammar --- llama_cpp/server/app.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index b08e510463..bb7f0b18e5 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -298,6 +298,11 @@ async def create_chat_completion( "user", } kwargs = body.model_dump(exclude=exclude) + if kwargs["response_format"] and kwargs["response_format"]["type"] == "json_object": + kwargs["response_format"] = None # set to None to ignore llama_cpp_python default json_object grammar + # print("empty response format") + if DEFAULT_GRAMMAR_FILE: + kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(default_grammar_text) llama = llama_proxy(body.model) if body.logit_bias is not None: kwargs["logit_bias"] = ( @@ -308,9 +313,6 @@ async def create_chat_completion( if body.grammar is not None: kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) - else: - if DEFAULT_GRAMMAR_FILE: - kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(default_grammar_text) iterator_or_completion: Union[ llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] ] = await run_in_threadpool(llama.create_chat_completion, **kwargs)