diff --git a/AI/talk_handler.py b/AI/talk_handler.py index a6c8e11..4f94380 100644 --- a/AI/talk_handler.py +++ b/AI/talk_handler.py @@ -28,6 +28,8 @@ SUMMARY_CHAR_BUDGET = 1_800 MAX_INPUT_CHARS = 2_000 SUMMARY_LINE_CHAR_LIMIT = 220 HISTORY_LINE_CHAR_LIMIT = 450 +RETRY_MIN_MAX_TOKENS = 384 +RETRY_MAX_MAX_TOKENS = 768 REPLY_RULES = { "mention": {"cooldown": 25, "min_user_messages": 1}, @@ -311,6 +313,7 @@ async def _call_llm( max_tokens: int, temperature: float, top_p: float, + disable_thinking: bool = False, ) -> str: url = f"{LLAMA_API_URL.rstrip('/')}/v1/chat/completions" payload = { @@ -319,6 +322,17 @@ async def _call_llm( "temperature": temperature, "top_p": top_p, } + if disable_thinking: + payload.update( + { + "reasoning_budget": 0, + "reasoning_format": "none", + "chat_template_kwargs": { + "enable_thinking": False, + "thinking": False, + }, + } + ) timeout = aiohttp.ClientTimeout(total=120) async with aiohttp.ClientSession(timeout=timeout) as session: async with session.post(url, json=payload) as resp: @@ -336,10 +350,33 @@ async def _call_llm( if not choices: logger.error("LLM API returned no choices: %s", _clip_text(str(data), 300)) raise RuntimeError(f"LLM API returned no choices: {data}") - content = choices[0].get("message", {}).get("content", "") + choice = choices[0] + message = choice.get("message", {}) or {} + finish_reason = choice.get("finish_reason") + content = message.get("content", "") + reasoning_content = (message.get("reasoning_content") or "").strip() cleaned_content = content.strip() + if not cleaned_content and reasoning_content and not disable_thinking: + retry_max_tokens = min(max(max_tokens * 2, RETRY_MIN_MAX_TOKENS), RETRY_MAX_MAX_TOKENS) + logger.warning( + "LLM returned reasoning_content without final content. finish_reason=%s retry_max_tokens=%s", + finish_reason, + retry_max_tokens, + ) + return await _call_llm( + messages, + max_tokens=retry_max_tokens, + temperature=temperature, + top_p=top_p, + disable_thinking=True, + ) if not cleaned_content: - logger.warning("LLM returned empty content. Raw response: %s", _clip_text(raw_text, 300)) + logger.warning( + "LLM returned empty content. finish_reason=%s disable_thinking=%s raw=%s", + finish_reason, + disable_thinking, + _clip_text(raw_text, 300), + ) return cleaned_content