From 0354ce999f09adc564787357369703d52de3d4a4 Mon Sep 17 00:00:00 2001 From: Hitonabi Date: Wed, 15 Jul 2026 09:46:36 +0200 Subject: [PATCH] Bild-Weiche v2: Coder-Ziele bekommen Vision-BESCHREIBUNG statt Umleitung Idee aus verwaistem (nie verdrahtetem) Patch in der Hermes-Quelle api_server.py - regelkonform in den MC2-Gateway umgezogen, Original als docs/archiv/hermes-api_server-vision-patch-verwaist.diff archiviert. Request mit Bild an coder: Bilder werden vorab von VL-30B beschrieben und als Text injiziert, die Code-Frage bleibt beim Spezialisten. Fallback bei Analyse-Fehler: bisherige Vision-Umleitung. Co-Authored-By: Claude Fable 5 --- backend/routers/gateway_proxy.py | 77 +++++++- backend/services/router_logic.py | 4 +- ...rmes-api_server-vision-patch-verwaist.diff | 164 ++++++++++++++++++ 3 files changed, 235 insertions(+), 10 deletions(-) create mode 100644 docs/archiv/hermes-api_server-vision-patch-verwaist.diff diff --git a/backend/routers/gateway_proxy.py b/backend/routers/gateway_proxy.py index 0e25ccb..f941362 100644 --- a/backend/routers/gateway_proxy.py +++ b/backend/routers/gateway_proxy.py @@ -5,7 +5,7 @@ from fastapi.responses import JSONResponse, StreamingResponse from config import LLAMA_SWAP_URL from services.gateway_stream import record_stream_chunk, record_usage -from services.router_logic import VISION_CAPABLE, choose_for_lane, has_image +from services.router_logic import IMAGE_PART_TYPES, VISION_CAPABLE, choose_for_lane, has_image from services.routing_policy import load_policy router = APIRouter(prefix="/v1") @@ -19,6 +19,61 @@ _LANG_DIRECTIVE = os.environ.get( "Quellcode, Bezeichner und Shell-Befehle bleiben unverändert.") +# Bild-Beschreibung für Coder-Ziele: Prompt bewusst auf wörtliche Wiedergabe von +# Code/Fehlermeldungen getrimmt — der Coder arbeitet nur mit diesem Text weiter. +_BILD_BESCHREIB_PROMPT = os.environ.get( + "MC_CODER_IMAGE_PROMPT", + "Beschreibe dieses Bild vollstaendig und praezise auf Deutsch: sichtbarer Text, " + "Code, Zahlen/Daten, UI-Elemente, Layout, Farben und alles Auffaellige. " + "Sichtbaren Code und Fehlermeldungen gib WOERTLICH wieder.") + + +def _ist_coder_alias(alias: str) -> bool: + """Coder-Ziele (coder, rohe Qwen-Coder-IDs) bekommen Bild-BESCHREIBUNGEN statt der + stumpfen Vision-Umleitung — die Code-Frage bleibt beim Spezialisten.""" + return "coder" in (alias or "").lower() + + +async def _bilder_fuer_coder_beschreiben(body: dict, client, vision_alias: str) -> bool: + """Ersetzt jeden Bild-Part durch eine Text-Beschreibung vom Vision-Modell. + + Idee aus einem verwaisten, nie verdrahteten Patch in der Hermes-Quelle + (gateway/platforms/api_server.py, 07/2026) — bei der Projekt-Review 15.07. + regelkonform hierher umgezogen (Archiv: docs/archiv/). True = alle Bilder + ersetzt; False = eine Analyse scheiterte, Aufrufer nutzt die normale + Vision-Umleitung als Fallback. + """ + fundstellen: list[tuple[dict, int, dict]] = [] + for m in body.get("messages") or []: + if not isinstance(m, dict) or not isinstance(m.get("content"), list): + continue + for i, part in enumerate(m["content"]): + if isinstance(part, dict) and part.get("type") in IMAGE_PART_TYPES: + fundstellen.append((m, i, part)) + for nr, (msg, idx, part) in enumerate(fundstellen, start=1): + frage = { + "model": vision_alias, + "stream": False, + "max_tokens": 700, + "messages": [{"role": "user", + "content": [part, {"type": "text", "text": _BILD_BESCHREIB_PROMPT}]}], + } + try: + # Grosszuegiger Timeout: VL-30B ist on-demand (Kaltladen ~30 s) + Beschreibung ~25 s. + r = await client.post(f"{LLAMA_SWAP_URL}/v1/chat/completions", json=frage, timeout=240.0) + text = "" + if r.status_code == 200: + text = ((r.json().get("choices") or [{}])[0].get("message") or {}).get("content") or "" + if not text.strip(): + return False + except Exception: + return False + msg["content"][idx] = {"type": "text", "text": ( + f"[Bild {nr}: dem Request lag ein Bild bei — {vision_alias} beschreibt es so:\n" + f"{text.strip()}]")} + return True + + def _inject_language(body: dict, alias: str) -> None: """Deutsch-Direktive anhängen. An die ERSTE System-Message (viele Chat-Templates erwarten nur eine), sonst als neue System-Message. `hermes` ausgenommen — Lucys @@ -101,23 +156,29 @@ async def _proxy(path: str, request: Request): routed = {"x-mc-routed-to": requested} pol = load_policy() + client = request.app.state.gw_client # geteilter Keep-Alive-Client (siehe app.py lifespan) # Bild-Weiche (Faden 11): Requests mit Bild-Anhang automatisch ans Vision-Modell umleiten, # sofern das Ziel nicht ohnehin bildfähig ist. Das MTP-Hirn (fast/hermes) kann keine Bilder — # so bleibt Lucy schnell, ohne dass jemand manuell das Modell wechselt (Entscheid Weg A). + # Coder-Sonderweg (15.07.): Coder-Ziele behalten den Request — die Bilder werden vorab vom + # Vision-Modell BESCHRIEBEN und als Text injiziert (Screenshot-Debugging bleibt beim Coder). vision_alias = pol.get("vision") if vision_alias and alias not in VISION_CAPABLE and has_image(body): - alias = vision_alias - body["model"] = alias - # HTTP-Header-Werte muessen latin-1 sein — kein '→' o.ae. (sonst 500, 13.07.). - routed = {"x-mc-routed-to": alias, "x-mc-route-reason": "Bild erkannt -> Vision-Modell", - "x-mc-lane": routed.get("x-mc-lane", "-")} + if _ist_coder_alias(alias) and await _bilder_fuer_coder_beschreiben(body, client, vision_alias): + routed = {"x-mc-routed-to": alias, + "x-mc-route-reason": "Bild beschrieben (Vision) -> bleibt beim Coder", + "x-mc-lane": routed.get("x-mc-lane", "-")} + else: + alias = vision_alias + body["model"] = alias + # HTTP-Header-Werte muessen latin-1 sein — kein '→' o.ae. (sonst 500, 13.07.). + routed = {"x-mc-routed-to": alias, "x-mc-route-reason": "Bild erkannt -> Vision-Modell", + "x-mc-lane": routed.get("x-mc-lane", "-")} # fast-Spur: Thinking aus für flotte Antworten (sofern Client es nicht selbst setzt). if pol["fast_no_think"] and alias == pol["fast"] and "chat_template_kwargs" not in body: body["chat_template_kwargs"] = {"enable_thinking": False} _inject_language(body, alias) url = f"{LLAMA_SWAP_URL}{path}" - - client = request.app.state.gw_client # geteilter Keep-Alive-Client (siehe app.py lifespan) if body.get("stream"): req = client.build_request("POST", url, json=body, timeout=None) r = await client.send(req, stream=True) diff --git a/backend/services/router_logic.py b/backend/services/router_logic.py index b07c522..a4cabaf 100644 --- a/backend/services/router_logic.py +++ b/backend/services/router_logic.py @@ -52,7 +52,7 @@ _CODE_HINT = re.compile( # Bild-Weiche (Faden 11): Aliase, die selbst Bilder koennen — die werden NIE umgeroutet. VISION_CAPABLE = {"vision", "scout"} -_IMAGE_PART_TYPES = {"image_url", "input_image", "image"} +IMAGE_PART_TYPES = {"image_url", "input_image", "image"} def has_image(body: dict) -> bool: @@ -64,7 +64,7 @@ def has_image(body: dict) -> bool: content = m.get("content") if isinstance(content, list): for part in content: - if isinstance(part, dict) and part.get("type") in _IMAGE_PART_TYPES: + if isinstance(part, dict) and part.get("type") in IMAGE_PART_TYPES: return True return False diff --git a/docs/archiv/hermes-api_server-vision-patch-verwaist.diff b/docs/archiv/hermes-api_server-vision-patch-verwaist.diff new file mode 100644 index 0000000..ccabf2d --- /dev/null +++ b/docs/archiv/hermes-api_server-vision-patch-verwaist.diff @@ -0,0 +1,164 @@ +diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py +index 628713224..415f3b2dc 100644 +--- a/gateway/platforms/api_server.py ++++ b/gateway/platforms/api_server.py +@@ -362,6 +362,159 @@ def _multimodal_validation_error(exc: ValueError, *, param: str) -> "web.Respons + ) + + ++def _is_coder_model(model_name: Optional[str]) -> bool: ++ """Return True if model_name looks like a coder model (case-insensitive 'coder' in name).""" ++ if not model_name: ++ return False ++ return "coder" in str(model_name).lower() ++ ++ ++def _extract_image_urls_from_messages(messages: List[Dict[str, Any]]) -> List[str]: ++ """Extract all image URLs from OpenAI-style messages payload.""" ++ image_urls: List[str] = [] ++ for msg in messages: ++ if not isinstance(msg, dict): ++ continue ++ content = msg.get("content") ++ if not content: ++ continue ++ # Normalize multimodal content to list of parts ++ parts: Any = content ++ if isinstance(content, str): ++ continue # text-only message has no images ++ if not isinstance(parts, list): ++ continue ++ for part in parts: ++ if not isinstance(part, dict): ++ continue ++ part_type = str(part.get("type", "")).strip().lower() ++ if part_type in {"image_url", "input_image"}: ++ img_ref = part.get("image_url") ++ if isinstance(img_ref, dict): ++ url = img_ref.get("url") ++ else: ++ url = img_ref ++ if isinstance(url, str) and url.strip(): ++ image_urls.append(url.strip()) ++ return image_urls ++ ++ ++async def _maybe_enrich_with_vision_if_coder( ++ body: Dict[str, Any], ++) -> Dict[str, Any]: ++ """ ++ For coder model requests with image attachments, pre-analyze images ++ with the vision fallback model and replace image parts with descriptive text. ++ ++ Returns a modified copy of body with image URLs replaced by text ++ descriptions in the messages array. ++ """ ++ model_name = body.get("model") ++ if not _is_coder_model(model_name): ++ return body ++ ++ messages = body.get("messages") ++ if not messages or not isinstance(messages, list): ++ return body ++ ++ image_urls = _extract_image_urls_from_messages(messages) ++ if not image_urls: ++ return body ++ ++ # Import here to avoid circular dependency issues ++ try: ++ from tools.vision_tools import vision_analyze_tool ++ except ImportError: ++ logger.warning( ++ "Vision tool not available for coder model enrichment. " ++ "Install hermes-agent tools to enable this feature." ++ ) ++ return body ++ ++ # Use the configured vision model or fall back to VL-30B ++ # We'll let vision_analyze_tool use its own default (auxiliary.vision) ++ # but we can optionally override by passing model parameter ++ # For now, let it auto-resolve via config ++ analysis_prompt = ( ++ "Describe everything visible in this image in thorough detail. " ++ "Include any text, code, data, objects, people, layout, colors, " ++ "and any other notable visual information." ++ ) ++ ++ # Build a mapping of URL -> description ++ url_to_desc: Dict[str, str] = {} ++ for img_url in image_urls: ++ try: ++ result_json = await vision_analyze_tool( ++ image_url=img_url, ++ user_prompt=analysis_prompt, ++ ) ++ result = json.loads(result_json) ++ if result.get("success"): ++ desc = result.get("analysis", "") ++ url_to_desc[img_url] = ( ++ f"[The user sent an image~ Here's what I can see:\n{desc}]\n" ++ f"[If you need a closer look, use vision_analyze with image_url: {img_url} ~]" ++ ) ++ else: ++ url_to_desc[img_url] = ( ++ "[The user sent an image but I couldn't quite see it this time (>_<) " ++ f"You can try looking at it yourself with vision_analyze using image_url: {img_url}]" ++ ) ++ except Exception as e: ++ logger.error("Vision auto-analysis error for image %s: %s", img_url, e) ++ url_to_desc[img_url] = ( ++ f"[The user sent an image but something went wrong when I tried to look at it~ " ++ f"You can try examining it yourself with vision_analyze using image_url: {img_url}]" ++ ) ++ ++ # Now replace image parts with text in a copy of messages ++ enriched_messages: List[Dict[str, Any]] = [] ++ for msg in messages: ++ msg_copy = dict(msg) ++ content = msg_copy.get("content") ++ if isinstance(content, str): ++ enriched_messages.append(msg_copy) ++ continue ++ ++ if not isinstance(content, list): ++ enriched_messages.append(msg_copy) ++ continue ++ ++ enriched_parts: List[Any] = [] ++ for part in content: ++ if not isinstance(part, dict): ++ enriched_parts.append(part) ++ continue ++ ++ part_type = str(part.get("type", "")).strip().lower() ++ if part_type in {"image_url", "input_image"}: ++ img_ref = part.get("image_url") ++ if isinstance(img_ref, dict): ++ img_url = img_ref.get("url") ++ else: ++ img_url = img_ref ++ if isinstance(img_url, str) and img_url in url_to_desc: ++ # Replace image part with text part ++ enriched_parts.append({ ++ "type": "text", ++ "text": url_to_desc[img_url], ++ }) ++ else: ++ # Keep original if URL not found (shouldn't happen) ++ enriched_parts.append(part) ++ else: ++ enriched_parts.append(part) ++ ++ msg_copy["content"] = enriched_parts ++ enriched_messages.append(msg_copy) ++ ++ # Return modified body ++ body_copy = dict(body) ++ body_copy["messages"] = enriched_messages ++ return body_copy ++ ++ + def _session_chat_user_message(body: Dict[str, Any], *, param: str = "message") -> tuple[Any, Optional["web.Response"]]: + """Parse and normalize session chat ``message`` / ``input`` like chat completions.""" + user_message = body.get("message") or body.get("input")