feat(v9): deterministischer Memory-Kurator (Dedupe) statt LLM-Cron
- routers/memory.py: POST /api/memory/dedupe — findet Dubletten (exakt/ enthalten/ähnlich via SequenceMatcher) je Kategorie, behält den vollständigsten Eintrag. Dry-Run (apply:false) + Anwenden. Kein LLM. - MemoryPanel: "🧹 Aufräumen"-Button (Vorschau → Bestätigung → löschen). Grund: LLM-Cron-Kurator hat Edits nur halluziniert/als Text ausgegeben statt auszuführen (Hermes 4 + Qwen). Deterministisch = verlässlich. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -103,6 +103,26 @@
|
|||||||
editId = m.id; editContent = m.content; editCategory = m.category
|
editId = m.id; editContent = m.content; editCategory = m.category
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Deterministischer Kurator: Dubletten (exakt/enthalten/ähnlich) je Kategorie zusammenführen
|
||||||
|
let deduping = $state(false)
|
||||||
|
async function dedupe() {
|
||||||
|
deduping = true
|
||||||
|
try {
|
||||||
|
const preview = await api('/api/memory/dedupe', { method: 'POST', body: JSON.stringify({ apply: false }) })
|
||||||
|
if (preview.duplicate_count === 0) { toast('Keine Dubletten gefunden 🎉'); return }
|
||||||
|
const ok = await confirmModal({
|
||||||
|
title: `${preview.duplicate_count} Dubletten entfernen?`,
|
||||||
|
body: `${preview.duplicate_count} doppelte Einträge werden gelöscht; die ${preview.groups.length} vollständigeren bleiben erhalten. Nur exakte/enthaltene/sehr ähnliche Einträge derselben Kategorie.`,
|
||||||
|
danger: true,
|
||||||
|
})
|
||||||
|
if (!ok) return
|
||||||
|
const res = await api('/api/memory/dedupe', { method: 'POST', body: JSON.stringify({ apply: true }) })
|
||||||
|
toast(`${res.removed} Dubletten entfernt`)
|
||||||
|
await load()
|
||||||
|
} catch (e: any) { toast('Fehler: ' + e.message, true) }
|
||||||
|
finally { deduping = false }
|
||||||
|
}
|
||||||
|
|
||||||
async function runImport() {
|
async function runImport() {
|
||||||
if (importLines.length === 0) return
|
if (importLines.length === 0) return
|
||||||
importBusy = true
|
importBusy = true
|
||||||
@@ -132,6 +152,9 @@
|
|||||||
<div class="sub">Fakten, Entscheidungen und Kontext — geteilt von allen KI-Tools via MCP.</div>
|
<div class="sub">Fakten, Entscheidungen und Kontext — geteilt von allen KI-Tools via MCP.</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="flex gap-2" style="flex-shrink:0;margin-top:4px">
|
<div class="flex gap-2" style="flex-shrink:0;margin-top:4px">
|
||||||
|
<button class="ghost" onclick={dedupe} disabled={deduping} title="Doppelte Einträge zusammenführen">
|
||||||
|
{deduping ? '…' : '🧹 Aufräumen'}
|
||||||
|
</button>
|
||||||
<button class="ghost" onclick={() => { importing = !importing; adding = false; importText = '' }}>
|
<button class="ghost" onclick={() => { importing = !importing; adding = false; importText = '' }}>
|
||||||
{importing ? 'Abbrechen' : '⇩ Import'}
|
{importing ? 'Abbrechen' : '⇩ Import'}
|
||||||
</button>
|
</button>
|
||||||
|
|||||||
@@ -6,9 +6,11 @@ SQLite aus stdlib — kein Vektor-Overhead, keine neue Abhaengigkeit.
|
|||||||
Alle MCP-Tools teilen denselben Speicher via mcp_memory.py-Wrapper.
|
Alle MCP-Tools teilen denselben Speicher via mcp_memory.py-Wrapper.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
from difflib import SequenceMatcher
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
@@ -98,6 +100,70 @@ def export_memories():
|
|||||||
return {"text": "\n".join(lines), "count": len(rows)}
|
return {"text": "\n".join(lines), "count": len(rows)}
|
||||||
|
|
||||||
|
|
||||||
|
def _norm(s: str) -> str:
|
||||||
|
"""Normalisiert Text fuer Dubletten-Vergleich: klein, ohne Satzzeichen, Whitespace kollabiert."""
|
||||||
|
s = re.sub(r"[^\w\s]", " ", s.lower(), flags=re.UNICODE)
|
||||||
|
return re.sub(r"\s+", " ", s).strip()
|
||||||
|
|
||||||
|
|
||||||
|
class _DedupeIn(BaseModel):
|
||||||
|
apply: bool = False # False = Dry-Run (nur Vorschau), True = wirklich loeschen
|
||||||
|
threshold: float = 0.85 # Aehnlichkeits-Schwelle (SequenceMatcher)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/memory/dedupe")
|
||||||
|
def dedupe_memories(body: _DedupeIn):
|
||||||
|
"""Deterministischer Kurator: findet Dubletten (exakt / enthalten / aehnlich) je Kategorie,
|
||||||
|
behaelt den vollstaendigsten (laengsten) Eintrag, entfernt die uebrigen. KEIN LLM.
|
||||||
|
Konservativ: nur innerhalb derselben Kategorie, Containment braucht vollstaendige Teilstring-Deckung."""
|
||||||
|
db = _db()
|
||||||
|
# laengste zuerst -> der vollstaendigste Eintrag einer Gruppe wird zum Kanon
|
||||||
|
rows = [dict(r) for r in db.execute(
|
||||||
|
"SELECT * FROM memories ORDER BY length(content) DESC, created_at ASC"
|
||||||
|
).fetchall()]
|
||||||
|
used: set[str] = set()
|
||||||
|
groups: list[dict] = []
|
||||||
|
for i, a in enumerate(rows):
|
||||||
|
if a["id"] in used:
|
||||||
|
continue
|
||||||
|
na = _norm(a["content"])
|
||||||
|
if not na:
|
||||||
|
continue
|
||||||
|
dups = []
|
||||||
|
for b in rows[i + 1:]:
|
||||||
|
if b["id"] in used or b["category"] != a["category"]:
|
||||||
|
continue
|
||||||
|
nb = _norm(b["content"])
|
||||||
|
if not nb:
|
||||||
|
continue
|
||||||
|
contained = nb in na or na in nb
|
||||||
|
ratio = SequenceMatcher(None, na, nb).ratio()
|
||||||
|
if contained or ratio >= body.threshold:
|
||||||
|
dups.append(b)
|
||||||
|
used.add(b["id"])
|
||||||
|
if dups:
|
||||||
|
used.add(a["id"])
|
||||||
|
groups.append({
|
||||||
|
"keep": {"id": a["id"], "content": a["content"], "category": a["category"]},
|
||||||
|
"remove": [{"id": d["id"], "content": d["content"]} for d in dups],
|
||||||
|
})
|
||||||
|
dup_count = sum(len(g["remove"]) for g in groups)
|
||||||
|
removed = 0
|
||||||
|
if body.apply:
|
||||||
|
for g in groups:
|
||||||
|
for d in g["remove"]:
|
||||||
|
db.execute("DELETE FROM memories WHERE id = ?", (d["id"],))
|
||||||
|
removed += 1
|
||||||
|
if removed:
|
||||||
|
db.commit()
|
||||||
|
return {
|
||||||
|
"groups": groups,
|
||||||
|
"duplicate_count": dup_count,
|
||||||
|
"removed": removed,
|
||||||
|
"applied": body.apply,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/memory")
|
@router.get("/memory")
|
||||||
def list_memories(q: str = "", category: str = ""):
|
def list_memories(q: str = "", category: str = ""):
|
||||||
db = _db()
|
db = _db()
|
||||||
|
|||||||
Vendored
+34
-34
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user