feat(v9): deterministischer Memory-Kurator (Dedupe) statt LLM-Cron
- routers/memory.py: POST /api/memory/dedupe — findet Dubletten (exakt/ enthalten/ähnlich via SequenceMatcher) je Kategorie, behält den vollständigsten Eintrag. Dry-Run (apply:false) + Anwenden. Kein LLM. - MemoryPanel: "🧹 Aufräumen"-Button (Vorschau → Bestätigung → löschen). Grund: LLM-Cron-Kurator hat Edits nur halluziniert/als Text ausgegeben statt auszuführen (Hermes 4 + Qwen). Deterministisch = verlässlich. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -103,6 +103,26 @@
|
||||
editId = m.id; editContent = m.content; editCategory = m.category
|
||||
}
|
||||
|
||||
// Deterministischer Kurator: Dubletten (exakt/enthalten/ähnlich) je Kategorie zusammenführen
|
||||
let deduping = $state(false)
|
||||
async function dedupe() {
|
||||
deduping = true
|
||||
try {
|
||||
const preview = await api('/api/memory/dedupe', { method: 'POST', body: JSON.stringify({ apply: false }) })
|
||||
if (preview.duplicate_count === 0) { toast('Keine Dubletten gefunden 🎉'); return }
|
||||
const ok = await confirmModal({
|
||||
title: `${preview.duplicate_count} Dubletten entfernen?`,
|
||||
body: `${preview.duplicate_count} doppelte Einträge werden gelöscht; die ${preview.groups.length} vollständigeren bleiben erhalten. Nur exakte/enthaltene/sehr ähnliche Einträge derselben Kategorie.`,
|
||||
danger: true,
|
||||
})
|
||||
if (!ok) return
|
||||
const res = await api('/api/memory/dedupe', { method: 'POST', body: JSON.stringify({ apply: true }) })
|
||||
toast(`${res.removed} Dubletten entfernt`)
|
||||
await load()
|
||||
} catch (e: any) { toast('Fehler: ' + e.message, true) }
|
||||
finally { deduping = false }
|
||||
}
|
||||
|
||||
async function runImport() {
|
||||
if (importLines.length === 0) return
|
||||
importBusy = true
|
||||
@@ -132,6 +152,9 @@
|
||||
<div class="sub">Fakten, Entscheidungen und Kontext — geteilt von allen KI-Tools via MCP.</div>
|
||||
</div>
|
||||
<div class="flex gap-2" style="flex-shrink:0;margin-top:4px">
|
||||
<button class="ghost" onclick={dedupe} disabled={deduping} title="Doppelte Einträge zusammenführen">
|
||||
{deduping ? '…' : '🧹 Aufräumen'}
|
||||
</button>
|
||||
<button class="ghost" onclick={() => { importing = !importing; adding = false; importText = '' }}>
|
||||
{importing ? 'Abbrechen' : '⇩ Import'}
|
||||
</button>
|
||||
|
||||
@@ -6,9 +6,11 @@ SQLite aus stdlib — kein Vektor-Overhead, keine neue Abhaengigkeit.
|
||||
Alle MCP-Tools teilen denselben Speicher via mcp_memory.py-Wrapper.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from difflib import SequenceMatcher
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
@@ -98,6 +100,70 @@ def export_memories():
|
||||
return {"text": "\n".join(lines), "count": len(rows)}
|
||||
|
||||
|
||||
def _norm(s: str) -> str:
|
||||
"""Normalisiert Text fuer Dubletten-Vergleich: klein, ohne Satzzeichen, Whitespace kollabiert."""
|
||||
s = re.sub(r"[^\w\s]", " ", s.lower(), flags=re.UNICODE)
|
||||
return re.sub(r"\s+", " ", s).strip()
|
||||
|
||||
|
||||
class _DedupeIn(BaseModel):
|
||||
apply: bool = False # False = Dry-Run (nur Vorschau), True = wirklich loeschen
|
||||
threshold: float = 0.85 # Aehnlichkeits-Schwelle (SequenceMatcher)
|
||||
|
||||
|
||||
@router.post("/memory/dedupe")
|
||||
def dedupe_memories(body: _DedupeIn):
|
||||
"""Deterministischer Kurator: findet Dubletten (exakt / enthalten / aehnlich) je Kategorie,
|
||||
behaelt den vollstaendigsten (laengsten) Eintrag, entfernt die uebrigen. KEIN LLM.
|
||||
Konservativ: nur innerhalb derselben Kategorie, Containment braucht vollstaendige Teilstring-Deckung."""
|
||||
db = _db()
|
||||
# laengste zuerst -> der vollstaendigste Eintrag einer Gruppe wird zum Kanon
|
||||
rows = [dict(r) for r in db.execute(
|
||||
"SELECT * FROM memories ORDER BY length(content) DESC, created_at ASC"
|
||||
).fetchall()]
|
||||
used: set[str] = set()
|
||||
groups: list[dict] = []
|
||||
for i, a in enumerate(rows):
|
||||
if a["id"] in used:
|
||||
continue
|
||||
na = _norm(a["content"])
|
||||
if not na:
|
||||
continue
|
||||
dups = []
|
||||
for b in rows[i + 1:]:
|
||||
if b["id"] in used or b["category"] != a["category"]:
|
||||
continue
|
||||
nb = _norm(b["content"])
|
||||
if not nb:
|
||||
continue
|
||||
contained = nb in na or na in nb
|
||||
ratio = SequenceMatcher(None, na, nb).ratio()
|
||||
if contained or ratio >= body.threshold:
|
||||
dups.append(b)
|
||||
used.add(b["id"])
|
||||
if dups:
|
||||
used.add(a["id"])
|
||||
groups.append({
|
||||
"keep": {"id": a["id"], "content": a["content"], "category": a["category"]},
|
||||
"remove": [{"id": d["id"], "content": d["content"]} for d in dups],
|
||||
})
|
||||
dup_count = sum(len(g["remove"]) for g in groups)
|
||||
removed = 0
|
||||
if body.apply:
|
||||
for g in groups:
|
||||
for d in g["remove"]:
|
||||
db.execute("DELETE FROM memories WHERE id = ?", (d["id"],))
|
||||
removed += 1
|
||||
if removed:
|
||||
db.commit()
|
||||
return {
|
||||
"groups": groups,
|
||||
"duplicate_count": dup_count,
|
||||
"removed": removed,
|
||||
"applied": body.apply,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/memory")
|
||||
def list_memories(q: str = "", category: str = ""):
|
||||
db = _db()
|
||||
|
||||
Vendored
+34
-34
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user