Initial commit: VKUS vitrina MVP (frontend + backend).

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-29 23:13:06 +07:00
commit 862e6b7b25
113 changed files with 50949 additions and 0 deletions
@@ -0,0 +1,214 @@
+"""
+Временный локальный RAG-слой внутри backend.
+
+Целевая архитектура выносит RAG на платформу ИИ (Docling + BERT + VectorDB).
+Для тестирования витрины используется упрощённый индекс TF-IDF по папке Manuals/.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+SUPPORTED_SUFFIXES = {".txt", ".md", ".docx"}
+
+TOKEN_RE = re.compile(r"[a-zA-Zа-яА-ЯёЁ0-9]{3,}")
+
+
+@dataclass
+class DocumentChunk:
+    id: str
+    text: str
+    source_title: str
+    source_path: str
+    system: str | None = None
+
+
+@dataclass
+class SearchHit:
+    chunk: DocumentChunk
+    score: float
+
+
+def tokenize(text: str) -> list[str]:
+    return [t.lower() for t in TOKEN_RE.findall(text)]
+
+
+def detect_system_from_text(text: str) -> str | None:
+    lower = text.lower()
+    rules = [
+        (r"meridium|меридиум", "Meridium 4"),
+        (r"есуид", "ЕСУИД"),
+        (r"\b1[сc]\b", "1С"),
+        (r"starlims|старлимс", "StarLims"),
+        (r"лимс|lims", "ЛИМС"),
+        (r"вкус|vkus", "ВКУС"),
+        (r"vpn|впн", "Общее"),
+    ]
+    for pattern, name in rules:
+        if re.search(pattern, lower):
+            return name
+    return None
+
+
+class LocalRagIndex:
+    def __init__(self) -> None:
+        self.chunks: list[DocumentChunk] = []
+        self._idf: dict[str, float] = {}
+        self._tf_vectors: list[dict[str, float]] = []
+
+    @property
+    def is_ready(self) -> bool:
+        return len(self.chunks) > 0
+
+    def build(self, documents_path: Path, chunk_size: int = 450) -> int:
+        self.chunks.clear()
+        self._idf.clear()
+        self._tf_vectors.clear()
+
+        if not documents_path.exists():
+            return 0
+
+        chunk_idx = 0
+        for file_path in sorted(documents_path.rglob("*")):
+            if not file_path.is_file():
+                continue
+            if file_path.suffix.lower() not in SUPPORTED_SUFFIXES:
+                continue
+
+            try:
+                content = _read_document_text(file_path)
+            except OSError:
+                continue
+
+            if not content:
+                continue
+
+            rel_path = str(file_path.relative_to(documents_path.parent))
+            title = file_path.stem
+            system = detect_system_from_text(content) or detect_system_from_text(str(file_path))
+
+            for piece in _split_into_chunks(content, chunk_size):
+                self.chunks.append(
+                    DocumentChunk(
+                        id=f"chunk-{chunk_idx}",
+                        text=piece,
+                        source_title=title,
+                        source_path=rel_path,
+                        system=system,
+                    )
+                )
+                chunk_idx += 1
+
+        self._build_tfidf()
+        return len(self.chunks)
+
+    def _build_tfidf(self) -> None:
+        doc_freq: dict[str, int] = {}
+        self._tf_vectors = []
+
+        for chunk in self.chunks:
+            tokens = tokenize(chunk.text)
+            if not tokens:
+                self._tf_vectors.append({})
+                continue
+            tf: dict[str, float] = {}
+            for token in tokens:
+                tf[token] = tf.get(token, 0.0) + 1.0
+            for token in tf:
+                tf[token] /= len(tokens)
+            self._tf_vectors.append(tf)
+            for token in set(tokens):
+                doc_freq[token] = doc_freq.get(token, 0) + 1
+
+        n_docs = max(len(self.chunks), 1)
+        self._idf = {token: math.log((1 + n_docs) / (1 + freq)) + 1 for token, freq in doc_freq.items()}
+
+    def search(self, query: str, top_k: int = 5, min_score: float = 0.0) -> list[SearchHit]:
+        if not self.chunks:
+            return []
+
+        q_tokens = tokenize(query)
+        if not q_tokens:
+            return []
+
+        q_tf: dict[str, float] = {}
+        for token in q_tokens:
+            q_tf[token] = q_tf.get(token, 0.0) + 1.0
+        for token in q_tf:
+            q_tf[token] /= len(q_tokens)
+        q_vec = {t: q_tf[t] * self._idf.get(t, 0.0) for t in q_tf}
+
+        hits: list[SearchHit] = []
+        for chunk, tf_vec in zip(self.chunks, self._tf_vectors):
+            score = _cosine(q_vec, {t: tf_vec[t] * self._idf.get(t, 0.0) for t in tf_vec})
+            if score >= min_score:
+                hits.append(SearchHit(chunk=chunk, score=score))
+
+        hits.sort(key=lambda h: h.score, reverse=True)
+        return hits[:top_k]
+
+
+def _read_document_text(file_path: Path) -> str:
+    suffix = file_path.suffix.lower()
+    if suffix in {".txt", ".md"}:
+        return file_path.read_text(encoding="utf-8", errors="ignore").strip()
+    if suffix == ".docx":
+        return _extract_docx_text(file_path)
+    return ""
+
+
+def _extract_docx_text(file_path: Path) -> str:
+    with zipfile.ZipFile(file_path) as archive:
+        xml = archive.read("word/document.xml").decode("utf-8", errors="ignore")
+
+    text = re.sub(r"</w:p>", "\n\n", xml)
+    text = re.sub(r"<[^>]+>", " ", text)
+    text = re.sub(r"[ \t]+", " ", text)
+    return re.sub(r"\n{3,}", "\n\n", text).strip()
+
+
+def _split_into_chunks(text: str, chunk_size: int) -> list[str]:
+    paragraphs = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
+    if not paragraphs:
+        return []
+
+    chunks: list[str] = []
+    current = ""
+
+    for para in paragraphs:
+        if len(para) > chunk_size:
+            if current:
+                chunks.append(current.strip())
+                current = ""
+            for i in range(0, len(para), chunk_size):
+                chunks.append(para[i : i + chunk_size].strip())
+            continue
+
+        candidate = f"{current}\n\n{para}".strip() if current else para
+        if len(candidate) <= chunk_size:
+            current = candidate
+        else:
+            if current:
+                chunks.append(current.strip())
+            current = para
+
+    if current:
+        chunks.append(current.strip())
+
+    return chunks
+
+
+def _cosine(a: dict[str, float], b: dict[str, float]) -> float:
+    if not a or not b:
+        return 0.0
+    common = set(a) & set(b)
+    dot = sum(a[t] * b[t] for t in common)
+    norm_a = math.sqrt(sum(v * v for v in a.values()))
+    norm_b = math.sqrt(sum(v * v for v in b.values()))
+    if norm_a == 0 or norm_b == 0:
+        return 0.0
+    return dot / (norm_a * norm_b)
@@ -0,0 +1,206 @@
+"""Оркестрация временного локального RAG и сборки ответа."""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from app.config import settings
+from app.data.knowledge_base import get_colleague_cases_for_system
+from app.models import ColleagueCase, ResponsePackage, Source
+from app.package_builder import (
+    build_warning,
+    build_why_suggested,
+    ensure_escalation_draft,
+    enrich_ticket_draft,
+)
+from app.rag.indexer import LocalRagIndex, SearchHit, detect_system_from_text
+
+if TYPE_CHECKING:
+    from app.llm.client import LlmClient
+
+_index: LocalRagIndex | None = None
+
+
+def get_index() -> LocalRagIndex:
+    global _index
+    if _index is None:
+        _index = LocalRagIndex()
+    return _index
+
+
+def init_local_rag() -> dict[str, object]:
+    """Построить индекс при старте backend."""
+    index = get_index()
+    if not settings.rag_local_enabled or settings.rag_mode != "local":
+        return {"enabled": False, "chunks": 0, "mode": settings.rag_mode}
+
+    count = index.build(settings.rag_documents_path, chunk_size=settings.rag_chunk_size)
+    return {
+        "enabled": True,
+        "mode": "local",
+        "chunks": count,
+        "documents_path": str(settings.rag_documents_path),
+    }
+
+
+def search_documents(query: str) -> list[SearchHit]:
+    if not settings.rag_local_enabled or settings.rag_mode != "local":
+        return []
+    index = get_index()
+    if not index.is_ready:
+        init_local_rag()
+    return index.search(query, top_k=settings.rag_top_k, min_score=settings.rag_min_score)
+
+
+def score_to_confidence(score: float) -> int:
+    """Нормализация cosine similarity в проценты уверенности."""
+    return min(95, max(35, int(score * 120)))
+
+
+def build_package_from_hits(
+    query: str,
+    hits: list[SearchHit],
+    *,
+    system: str | None = None,
+    conversation: list[dict[str, str]] | None = None,
+) -> ResponsePackage | None:
+    if not hits:
+        return None
+
+    top = hits[0]
+    confidence = score_to_confidence(top.score)
+    resolved_system = system or top.chunk.system or detect_system_from_text(query) or "Общее"
+
+    recommendation = _extract_recommendation(top.chunk.text)
+    steps = _extract_steps(top.chunk.text, hits)
+    sources = _unique_sources(hits)
+    colleague_cases = [
+        ColleagueCase.model_validate(case) for case in get_colleague_cases_for_system(resolved_system)
+    ]
+    priority = "Средний" if confidence >= settings.confidence_threshold else "Высокий"
+    warning = build_warning(recommendation, colleague_cases) if colleague_cases else None
+    why_suggested = build_why_suggested(
+        system=resolved_system,
+        sources=sources,
+        confidence=confidence,
+        source_title=top.chunk.source_title,
+    )
+
+    package = ResponsePackage(
+        recommendation=recommendation,
+        steps=steps,
+        sources=sources,
+        colleague_cases=colleague_cases or None,
+        warning=warning,
+        why_suggested=why_suggested,
+        confidence=confidence,
+        system=resolved_system,
+        ticket_draft=enrich_ticket_draft(
+            query,
+            resolved_system,
+            priority,
+            conversation=conversation,
+            sources=sources,
+            colleague_cases=colleague_cases,
+            recommendation=recommendation,
+        ),
+    )
+    return ensure_escalation_draft(package, query, conversation=conversation)
+
+
+def build_package_with_llm(
+    query: str,
+    hits: list[SearchHit],
+    llm_client: "LlmClient",
+    greeting: str,
+    *,
+    system: str | None = None,
+    conversation: list[dict[str, str]] | None = None,
+) -> tuple[str, ResponsePackage] | None:
+    context = "\n\n---\n\n".join(
+        f"[{h.chunk.source_title}]\n{h.chunk.text}" for h in hits[: settings.rag_top_k]
+    )
+    llm_text = llm_client.generate_with_context(query, context)
+    if not llm_text:
+        return None
+
+    package = build_package_from_hits(query, hits, system=system, conversation=conversation)
+    if package is None:
+        return None
+
+    package.recommendation = llm_text
+    if package.colleague_cases:
+        package.warning = build_warning(llm_text, package.colleague_cases)
+    if package.ticket_draft:
+        package.ticket_draft = enrich_ticket_draft(
+            query,
+            package.system or system or "Общее",
+            package.ticket_draft.priority,
+            conversation=conversation,
+            sources=package.sources,
+            colleague_cases=package.colleague_cases,
+            recommendation=llm_text,
+        )
+    package = ensure_escalation_draft(package, query, conversation=conversation)
+    threshold = settings.confidence_threshold
+    intro = (
+        f"{greeting}нашёл материалы в инструкциях и подготовил ответ:"
+        if package.confidence >= threshold
+        else (
+            f"{greeting}уверенность ниже {threshold}% — ответ может быть неполным. "
+            "Подготовил черновик заявки, его можно сразу отправить в поддержку:"
+        )
+    )
+    return intro, package
+
+
+def _extract_recommendation(text: str) -> str:
+    lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
+    if not lines:
+        return text[:400]
+    # Первая содержательная строка или абзац
+    paragraph = lines[0]
+    if len(paragraph) < 80 and len(lines) > 1:
+        paragraph = f"{paragraph} {lines[1]}"
+    return paragraph[:500]
+
+
+def _extract_steps(text: str, hits: list[SearchHit]) -> list[str]:
+    steps: list[str] = []
+    combined = "\n".join(h.chunk.text for h in hits[:3])
+
+    for line in combined.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        if re.match(r"^(\d+[\.\)]|[-•*])\s+", line):
+            steps.append(re.sub(r"^(\d+[\.\)]|[-•*])\s+", "", line))
+        elif "http" in line.lower() or line.lower().startswith("откройте"):
+            steps.append(line)
+        if len(steps) >= 6:
+            break
+
+    if not steps:
+        for line in combined.splitlines():
+            line = line.strip()
+            if len(line) > 20:
+                steps.append(line)
+            if len(steps) >= 4:
+                break
+
+    return steps[:6]
+
+
+def _unique_sources(hits: list[SearchHit]) -> list[Source]:
+    seen: set[str] = set()
+    sources: list[Source] = []
+    for hit in hits:
+        key = hit.chunk.source_path
+        if key in seen:
+            continue
+        seen.add(key)
+        sources.append(Source(title=hit.chunk.source_title, url=None))
+        if len(sources) >= 4:
+            break
+    return sources