Add parser builder: one-shot DeepSeek profile for Telegram extract_mode=profile.

Generate static HeuristicProfile in CA admin, preview and run without LLM on each post via shared interpreter in CP batch and listener. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-16 14:14:18 +03:00
parent 6dff3c1c3d
commit 699e9be503
23 changed files with 1042 additions and 20 deletions
@@ -0,0 +1,130 @@
+"""Parser builder: one-shot DeepSeek profile generation + static preview."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from typing import Any
+
+import httpx
+
+from contracts.heuristic_profile import (
+    HeuristicProfile,
+    TARGET_FIELDS,
+    apply_profile,
+    target_field_specs,
+)
+
+logger = logging.getLogger("ca.parser_builder")
+
+GENERATE_SYSTEM = (
+    "You design a static heuristic parser profile for structured Telegram posts. "
+    "Output valid JSON only, no markdown, no Python code. "
+    "The profile is applied with regex/line/marker rules at runtime — never with an LLM."
+)
+
+
+def deepseek_enabled() -> bool:
+    return bool(os.getenv("DEEPSEEK_API_KEY", "").strip())
+
+
+def deepseek_settings() -> dict[str, str]:
+    return {
+        "api_key": os.getenv("DEEPSEEK_API_KEY", "").strip(),
+        "base_url": os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com").rstrip("/"),
+        "model": os.getenv("DEEPSEEK_MODEL", "deepseek-chat"),
+    }
+
+
+def get_target_fields() -> list[dict[str, str]]:
+    return target_field_specs()
+
+
+def preview_with_profile(sample_post: str, profile: dict[str, Any] | HeuristicProfile) -> dict[str, str]:
+    return apply_profile(sample_post, profile)
+
+
+def _extract_json_object(content: str) -> dict[str, Any]:
+    content = (content or "").strip()
+    if content.startswith("```"):
+        content = re.sub(r"^```(?:json)?\s*", "", content)
+        content = re.sub(r"\s*```$", "", content)
+    try:
+        parsed = json.loads(content)
+    except json.JSONDecodeError:
+        match = re.search(r"\{[\s\S]*\}", content)
+        if not match:
+            raise
+        parsed = json.loads(match.group(0))
+    if not isinstance(parsed, dict):
+        raise ValueError("LLM response must be a JSON object")
+    return parsed
+
+
+async def generate_profile(sample_post: str) -> HeuristicProfile:
+    settings = deepseek_settings()
+    if not settings["api_key"]:
+        raise RuntimeError(
+            "DEEPSEEK_API_KEY is not set on ca-api. Add it to .env for parser generation."
+        )
+
+    sample = (sample_post or "").strip()
+    if not sample:
+        raise ValueError("sample_post is required")
+
+    fields_help = "\n".join(
+        f"- {spec['name']}: {spec['description']}" for spec in target_field_specs()
+    )
+    user_prompt = (
+        "Build a HeuristicProfile JSON for this sample Telegram post.\n\n"
+        "Schema:\n"
+        '{"version": 1, "notes": "...", "fields": {'
+        '"<field>": {"strategy": "regex|line|after_marker|between|full_text|literal", '
+        '"pattern": "...", "group": 1, "line_index": 0, "marker": "...", '
+        '"end_marker": "...", "value": "...", "flags": "im", "strip": true}'
+        "}}\n\n"
+        "Rules:\n"
+        "- Only include fields you can extract reliably from the sample.\n"
+        f"- Allowed field names: {', '.join(TARGET_FIELDS)}.\n"
+        "- Prefer regex / line / after_marker / between over literal.\n"
+        "- Use literal only for constant topic tags.\n"
+        "- coords should capture 'lat, lon' when present.\n"
+        "- event_date should capture DD.MM.YYYY or similar.\n"
+        "- Do not invent Python code; only declarative rules.\n\n"
+        f"Target fields:\n{fields_help}\n\n"
+        f"Sample post:\n{sample[:12000]}"
+    )
+
+    payload = {
+        "model": settings["model"],
+        "messages": [
+            {"role": "system", "content": GENERATE_SYSTEM},
+            {"role": "user", "content": user_prompt},
+        ],
+        "temperature": 0.1,
+        "response_format": {"type": "json_object"},
+    }
+
+    url = f"{settings['base_url']}/chat/completions"
+    async with httpx.AsyncClient(timeout=90.0) as client:
+        response = await client.post(
+            url,
+            headers={
+                "Authorization": f"Bearer {settings['api_key']}",
+                "Content-Type": "application/json",
+            },
+            json=payload,
+        )
+        response.raise_for_status()
+        data = response.json()
+
+    content = data["choices"][0]["message"]["content"]
+    raw = _extract_json_object(content)
+    # Accept either top-level profile or {"profile": {...}}
+    if "fields" not in raw and isinstance(raw.get("profile"), dict):
+        raw = raw["profile"]
+    if "version" not in raw:
+        raw["version"] = 1
+    return HeuristicProfile.model_validate(raw)