gitrusprusandCursor
699e9be503
Generate static HeuristicProfile in CA admin, preview and run without LLM on each post via shared interpreter in CP batch and listener. Co-authored-by: Cursor <cursoragent@cursor.com>
131 lines
4.3 KiB
Python
131 lines
4.3 KiB
Python
"""Parser builder: one-shot DeepSeek profile generation + static preview."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from contracts.heuristic_profile import (
|
|
HeuristicProfile,
|
|
TARGET_FIELDS,
|
|
apply_profile,
|
|
target_field_specs,
|
|
)
|
|
|
|
logger = logging.getLogger("ca.parser_builder")
|
|
|
|
GENERATE_SYSTEM = (
|
|
"You design a static heuristic parser profile for structured Telegram posts. "
|
|
"Output valid JSON only, no markdown, no Python code. "
|
|
"The profile is applied with regex/line/marker rules at runtime — never with an LLM."
|
|
)
|
|
|
|
|
|
def deepseek_enabled() -> bool:
|
|
return bool(os.getenv("DEEPSEEK_API_KEY", "").strip())
|
|
|
|
|
|
def deepseek_settings() -> dict[str, str]:
|
|
return {
|
|
"api_key": os.getenv("DEEPSEEK_API_KEY", "").strip(),
|
|
"base_url": os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com").rstrip("/"),
|
|
"model": os.getenv("DEEPSEEK_MODEL", "deepseek-chat"),
|
|
}
|
|
|
|
|
|
def get_target_fields() -> list[dict[str, str]]:
|
|
return target_field_specs()
|
|
|
|
|
|
def preview_with_profile(sample_post: str, profile: dict[str, Any] | HeuristicProfile) -> dict[str, str]:
|
|
return apply_profile(sample_post, profile)
|
|
|
|
|
|
def _extract_json_object(content: str) -> dict[str, Any]:
|
|
content = (content or "").strip()
|
|
if content.startswith("```"):
|
|
content = re.sub(r"^```(?:json)?\s*", "", content)
|
|
content = re.sub(r"\s*```$", "", content)
|
|
try:
|
|
parsed = json.loads(content)
|
|
except json.JSONDecodeError:
|
|
match = re.search(r"\{[\s\S]*\}", content)
|
|
if not match:
|
|
raise
|
|
parsed = json.loads(match.group(0))
|
|
if not isinstance(parsed, dict):
|
|
raise ValueError("LLM response must be a JSON object")
|
|
return parsed
|
|
|
|
|
|
async def generate_profile(sample_post: str) -> HeuristicProfile:
|
|
settings = deepseek_settings()
|
|
if not settings["api_key"]:
|
|
raise RuntimeError(
|
|
"DEEPSEEK_API_KEY is not set on ca-api. Add it to .env for parser generation."
|
|
)
|
|
|
|
sample = (sample_post or "").strip()
|
|
if not sample:
|
|
raise ValueError("sample_post is required")
|
|
|
|
fields_help = "\n".join(
|
|
f"- {spec['name']}: {spec['description']}" for spec in target_field_specs()
|
|
)
|
|
user_prompt = (
|
|
"Build a HeuristicProfile JSON for this sample Telegram post.\n\n"
|
|
"Schema:\n"
|
|
'{"version": 1, "notes": "...", "fields": {'
|
|
'"<field>": {"strategy": "regex|line|after_marker|between|full_text|literal", '
|
|
'"pattern": "...", "group": 1, "line_index": 0, "marker": "...", '
|
|
'"end_marker": "...", "value": "...", "flags": "im", "strip": true}'
|
|
"}}\n\n"
|
|
"Rules:\n"
|
|
"- Only include fields you can extract reliably from the sample.\n"
|
|
f"- Allowed field names: {', '.join(TARGET_FIELDS)}.\n"
|
|
"- Prefer regex / line / after_marker / between over literal.\n"
|
|
"- Use literal only for constant topic tags.\n"
|
|
"- coords should capture 'lat, lon' when present.\n"
|
|
"- event_date should capture DD.MM.YYYY or similar.\n"
|
|
"- Do not invent Python code; only declarative rules.\n\n"
|
|
f"Target fields:\n{fields_help}\n\n"
|
|
f"Sample post:\n{sample[:12000]}"
|
|
)
|
|
|
|
payload = {
|
|
"model": settings["model"],
|
|
"messages": [
|
|
{"role": "system", "content": GENERATE_SYSTEM},
|
|
{"role": "user", "content": user_prompt},
|
|
],
|
|
"temperature": 0.1,
|
|
"response_format": {"type": "json_object"},
|
|
}
|
|
|
|
url = f"{settings['base_url']}/chat/completions"
|
|
async with httpx.AsyncClient(timeout=90.0) as client:
|
|
response = await client.post(
|
|
url,
|
|
headers={
|
|
"Authorization": f"Bearer {settings['api_key']}",
|
|
"Content-Type": "application/json",
|
|
},
|
|
json=payload,
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
content = data["choices"][0]["message"]["content"]
|
|
raw = _extract_json_object(content)
|
|
# Accept either top-level profile or {"profile": {...}}
|
|
if "fields" not in raw and isinstance(raw.get("profile"), dict):
|
|
raw = raw["profile"]
|
|
if "version" not in raw:
|
|
raw["version"] = 1
|
|
return HeuristicProfile.model_validate(raw)
|