Add parser builder: one-shot DeepSeek profile for Telegram extract_mode=profile.

Generate static HeuristicProfile in CA admin, preview and run without LLM on each post via shared interpreter in CP batch and listener.

Co-authored-by: Cursor <cursoragent@cursor.com>

centers/analytics/api/app/main.py

@@ -13,7 +13,7 @@ for _candidate in (_HERE.parent, *_HERE.parents):
         break
 
 from .database import Base, engine, get_db
-from .routers import admin, internal, map, objects, v1
+from .routers import admin, internal, map, objects, parser_builder, v1
 from .seed import seed_objects, seed_test_consumer
 from .services.migrations import migrate_schema
 from .services.scheduler import start_scheduler
@@ -50,4 +50,5 @@ app.include_router(objects.router)
 app.include_router(map.router)
 app.include_router(internal.router)
 app.include_router(admin.router)
+app.include_router(parser_builder.router)
 app.include_router(v1.router)

centers/analytics/api/app/routers/internal.py

@@ -71,5 +71,11 @@ def listener_subscriptions(
         channel = job.source_config.get("channel") if job.source_config else None
         if not channel:
             continue
-        result.append(ListenerSubscription(job_id=job.id, channel=str(channel)))
+        result.append(
+            ListenerSubscription(
+                job_id=job.id,
+                channel=str(channel),
+                source_config=dict(job.source_config or {}),
+            )
+        )
     return result

centers/analytics/api/app/routers/parser_builder.py

@@ -0,0 +1,108 @@
+"""Admin API: construct static telegram parsers from a sample post."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel, Field
+from sqlalchemy.orm import Session
+
+from contracts.heuristic_profile import HeuristicProfile
+from contracts.sources import TelegramSourceConfig
+
+from ..database import get_db
+from ..models import ParseJob
+from ..schemas import ParseJobRead
+from ..services.jobs import enqueue_job
+from ..services import parser_builder as builder
+
+router = APIRouter(prefix="/admin/parser-builder", tags=["parser-builder"])
+
+
+class GenerateRequest(BaseModel):
+    sample_post: str = Field(min_length=1)
+
+
+class GenerateResponse(BaseModel):
+    profile: dict[str, Any]
+
+
+class PreviewRequest(BaseModel):
+    sample_post: str = Field(min_length=1)
+    profile: dict[str, Any]
+
+
+class PreviewResponse(BaseModel):
+    fields: dict[str, str]
+
+
+class CreateProfileJobRequest(BaseModel):
+    channel: str = Field(min_length=1)
+    profile: dict[str, Any]
+    sample_post: str | None = None
+    limit: int = Field(default=100, ge=1, le=1000)
+    interval_seconds: int = Field(default=3600, ge=60, le=604800)
+    is_active: bool = True
+
+
+@router.get("/target-fields")
+def target_fields():
+    return {"fields": builder.get_target_fields()}
+
+
+@router.post("/generate", response_model=GenerateResponse)
+async def generate_profile(payload: GenerateRequest):
+    if not builder.deepseek_enabled():
+        raise HTTPException(
+            status_code=503,
+            detail="DEEPSEEK_API_KEY is not set on ca-api. Add it to .env for parser generation.",
+        )
+    try:
+        profile = await builder.generate_profile(payload.sample_post)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:
+        raise HTTPException(
+            status_code=502,
+            detail=f"DeepSeek generate failed: {exc}",
+        ) from exc
+    return GenerateResponse(profile=profile.model_dump())
+
+
+@router.post("/preview", response_model=PreviewResponse)
+def preview_profile(payload: PreviewRequest):
+    try:
+        HeuristicProfile.model_validate(payload.profile)
+        fields = builder.preview_with_profile(payload.sample_post, payload.profile)
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    return PreviewResponse(fields=fields)
+
+
+@router.post("/jobs", response_model=ParseJobRead, status_code=201)
+def create_profile_job(payload: CreateProfileJobRequest, db: Session = Depends(get_db)):
+    try:
+        cfg = TelegramSourceConfig(
+            channel=payload.channel.strip(),
+            limit=payload.limit,
+            extract_mode="profile",
+            heuristic_profile=payload.profile,
+            sample_post=payload.sample_post,
+        )
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    job = ParseJob(
+        source_type="telegram",
+        source_config=cfg.model_dump(),
+        interval_seconds=payload.interval_seconds,
+        is_active=payload.is_active,
+        status="queued",
+    )
+    db.add(job)
+    db.commit()
+    db.refresh(job)
+
+    enqueue_job(job.id, job.source_type, job.source_config)
+    return job

centers/analytics/api/app/schemas.py

@@ -117,6 +117,7 @@ class IngestRequest(BaseModel):
 class ListenerSubscription(BaseModel):
     job_id: int
     channel: str
+    source_config: dict[str, Any] = Field(default_factory=dict)
 
 
 class IngestResponse(BaseModel):

centers/analytics/api/app/services/parser_builder.py

@@ -0,0 +1,130 @@
+"""Parser builder: one-shot DeepSeek profile generation + static preview."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from typing import Any
+
+import httpx
+
+from contracts.heuristic_profile import (
+    HeuristicProfile,
+    TARGET_FIELDS,
+    apply_profile,
+    target_field_specs,
+)
+
+logger = logging.getLogger("ca.parser_builder")
+
+GENERATE_SYSTEM = (
+    "You design a static heuristic parser profile for structured Telegram posts. "
+    "Output valid JSON only, no markdown, no Python code. "
+    "The profile is applied with regex/line/marker rules at runtime — never with an LLM."
+)
+
+
+def deepseek_enabled() -> bool:
+    return bool(os.getenv("DEEPSEEK_API_KEY", "").strip())
+
+
+def deepseek_settings() -> dict[str, str]:
+    return {
+        "api_key": os.getenv("DEEPSEEK_API_KEY", "").strip(),
+        "base_url": os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com").rstrip("/"),
+        "model": os.getenv("DEEPSEEK_MODEL", "deepseek-chat"),
+    }
+
+
+def get_target_fields() -> list[dict[str, str]]:
+    return target_field_specs()
+
+
+def preview_with_profile(sample_post: str, profile: dict[str, Any] | HeuristicProfile) -> dict[str, str]:
+    return apply_profile(sample_post, profile)
+
+
+def _extract_json_object(content: str) -> dict[str, Any]:
+    content = (content or "").strip()
+    if content.startswith("```"):
+        content = re.sub(r"^```(?:json)?\s*", "", content)
+        content = re.sub(r"\s*```$", "", content)
+    try:
+        parsed = json.loads(content)
+    except json.JSONDecodeError:
+        match = re.search(r"\{[\s\S]*\}", content)
+        if not match:
+            raise
+        parsed = json.loads(match.group(0))
+    if not isinstance(parsed, dict):
+        raise ValueError("LLM response must be a JSON object")
+    return parsed
+
+
+async def generate_profile(sample_post: str) -> HeuristicProfile:
+    settings = deepseek_settings()
+    if not settings["api_key"]:
+        raise RuntimeError(
+            "DEEPSEEK_API_KEY is not set on ca-api. Add it to .env for parser generation."
+        )
+
+    sample = (sample_post or "").strip()
+    if not sample:
+        raise ValueError("sample_post is required")
+
+    fields_help = "\n".join(
+        f"- {spec['name']}: {spec['description']}" for spec in target_field_specs()
+    )
+    user_prompt = (
+        "Build a HeuristicProfile JSON for this sample Telegram post.\n\n"
+        "Schema:\n"
+        '{"version": 1, "notes": "...", "fields": {'
+        '"<field>": {"strategy": "regex|line|after_marker|between|full_text|literal", '
+        '"pattern": "...", "group": 1, "line_index": 0, "marker": "...", '
+        '"end_marker": "...", "value": "...", "flags": "im", "strip": true}'
+        "}}\n\n"
+        "Rules:\n"
+        "- Only include fields you can extract reliably from the sample.\n"
+        f"- Allowed field names: {', '.join(TARGET_FIELDS)}.\n"
+        "- Prefer regex / line / after_marker / between over literal.\n"
+        "- Use literal only for constant topic tags.\n"
+        "- coords should capture 'lat, lon' when present.\n"
+        "- event_date should capture DD.MM.YYYY or similar.\n"
+        "- Do not invent Python code; only declarative rules.\n\n"
+        f"Target fields:\n{fields_help}\n\n"
+        f"Sample post:\n{sample[:12000]}"
+    )
+
+    payload = {
+        "model": settings["model"],
+        "messages": [
+            {"role": "system", "content": GENERATE_SYSTEM},
+            {"role": "user", "content": user_prompt},
+        ],
+        "temperature": 0.1,
+        "response_format": {"type": "json_object"},
+    }
+
+    url = f"{settings['base_url']}/chat/completions"
+    async with httpx.AsyncClient(timeout=90.0) as client:
+        response = await client.post(
+            url,
+            headers={
+                "Authorization": f"Bearer {settings['api_key']}",
+                "Content-Type": "application/json",
+            },
+            json=payload,
+        )
+        response.raise_for_status()
+        data = response.json()
+
+    content = data["choices"][0]["message"]["content"]
+    raw = _extract_json_object(content)
+    # Accept either top-level profile or {"profile": {...}}
+    if "fields" not in raw and isinstance(raw.get("profile"), dict):
+        raw = raw["profile"]
+    if "version" not in raw:
+        raw["version"] = 1
+    return HeuristicProfile.model_validate(raw)