Add reusable LLM parser profiles with multi-event extract.

Support kind=llm profiles (instruction/schema), optional multi-event posts via #eN URLs, and recover stale running/queued parse jobs after worker crashes.

Co-authored-by: Cursor <cursoragent@cursor.com>

centers/analytics/api/app/routers/parser_profiles.py

@@ -9,6 +9,7 @@ from pydantic import BaseModel, Field
 from sqlalchemy.orm import Session
 
 from contracts.heuristic_profile import HeuristicProfile
+from contracts.llm_profile import LlmProfile
 
 from ..database import get_db
 from ..deps import verify_admin
@@ -44,6 +45,22 @@ class PreviewRequest(BaseModel):
 
 class PreviewResponse(BaseModel):
     fields: dict[str, str]
+    matched: bool = True
+    missing_required: list[str] = Field(default_factory=list)
+
+
+class PreviewLlmRequest(BaseModel):
+    sample_post: str = Field(min_length=1)
+    llm_profile: dict[str, Any]
+
+
+class PreviewLlmResponse(BaseModel):
+    fields: dict[str, str]
+    events: list[dict[str, str]] = Field(default_factory=list)
+    matched: bool = True
+    missing_required: list[str] = Field(default_factory=list)
+    is_event: bool = True
+    matched_count: int = 0
 
 
 def _validate_heuristic_profile(raw: dict[str, Any] | None) -> dict[str, Any] | None:
@@ -55,9 +72,33 @@ def _validate_heuristic_profile(raw: dict[str, Any] | None) -> dict[str, Any] |
         raise HTTPException(status_code=400, detail=f"Invalid heuristic_profile: {exc}") from exc
 
 
-def _profile_status(heuristic_profile: dict | None, explicit: str | None = None) -> str:
+def _validate_llm_profile(raw: dict[str, Any] | None) -> dict[str, Any] | None:
+    if raw is None:
+        return None
+    try:
+        return LlmProfile.model_validate(raw).model_dump()
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=f"Invalid llm_profile: {exc}") from exc
+
+
+def _normalize_kind(kind: str | None) -> str:
+    value = (kind or "heuristic").strip().lower()
+    if value not in ("heuristic", "llm"):
+        raise HTTPException(status_code=400, detail="kind must be heuristic or llm")
+    return value
+
+
+def _profile_status(
+    *,
+    kind: str,
+    heuristic_profile: dict | None,
+    llm_profile: dict | None,
+    explicit: str | None = None,
+) -> str:
     if explicit:
         return explicit
+    if kind == "llm":
+        return "ready" if llm_profile else "draft"
     return "ready" if heuristic_profile else "draft"
 
 
@@ -66,6 +107,18 @@ def target_fields():
     return {"fields": builder.get_target_fields()}
 
 
+@router.get("/llm-defaults")
+def llm_defaults():
+    from contracts.llm_profile import DEFAULT_EXTRACT_SCHEMA, DEFAULT_INSTRUCTION
+
+    return {
+        "instruction": DEFAULT_INSTRUCTION,
+        "extract_schema": DEFAULT_EXTRACT_SCHEMA,
+        "required_fields": [],
+        "multi_event": False,
+    }
+
+
 @router.post("/generate", response_model=GenerateResponse)
 async def generate_profile(payload: GenerateRequest):
     if not builder.deepseek_enabled():
@@ -87,6 +140,9 @@ async def generate_profile(payload: GenerateRequest):
             detail=f"DeepSeek generate failed: {exc}",
         ) from exc
     dumped = profile.model_dump()
+    if payload.current_profile and isinstance(payload.current_profile.get("required_fields"), list):
+        dumped["required_fields"] = payload.current_profile["required_fields"]
+        dumped = HeuristicProfile.model_validate(dumped).model_dump()
     preview = builder.preview_with_profile(payload.sample_post, dumped)
     return GenerateResponse(
         profile=dumped,
@@ -102,10 +158,40 @@ def preview_profile(payload: PreviewRequest):
         raise HTTPException(status_code=400, detail="heuristic_profile is required")
     try:
         HeuristicProfile.model_validate(raw)
-        fields = builder.preview_with_profile(payload.sample_post, raw)
+        fields, matched, missing = builder.match_preview(payload.sample_post, raw)
     except Exception as exc:
         raise HTTPException(status_code=400, detail=str(exc)) from exc
-    return PreviewResponse(fields=fields)
+    return PreviewResponse(fields=fields, matched=matched, missing_required=missing)
+
+
+@router.post("/preview-llm", response_model=PreviewLlmResponse)
+async def preview_llm_profile(payload: PreviewLlmRequest):
+    if not builder.deepseek_enabled():
+        raise HTTPException(
+            status_code=503,
+            detail="DEEPSEEK_API_KEY is not set on ca-api. Add it to .env for LLM preview.",
+        )
+    try:
+        LlmProfile.model_validate(payload.llm_profile)
+        events, fields, matched, missing, is_event, matched_count = await builder.preview_llm_extract(
+            payload.sample_post,
+            payload.llm_profile,
+        )
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:
+        raise HTTPException(
+            status_code=502,
+            detail=f"DeepSeek LLM preview failed: {exc}",
+        ) from exc
+    return PreviewLlmResponse(
+        fields=fields,
+        events=events,
+        matched=matched,
+        missing_required=missing,
+        is_event=is_event,
+        matched_count=matched_count,
+    )
 
 
 @router.get("", response_model=list[ParserProfileRead])
@@ -115,12 +201,26 @@ def list_profiles(db: Session = Depends(get_db)):
 
 @router.post("", response_model=ParserProfileRead, status_code=201)
 def create_profile(payload: ParserProfileCreate, db: Session = Depends(get_db)):
+    kind = _normalize_kind(payload.kind)
     heuristic = _validate_heuristic_profile(payload.heuristic_profile)
+    llm = _validate_llm_profile(payload.llm_profile)
+    if kind == "heuristic" and llm and not heuristic:
+        # ignore stray llm blob when creating heuristic
+        llm = None
+    if kind == "llm" and heuristic and not llm:
+        heuristic = None
     profile = ParserProfile(
         name=payload.name.strip(),
+        kind=kind,
         sample_post=payload.sample_post or "",
-        heuristic_profile=heuristic,
-        status=_profile_status(heuristic, payload.status),
+        heuristic_profile=heuristic if kind == "heuristic" else None,
+        llm_profile=llm if kind == "llm" else None,
+        status=_profile_status(
+            kind=kind,
+            heuristic_profile=heuristic if kind == "heuristic" else None,
+            llm_profile=llm if kind == "llm" else None,
+            explicit=payload.status,
+        ),
     )
     db.add(profile)
     db.commit()
@@ -150,12 +250,33 @@ def update_profile(
         profile.name = payload.name.strip()
     if payload.sample_post is not None:
         profile.sample_post = payload.sample_post
+    if payload.kind is not None:
+        profile.kind = _normalize_kind(payload.kind)
+    kind = _normalize_kind(profile.kind)
+
     if "heuristic_profile" in payload.model_fields_set:
         profile.heuristic_profile = _validate_heuristic_profile(payload.heuristic_profile)
+    if "llm_profile" in payload.model_fields_set:
+        profile.llm_profile = _validate_llm_profile(payload.llm_profile)
+
+    # Keep only the blob matching kind
+    if kind == "heuristic":
+        profile.llm_profile = None
+    else:
+        profile.heuristic_profile = None
+
     if payload.status is not None:
         profile.status = payload.status
-    elif "heuristic_profile" in payload.model_fields_set:
-        profile.status = _profile_status(profile.heuristic_profile)
+    elif (
+        "heuristic_profile" in payload.model_fields_set
+        or "llm_profile" in payload.model_fields_set
+        or payload.kind is not None
+    ):
+        profile.status = _profile_status(
+            kind=kind,
+            heuristic_profile=profile.heuristic_profile,
+            llm_profile=profile.llm_profile,
+        )
 
     db.commit()
     db.refresh(profile)