Add reusable LLM parser profiles with multi-event extract.

Support kind=llm profiles (instruction/schema), optional multi-event posts via #eN URLs, and recover stale running/queued parse jobs after worker crashes.

Co-authored-by: Cursor <cursoragent@cursor.com>

centers/analytics/api/app/routers/admin.py

@@ -87,7 +87,11 @@ def create_parse_job(payload: ParseJobCreate, db: Session = Depends(get_db)):
             raise HTTPException(status_code=404, detail="Profile not found")
         if not channel:
             raise HTTPException(status_code=404, detail="Channel not found")
-        if not profile.heuristic_profile:
+        kind = (profile.kind or "heuristic").strip().lower()
+        if kind == "llm":
+            if not profile.llm_profile:
+                raise HTTPException(status_code=400, detail="LLM profile has no llm_profile")
+        elif not profile.heuristic_profile:
             raise HTTPException(status_code=400, detail="Profile has no heuristic_profile")
         if not channel.is_active:
             raise HTTPException(status_code=400, detail="Channel is inactive")
@@ -146,7 +150,10 @@ def retry_parse_job(job_id: int, db: Session = Depends(get_db)):
     job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
     if not job:
         raise HTTPException(status_code=404, detail="Job not found")
-    if job.status in ("queued", "running"):
+
+    from ..services.job_stale import is_stale_job
+
+    if job.status in ("queued", "running") and not is_stale_job(job):
         raise HTTPException(status_code=409, detail="Job is already running or queued")
 
     job.status = "queued"
@@ -218,7 +225,14 @@ def delete_parse_job(job_id: int, db: Session = Depends(get_db)):
     if not job:
         raise HTTPException(status_code=404, detail="Job not found")
     if job.status == "running":
-        raise HTTPException(status_code=409, detail="Cannot delete a running job")
+        from ..services.job_stale import is_stale_job
+
+        if not is_stale_job(job):
+            raise HTTPException(status_code=409, detail="Cannot delete a running job")
+        # Stale running — allow delete after marking failed for audit trail
+        job.status = "failed"
+        job.last_error = "Deleted while stale running"
+        db.commit()
 
     db.delete(job)
     db.commit()

centers/analytics/api/app/routers/internal.py

@@ -52,7 +52,8 @@ def update_job_status(
         raise HTTPException(status_code=404, detail="Job not found")
 
     job.status = status
-    if status in ("completed", "failed"):
+    # Anchor staleness detection: running/queued start, and terminal finish
+    if status in ("running", "queued", "completed", "failed"):
         job.last_run_at = datetime.now(timezone.utc)
     job.last_error = error
     db.commit()

centers/analytics/api/app/routers/parser_profiles.py

@@ -9,6 +9,7 @@ from pydantic import BaseModel, Field
 from sqlalchemy.orm import Session
 
 from contracts.heuristic_profile import HeuristicProfile
+from contracts.llm_profile import LlmProfile
 
 from ..database import get_db
 from ..deps import verify_admin
@@ -44,6 +45,22 @@ class PreviewRequest(BaseModel):
 
 class PreviewResponse(BaseModel):
     fields: dict[str, str]
+    matched: bool = True
+    missing_required: list[str] = Field(default_factory=list)
+
+
+class PreviewLlmRequest(BaseModel):
+    sample_post: str = Field(min_length=1)
+    llm_profile: dict[str, Any]
+
+
+class PreviewLlmResponse(BaseModel):
+    fields: dict[str, str]
+    events: list[dict[str, str]] = Field(default_factory=list)
+    matched: bool = True
+    missing_required: list[str] = Field(default_factory=list)
+    is_event: bool = True
+    matched_count: int = 0
 
 
 def _validate_heuristic_profile(raw: dict[str, Any] | None) -> dict[str, Any] | None:
@@ -55,9 +72,33 @@ def _validate_heuristic_profile(raw: dict[str, Any] | None) -> dict[str, Any] |
         raise HTTPException(status_code=400, detail=f"Invalid heuristic_profile: {exc}") from exc
 
 
-def _profile_status(heuristic_profile: dict | None, explicit: str | None = None) -> str:
+def _validate_llm_profile(raw: dict[str, Any] | None) -> dict[str, Any] | None:
+    if raw is None:
+        return None
+    try:
+        return LlmProfile.model_validate(raw).model_dump()
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=f"Invalid llm_profile: {exc}") from exc
+
+
+def _normalize_kind(kind: str | None) -> str:
+    value = (kind or "heuristic").strip().lower()
+    if value not in ("heuristic", "llm"):
+        raise HTTPException(status_code=400, detail="kind must be heuristic or llm")
+    return value
+
+
+def _profile_status(
+    *,
+    kind: str,
+    heuristic_profile: dict | None,
+    llm_profile: dict | None,
+    explicit: str | None = None,
+) -> str:
     if explicit:
         return explicit
+    if kind == "llm":
+        return "ready" if llm_profile else "draft"
     return "ready" if heuristic_profile else "draft"
 
 
@@ -66,6 +107,18 @@ def target_fields():
     return {"fields": builder.get_target_fields()}
 
 
+@router.get("/llm-defaults")
+def llm_defaults():
+    from contracts.llm_profile import DEFAULT_EXTRACT_SCHEMA, DEFAULT_INSTRUCTION
+
+    return {
+        "instruction": DEFAULT_INSTRUCTION,
+        "extract_schema": DEFAULT_EXTRACT_SCHEMA,
+        "required_fields": [],
+        "multi_event": False,
+    }
+
+
 @router.post("/generate", response_model=GenerateResponse)
 async def generate_profile(payload: GenerateRequest):
     if not builder.deepseek_enabled():
@@ -87,6 +140,9 @@ async def generate_profile(payload: GenerateRequest):
             detail=f"DeepSeek generate failed: {exc}",
         ) from exc
     dumped = profile.model_dump()
+    if payload.current_profile and isinstance(payload.current_profile.get("required_fields"), list):
+        dumped["required_fields"] = payload.current_profile["required_fields"]
+        dumped = HeuristicProfile.model_validate(dumped).model_dump()
     preview = builder.preview_with_profile(payload.sample_post, dumped)
     return GenerateResponse(
         profile=dumped,
@@ -102,10 +158,40 @@ def preview_profile(payload: PreviewRequest):
         raise HTTPException(status_code=400, detail="heuristic_profile is required")
     try:
         HeuristicProfile.model_validate(raw)
-        fields = builder.preview_with_profile(payload.sample_post, raw)
+        fields, matched, missing = builder.match_preview(payload.sample_post, raw)
     except Exception as exc:
         raise HTTPException(status_code=400, detail=str(exc)) from exc
-    return PreviewResponse(fields=fields)
+    return PreviewResponse(fields=fields, matched=matched, missing_required=missing)
+
+
+@router.post("/preview-llm", response_model=PreviewLlmResponse)
+async def preview_llm_profile(payload: PreviewLlmRequest):
+    if not builder.deepseek_enabled():
+        raise HTTPException(
+            status_code=503,
+            detail="DEEPSEEK_API_KEY is not set on ca-api. Add it to .env for LLM preview.",
+        )
+    try:
+        LlmProfile.model_validate(payload.llm_profile)
+        events, fields, matched, missing, is_event, matched_count = await builder.preview_llm_extract(
+            payload.sample_post,
+            payload.llm_profile,
+        )
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:
+        raise HTTPException(
+            status_code=502,
+            detail=f"DeepSeek LLM preview failed: {exc}",
+        ) from exc
+    return PreviewLlmResponse(
+        fields=fields,
+        events=events,
+        matched=matched,
+        missing_required=missing,
+        is_event=is_event,
+        matched_count=matched_count,
+    )
 
 
 @router.get("", response_model=list[ParserProfileRead])
@@ -115,12 +201,26 @@ def list_profiles(db: Session = Depends(get_db)):
 
 @router.post("", response_model=ParserProfileRead, status_code=201)
 def create_profile(payload: ParserProfileCreate, db: Session = Depends(get_db)):
+    kind = _normalize_kind(payload.kind)
     heuristic = _validate_heuristic_profile(payload.heuristic_profile)
+    llm = _validate_llm_profile(payload.llm_profile)
+    if kind == "heuristic" and llm and not heuristic:
+        # ignore stray llm blob when creating heuristic
+        llm = None
+    if kind == "llm" and heuristic and not llm:
+        heuristic = None
     profile = ParserProfile(
         name=payload.name.strip(),
+        kind=kind,
         sample_post=payload.sample_post or "",
-        heuristic_profile=heuristic,
-        status=_profile_status(heuristic, payload.status),
+        heuristic_profile=heuristic if kind == "heuristic" else None,
+        llm_profile=llm if kind == "llm" else None,
+        status=_profile_status(
+            kind=kind,
+            heuristic_profile=heuristic if kind == "heuristic" else None,
+            llm_profile=llm if kind == "llm" else None,
+            explicit=payload.status,
+        ),
     )
     db.add(profile)
     db.commit()
@@ -150,12 +250,33 @@ def update_profile(
         profile.name = payload.name.strip()
     if payload.sample_post is not None:
         profile.sample_post = payload.sample_post
+    if payload.kind is not None:
+        profile.kind = _normalize_kind(payload.kind)
+    kind = _normalize_kind(profile.kind)
+
     if "heuristic_profile" in payload.model_fields_set:
         profile.heuristic_profile = _validate_heuristic_profile(payload.heuristic_profile)
+    if "llm_profile" in payload.model_fields_set:
+        profile.llm_profile = _validate_llm_profile(payload.llm_profile)
+
+    # Keep only the blob matching kind
+    if kind == "heuristic":
+        profile.llm_profile = None
+    else:
+        profile.heuristic_profile = None
+
     if payload.status is not None:
         profile.status = payload.status
-    elif "heuristic_profile" in payload.model_fields_set:
-        profile.status = _profile_status(profile.heuristic_profile)
+    elif (
+        "heuristic_profile" in payload.model_fields_set
+        or "llm_profile" in payload.model_fields_set
+        or payload.kind is not None
+    ):
+        profile.status = _profile_status(
+            kind=kind,
+            heuristic_profile=profile.heuristic_profile,
+            llm_profile=profile.llm_profile,
+        )
 
     db.commit()
     db.refresh(profile)