Add reusable LLM parser profiles with multi-event extract.

Support kind=llm profiles (instruction/schema), optional multi-event posts via #eN URLs, and recover stale running/queued parse jobs after worker crashes.

Co-authored-by: Cursor <cursoragent@cursor.com>

contracts/heuristic_profile.py

@@ -17,6 +17,8 @@ TARGET_FIELDS: tuple[str, ...] = (
     "region",
 )
 
+COORDS_RE = re.compile(r"(-?\d{1,3}\.\d+)\s*,\s*(-?\d{1,3}\.\d+)")
+
 Strategy = Literal["regex", "line", "after_marker", "between", "full_text", "literal"]
 
 
@@ -45,8 +47,24 @@ class FieldRule(BaseModel):
 class HeuristicProfile(BaseModel):
     version: Literal[1] = 1
     fields: dict[str, FieldRule] = Field(default_factory=dict)
+    required_fields: list[str] = Field(default_factory=list)
     notes: str = ""
 
+    @field_validator("required_fields")
+    @classmethod
+    def known_required(cls, value: list[str]) -> list[str]:
+        seen: list[str] = []
+        unknown: list[str] = []
+        for name in value:
+            if name not in TARGET_FIELDS:
+                unknown.append(name)
+                continue
+            if name not in seen:
+                seen.append(name)
+        if unknown:
+            raise ValueError(f"Unknown required_fields: {sorted(set(unknown))}")
+        return seen
+
     @model_validator(mode="after")
     def known_fields_only(self) -> "HeuristicProfile":
         unknown = set(self.fields) - set(TARGET_FIELDS)
@@ -163,3 +181,38 @@ def apply_profile(text: str, profile: HeuristicProfile | dict[str, Any]) -> dict
     for name, rule in profile.fields.items():
         result[name] = _apply_rule(text, rule)
     return result
+
+
+def parse_coords(raw: str) -> tuple[float | None, float | None]:
+    match = COORDS_RE.search(raw or "")
+    if not match:
+        return None, None
+    return float(match.group(1)), float(match.group(2))
+
+
+def _field_is_filled(name: str, value: str) -> bool:
+    if not (value or "").strip():
+        return False
+    if name == "coords":
+        lat, lng = parse_coords(value)
+        return lat is not None and lng is not None
+    return True
+
+
+def match_profile(
+    text: str,
+    profile: HeuristicProfile | dict[str, Any],
+) -> tuple[bool, dict[str, str], list[str]]:
+    """Apply profile and report whether required_fields are filled.
+
+    Empty required_fields means no gate (legacy profiles ingest every post).
+    """
+    if isinstance(profile, dict):
+        profile = HeuristicProfile.model_validate(profile)
+    fields = apply_profile(text, profile)
+    missing = [
+        name
+        for name in profile.required_fields
+        if not _field_is_filled(name, fields.get(name, ""))
+    ]
+    return (not missing, fields, missing)

contracts/llm_profile.py

@@ -0,0 +1,86 @@
+"""Reusable LLM extract profile (CA storage + flattened into TelegramSourceConfig)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+from contracts.heuristic_profile import TARGET_FIELDS
+
+# LLM defaults omit region (same as workers/llm_extract.DEFAULT_EXTRACT_SCHEMA).
+LLM_SCHEMA_FIELDS: tuple[str, ...] = (
+    "title",
+    "locality",
+    "event_date",
+    "description",
+    "coords",
+    "topic",
+)
+
+DEFAULT_EXTRACT_SCHEMA: dict[str, str] = {
+    "title": "string — short event title",
+    "locality": "string — place / settlement name",
+    "event_date": "string — date as DD.MM.YYYY or YYYY-MM-DD if known",
+    "description": "string — concise event summary",
+    "coords": "string — latitude, longitude if present else empty",
+    "topic": "string — short topic tag",
+}
+
+DEFAULT_INSTRUCTION = (
+    "Extract structured military/news event fields from the text. "
+    "If the text is not an event, return is_event=false. "
+    "Respond with a single JSON object only."
+)
+
+
+class LlmProfile(BaseModel):
+    instruction: str | None = None
+    extract_schema: dict[str, str] = Field(default_factory=lambda: dict(DEFAULT_EXTRACT_SCHEMA))
+    required_fields: list[str] = Field(default_factory=list)
+    # When true, LLM may return multiple events per post (array "events")
+    multi_event: bool = False
+
+    @field_validator("instruction", mode="before")
+    @classmethod
+    def empty_instruction_to_none(cls, value: Any) -> Any:
+        if value is None:
+            return None
+        if isinstance(value, str) and not value.strip():
+            return None
+        return value
+
+    @field_validator("required_fields")
+    @classmethod
+    def known_required(cls, value: list[str]) -> list[str]:
+        seen: list[str] = []
+        unknown: list[str] = []
+        for name in value:
+            if name not in TARGET_FIELDS:
+                unknown.append(name)
+                continue
+            if name not in seen:
+                seen.append(name)
+        if unknown:
+            raise ValueError(f"Unknown required_fields: {sorted(set(unknown))}")
+        return seen
+
+    @model_validator(mode="after")
+    def known_schema_keys(self) -> "LlmProfile":
+        if not self.extract_schema:
+            raise ValueError("extract_schema must not be empty")
+        unknown = set(self.extract_schema) - set(TARGET_FIELDS)
+        if unknown:
+            raise ValueError(f"Unknown extract_schema fields: {sorted(unknown)}")
+        return self
+
+
+def match_llm_required(fields: dict[str, Any], required_fields: list[str]) -> bool:
+    """True if all required_fields are non-empty strings (empty required = no gate)."""
+    if not required_fields:
+        return True
+    for name in required_fields:
+        val = fields.get(name)
+        if val is None or not str(val).strip():
+            return False
+    return True

contracts/sources.py

@@ -15,6 +15,10 @@ class TelegramSourceConfig(BaseModel):
     extract_schema: dict[str, str] | None = None
     instruction: str | None = None
     heuristic_profile: dict | None = None
+    # Post-extract gate for extract_mode=llm (empty = only is_event filter)
+    required_fields: list[str] = Field(default_factory=list)
+    # Split one post into N events when extract_mode=llm (opt-in)
+    multi_event: bool = False
     sample_post: str | None = None  # audit / re-generate; not required at runtime
 
     @field_validator("channel")
@@ -22,15 +26,37 @@ class TelegramSourceConfig(BaseModel):
     def strip_channel(cls, value: str) -> str:
         return value.strip()
 
-    @model_validator(mode="after")
-    def profile_requires_rules(self) -> "TelegramSourceConfig":
-        if self.extract_mode != "profile":
-            return self
-        if not self.heuristic_profile:
-            raise ValueError("heuristic_profile required when extract_mode=profile")
-        from contracts.heuristic_profile import HeuristicProfile
+    @field_validator("required_fields")
+    @classmethod
+    def known_required(cls, value: list[str]) -> list[str]:
+        from contracts.heuristic_profile import TARGET_FIELDS
 
-        HeuristicProfile.model_validate(self.heuristic_profile)
+        seen: list[str] = []
+        unknown: list[str] = []
+        for name in value:
+            if name not in TARGET_FIELDS:
+                unknown.append(name)
+                continue
+            if name not in seen:
+                seen.append(name)
+        if unknown:
+            raise ValueError(f"Unknown required_fields: {sorted(set(unknown))}")
+        return seen
+
+    @model_validator(mode="after")
+    def validate_extract_mode(self) -> "TelegramSourceConfig":
+        if self.extract_mode == "profile":
+            if not self.heuristic_profile:
+                raise ValueError("heuristic_profile required when extract_mode=profile")
+            from contracts.heuristic_profile import HeuristicProfile
+
+            HeuristicProfile.model_validate(self.heuristic_profile)
+        elif self.extract_mode == "llm" and self.extract_schema is not None:
+            from contracts.heuristic_profile import TARGET_FIELDS
+
+            unknown = set(self.extract_schema) - set(TARGET_FIELDS)
+            if unknown:
+                raise ValueError(f"Unknown extract_schema fields: {sorted(unknown)}")
         return self