Add reusable LLM parser profiles with multi-event extract.
Support kind=llm profiles (instruction/schema), optional multi-event posts via #eN URLs, and recover stale running/queued parse jobs after worker crashes. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -17,6 +17,8 @@ TARGET_FIELDS: tuple[str, ...] = (
|
||||
"region",
|
||||
)
|
||||
|
||||
COORDS_RE = re.compile(r"(-?\d{1,3}\.\d+)\s*,\s*(-?\d{1,3}\.\d+)")
|
||||
|
||||
Strategy = Literal["regex", "line", "after_marker", "between", "full_text", "literal"]
|
||||
|
||||
|
||||
@@ -45,8 +47,24 @@ class FieldRule(BaseModel):
|
||||
class HeuristicProfile(BaseModel):
|
||||
version: Literal[1] = 1
|
||||
fields: dict[str, FieldRule] = Field(default_factory=dict)
|
||||
required_fields: list[str] = Field(default_factory=list)
|
||||
notes: str = ""
|
||||
|
||||
@field_validator("required_fields")
|
||||
@classmethod
|
||||
def known_required(cls, value: list[str]) -> list[str]:
|
||||
seen: list[str] = []
|
||||
unknown: list[str] = []
|
||||
for name in value:
|
||||
if name not in TARGET_FIELDS:
|
||||
unknown.append(name)
|
||||
continue
|
||||
if name not in seen:
|
||||
seen.append(name)
|
||||
if unknown:
|
||||
raise ValueError(f"Unknown required_fields: {sorted(set(unknown))}")
|
||||
return seen
|
||||
|
||||
@model_validator(mode="after")
|
||||
def known_fields_only(self) -> "HeuristicProfile":
|
||||
unknown = set(self.fields) - set(TARGET_FIELDS)
|
||||
@@ -163,3 +181,38 @@ def apply_profile(text: str, profile: HeuristicProfile | dict[str, Any]) -> dict
|
||||
for name, rule in profile.fields.items():
|
||||
result[name] = _apply_rule(text, rule)
|
||||
return result
|
||||
|
||||
|
||||
def parse_coords(raw: str) -> tuple[float | None, float | None]:
|
||||
match = COORDS_RE.search(raw or "")
|
||||
if not match:
|
||||
return None, None
|
||||
return float(match.group(1)), float(match.group(2))
|
||||
|
||||
|
||||
def _field_is_filled(name: str, value: str) -> bool:
|
||||
if not (value or "").strip():
|
||||
return False
|
||||
if name == "coords":
|
||||
lat, lng = parse_coords(value)
|
||||
return lat is not None and lng is not None
|
||||
return True
|
||||
|
||||
|
||||
def match_profile(
|
||||
text: str,
|
||||
profile: HeuristicProfile | dict[str, Any],
|
||||
) -> tuple[bool, dict[str, str], list[str]]:
|
||||
"""Apply profile and report whether required_fields are filled.
|
||||
|
||||
Empty required_fields means no gate (legacy profiles ingest every post).
|
||||
"""
|
||||
if isinstance(profile, dict):
|
||||
profile = HeuristicProfile.model_validate(profile)
|
||||
fields = apply_profile(text, profile)
|
||||
missing = [
|
||||
name
|
||||
for name in profile.required_fields
|
||||
if not _field_is_filled(name, fields.get(name, ""))
|
||||
]
|
||||
return (not missing, fields, missing)
|
||||
|
||||
Reference in New Issue
Block a user