Introduce reusable ParserProfile and ParseChannel, pair jobs with enqueue flatten, Events admin CRUD, and drop inline/legacy parser-builder UI and aliases. Co-authored-by: Cursor <cursoragent@cursor.com>
166 lines
5.0 KiB
Python
166 lines
5.0 KiB
Python
"""Declarative heuristic profile: schema + static rule interpreter (CA preview + CP runtime)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from typing import Any, Literal
|
|
|
|
from pydantic import BaseModel, Field, field_validator, model_validator
|
|
|
|
TARGET_FIELDS: tuple[str, ...] = (
|
|
"title",
|
|
"description",
|
|
"locality",
|
|
"event_date",
|
|
"coords",
|
|
"topic",
|
|
"region",
|
|
)
|
|
|
|
Strategy = Literal["regex", "line", "after_marker", "between", "full_text", "literal"]
|
|
|
|
|
|
class FieldRule(BaseModel):
|
|
strategy: Strategy
|
|
# regex: named or group(1); line: 0-based index; after_marker/between: markers
|
|
pattern: str | None = None
|
|
group: int = 1
|
|
line_index: int | None = None
|
|
marker: str | None = None
|
|
end_marker: str | None = None
|
|
value: str | None = None # literal
|
|
flags: str = "" # e.g. "im" → re.I|re.M
|
|
strip: bool = True
|
|
|
|
@field_validator("pattern", "marker", "end_marker", "value", mode="before")
|
|
@classmethod
|
|
def empty_to_none(cls, value: Any) -> Any:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, str) and not value.strip():
|
|
return None
|
|
return value
|
|
|
|
|
|
class HeuristicProfile(BaseModel):
|
|
version: Literal[1] = 1
|
|
fields: dict[str, FieldRule] = Field(default_factory=dict)
|
|
notes: str = ""
|
|
|
|
@model_validator(mode="after")
|
|
def known_fields_only(self) -> "HeuristicProfile":
|
|
unknown = set(self.fields) - set(TARGET_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"Unknown profile fields: {sorted(unknown)}")
|
|
return self
|
|
|
|
|
|
def target_field_specs() -> list[dict[str, str]]:
|
|
"""Fixed target table for profile UI (roadmap: custom tables later)."""
|
|
return [
|
|
{"name": "title", "type": "string", "description": "Short event title"},
|
|
{"name": "description", "type": "string", "description": "Event summary / body"},
|
|
{"name": "locality", "type": "string", "description": "Place / settlement name"},
|
|
{
|
|
"name": "event_date",
|
|
"type": "string",
|
|
"description": "Date as DD.MM.YYYY or YYYY-MM-DD",
|
|
},
|
|
{
|
|
"name": "coords",
|
|
"type": "string",
|
|
"description": "Latitude, longitude if present",
|
|
},
|
|
{"name": "topic", "type": "string", "description": "Short topic tag"},
|
|
{"name": "region", "type": "string", "description": "Region (optional)"},
|
|
]
|
|
|
|
|
|
def _compile_flags(flags: str) -> int:
|
|
mapping = {
|
|
"i": re.IGNORECASE,
|
|
"m": re.MULTILINE,
|
|
"s": re.DOTALL,
|
|
}
|
|
result = 0
|
|
for ch in (flags or "").lower():
|
|
result |= mapping.get(ch, 0)
|
|
return result
|
|
|
|
|
|
def _apply_rule(text: str, rule: FieldRule) -> str:
|
|
raw = text or ""
|
|
value = ""
|
|
|
|
if rule.strategy == "literal":
|
|
value = rule.value or ""
|
|
elif rule.strategy == "full_text":
|
|
value = raw
|
|
elif rule.strategy == "line":
|
|
lines = raw.splitlines()
|
|
idx = 0 if rule.line_index is None else rule.line_index
|
|
if 0 <= idx < len(lines):
|
|
value = lines[idx]
|
|
elif rule.strategy == "regex":
|
|
if not rule.pattern:
|
|
return ""
|
|
match = re.search(rule.pattern, raw, _compile_flags(rule.flags))
|
|
if match:
|
|
try:
|
|
value = match.group(rule.group)
|
|
except IndexError:
|
|
value = match.group(0)
|
|
elif rule.strategy == "after_marker":
|
|
marker = rule.marker or ""
|
|
if not marker:
|
|
return ""
|
|
pos = raw.find(marker)
|
|
if pos < 0:
|
|
return ""
|
|
start = pos + len(marker)
|
|
rest = raw[start:]
|
|
if rule.end_marker:
|
|
end = rest.find(rule.end_marker)
|
|
value = rest[:end] if end >= 0 else rest
|
|
elif rule.pattern:
|
|
match = re.search(rule.pattern, rest, _compile_flags(rule.flags))
|
|
if match:
|
|
try:
|
|
value = match.group(rule.group)
|
|
except IndexError:
|
|
value = match.group(0)
|
|
else:
|
|
# first non-empty line after marker
|
|
for line in rest.splitlines():
|
|
if line.strip():
|
|
value = line
|
|
break
|
|
elif rule.strategy == "between":
|
|
start_m = rule.marker or ""
|
|
end_m = rule.end_marker or ""
|
|
if not start_m or not end_m:
|
|
return ""
|
|
start = raw.find(start_m)
|
|
if start < 0:
|
|
return ""
|
|
start += len(start_m)
|
|
end = raw.find(end_m, start)
|
|
if end < 0:
|
|
return ""
|
|
value = raw[start:end]
|
|
|
|
if rule.strip:
|
|
value = value.strip()
|
|
return value
|
|
|
|
|
|
def apply_profile(text: str, profile: HeuristicProfile | dict[str, Any]) -> dict[str, str]:
|
|
"""Apply static rules to post text → string field map (no LLM)."""
|
|
if isinstance(profile, dict):
|
|
profile = HeuristicProfile.model_validate(profile)
|
|
|
|
result: dict[str, str] = {name: "" for name in TARGET_FIELDS}
|
|
for name, rule in profile.fields.items():
|
|
result[name] = _apply_rule(text, rule)
|
|
return result
|