gitrusprusandCursor
3f9dc6643b
Support kind=llm profiles (instruction/schema), optional multi-event posts via #eN URLs, and recover stale running/queued parse jobs after worker crashes. Co-authored-by: Cursor <cursoragent@cursor.com>
126 lines
4.5 KiB
Python
126 lines
4.5 KiB
Python
"""Per-source_type source_config schemas (CA admin + CP adapters)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Literal
|
|
|
|
from pydantic import BaseModel, Field, field_validator, model_validator
|
|
|
|
|
|
class TelegramSourceConfig(BaseModel):
|
|
channel: str = Field(min_length=1)
|
|
limit: int = Field(default=100, ge=1, le=1000)
|
|
# heuristic = legacy telegram_events; llm = DeepSeek per post; profile = static rules
|
|
extract_mode: Literal["heuristic", "llm", "profile"] = "heuristic"
|
|
extract_schema: dict[str, str] | None = None
|
|
instruction: str | None = None
|
|
heuristic_profile: dict | None = None
|
|
# Post-extract gate for extract_mode=llm (empty = only is_event filter)
|
|
required_fields: list[str] = Field(default_factory=list)
|
|
# Split one post into N events when extract_mode=llm (opt-in)
|
|
multi_event: bool = False
|
|
sample_post: str | None = None # audit / re-generate; not required at runtime
|
|
|
|
@field_validator("channel")
|
|
@classmethod
|
|
def strip_channel(cls, value: str) -> str:
|
|
return value.strip()
|
|
|
|
@field_validator("required_fields")
|
|
@classmethod
|
|
def known_required(cls, value: list[str]) -> list[str]:
|
|
from contracts.heuristic_profile import TARGET_FIELDS
|
|
|
|
seen: list[str] = []
|
|
unknown: list[str] = []
|
|
for name in value:
|
|
if name not in TARGET_FIELDS:
|
|
unknown.append(name)
|
|
continue
|
|
if name not in seen:
|
|
seen.append(name)
|
|
if unknown:
|
|
raise ValueError(f"Unknown required_fields: {sorted(set(unknown))}")
|
|
return seen
|
|
|
|
@model_validator(mode="after")
|
|
def validate_extract_mode(self) -> "TelegramSourceConfig":
|
|
if self.extract_mode == "profile":
|
|
if not self.heuristic_profile:
|
|
raise ValueError("heuristic_profile required when extract_mode=profile")
|
|
from contracts.heuristic_profile import HeuristicProfile
|
|
|
|
HeuristicProfile.model_validate(self.heuristic_profile)
|
|
elif self.extract_mode == "llm" and self.extract_schema is not None:
|
|
from contracts.heuristic_profile import TARGET_FIELDS
|
|
|
|
unknown = set(self.extract_schema) - set(TARGET_FIELDS)
|
|
if unknown:
|
|
raise ValueError(f"Unknown extract_schema fields: {sorted(unknown)}")
|
|
return self
|
|
|
|
|
|
class Crawl4AISourceConfig(BaseModel):
|
|
urls: list[str] = Field(min_length=1)
|
|
extract_schema: dict[str, str] = Field(
|
|
default_factory=lambda: {
|
|
"title": "string",
|
|
"locality": "string",
|
|
"event_date": "string",
|
|
"description": "string",
|
|
"coords": "string",
|
|
"topic": "string",
|
|
}
|
|
)
|
|
domain_profile: str = "generic_news"
|
|
extract_mode: Literal["heuristic", "llm"] = "heuristic"
|
|
instruction: str | None = None
|
|
|
|
@field_validator("urls")
|
|
@classmethod
|
|
def non_empty_urls(cls, value: list[str]) -> list[str]:
|
|
cleaned = [u.strip() for u in value if u and u.strip()]
|
|
if not cleaned:
|
|
raise ValueError("urls must contain at least one URL")
|
|
return cleaned
|
|
|
|
|
|
class ViinaSourceConfig(BaseModel):
|
|
urls: list[str] = Field(default_factory=list)
|
|
texts: list[str] = Field(default_factory=list)
|
|
input_mode: Literal["urls", "texts", "mixed"] = "urls"
|
|
|
|
@field_validator("urls")
|
|
@classmethod
|
|
def strip_urls(cls, value: list[str]) -> list[str]:
|
|
return [u.strip() for u in value if u and u.strip()]
|
|
|
|
@field_validator("texts")
|
|
@classmethod
|
|
def strip_texts(cls, value: list[str]) -> list[str]:
|
|
return [t.strip() for t in value if t and t.strip()]
|
|
|
|
@model_validator(mode="after")
|
|
def require_inputs(self) -> "ViinaSourceConfig":
|
|
if self.input_mode == "urls" and not self.urls:
|
|
raise ValueError("urls required when input_mode=urls")
|
|
if self.input_mode == "texts" and not self.texts:
|
|
raise ValueError("texts required when input_mode=texts")
|
|
if self.input_mode == "mixed" and not self.urls and not self.texts:
|
|
raise ValueError("urls or texts required when input_mode=mixed")
|
|
return self
|
|
|
|
|
|
CONFIG_MODELS: dict[str, type[BaseModel]] = {
|
|
"telegram": TelegramSourceConfig,
|
|
"crawl4ai": Crawl4AISourceConfig,
|
|
"viina": ViinaSourceConfig,
|
|
}
|
|
|
|
|
|
def parse_source_config(source_type: str, raw: dict) -> BaseModel:
|
|
model = CONFIG_MODELS.get(source_type)
|
|
if model is None:
|
|
raise ValueError(f"Unknown source_type: {source_type}")
|
|
return model.model_validate(raw or {})
|