Files
gitrusprusandCursor 3f9dc6643b Add reusable LLM parser profiles with multi-event extract.
Support kind=llm profiles (instruction/schema), optional multi-event posts via #eN URLs, and recover stale running/queued parse jobs after worker crashes.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-13 20:22:38 +03:00

349 lines
9.3 KiB
Python

from datetime import datetime
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
ObjectType = Literal["point", "marker", "zone", "other"]
class MapObjectCreate(BaseModel):
name: str = Field(min_length=1, max_length=255)
description: str = ""
type: ObjectType
latitude: float = Field(ge=-90, le=90)
longitude: float = Field(ge=-180, le=180)
created_at: datetime | None = None
class MapObjectRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
name: str
description: str
type: ObjectType
latitude: float
longitude: float
event_id: int | None = None
created_at: datetime
class MapObjectWithEvent(MapObjectRead):
event_date: datetime | None = None
locality: str | None = None
region: str | None = None
topic: str | None = None
source_type: str | None = None
source_url: str | None = None
title: str | None = None
class MapDateBounds(BaseModel):
min: str | None = None
max: str | None = None
class MapFiltersRead(BaseModel):
regions: list[str]
topics: list[str]
source_types: list[str]
available_dates: list[str]
date_bounds: MapDateBounds
class MapObjectUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=255)
description: str | None = None
type: ObjectType | None = None
latitude: float | None = Field(default=None, ge=-90, le=90)
longitude: float | None = Field(default=None, ge=-180, le=180)
created_at: datetime | None = None
class ObjectMediaRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
object_id: int
original_name: str
content_type: str
size: int
created_at: datetime
url: str
class EventRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
source_type: str
source_url: str
raw_text: str
title: str
description: str
locality: str
latitude: float | None
longitude: float | None
event_date: datetime | None
ingested_at: datetime
region: str | None
topic: str | None
tags: list[str] | None
metadata: dict[str, Any] | None = Field(validation_alias="metadata_")
class EventCreate(BaseModel):
source_type: str = Field(default="manual", min_length=1, max_length=50)
source_url: str = Field(min_length=1, max_length=512)
raw_text: str = ""
title: str = ""
description: str = ""
locality: str = ""
latitude: float | None = Field(default=None, ge=-90, le=90)
longitude: float | None = Field(default=None, ge=-180, le=180)
event_date: datetime | None = None
region: str | None = None
topic: str | None = None
tags: list[str] | None = None
metadata: dict[str, Any] | None = None
class EventUpdate(BaseModel):
source_type: str | None = Field(default=None, min_length=1, max_length=50)
source_url: str | None = Field(default=None, min_length=1, max_length=512)
raw_text: str | None = None
title: str | None = None
description: str | None = None
locality: str | None = None
latitude: float | None = Field(default=None, ge=-90, le=90)
longitude: float | None = Field(default=None, ge=-180, le=180)
event_date: datetime | None = None
region: str | None = None
topic: str | None = None
tags: list[str] | None = None
metadata: dict[str, Any] | None = None
class IngestEventItem(BaseModel):
source_type: str = "telegram"
source_url: str
raw_text: str = ""
title: str = ""
description: str = ""
locality: str = ""
latitude: float | None = None
longitude: float | None = None
event_date: datetime | None = None
region: str | None = None
topic: str | None = None
tags: list[str] | None = None
metadata: dict[str, Any] | None = None
class IngestRequest(BaseModel):
job_id: int | None = None
events: list[IngestEventItem] = Field(default_factory=list)
listener: bool = False
class ListenerSubscription(BaseModel):
job_id: int
channel: str
source_config: dict[str, Any] = Field(default_factory=dict)
class IngestResponse(BaseModel):
ingested: int
updated: int
skipped: int = 0
map_objects_synced: int
class ParserProfileCreate(BaseModel):
name: str = Field(min_length=1, max_length=255)
kind: str = Field(default="heuristic", pattern="^(heuristic|llm)$")
sample_post: str = ""
heuristic_profile: dict[str, Any] | None = None
llm_profile: dict[str, Any] | None = None
status: str | None = None
class ParserProfileUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=255)
kind: str | None = Field(default=None, pattern="^(heuristic|llm)$")
sample_post: str | None = None
heuristic_profile: dict[str, Any] | None = None
llm_profile: dict[str, Any] | None = None
status: str | None = None
class ParserProfileRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
name: str
kind: str = "heuristic"
sample_post: str
heuristic_profile: dict[str, Any] | None
llm_profile: dict[str, Any] | None = None
status: str
created_at: datetime
class ParseChannelCreate(BaseModel):
name: str = Field(min_length=1, max_length=255)
source_type: str = "telegram"
channel: str = Field(min_length=1, max_length=255)
is_active: bool = True
class ParseChannelUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=255)
source_type: str | None = None
channel: str | None = Field(default=None, min_length=1, max_length=255)
is_active: bool | None = None
class ParseChannelRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
name: str
source_type: str
channel: str
is_active: bool
created_at: datetime
class ParseJobCreate(BaseModel):
source_type: str = "telegram"
source_config: dict[str, Any] = Field(default_factory=dict)
profile_id: int | None = None
channel_id: int | None = None
limit: int = Field(default=100, ge=1, le=1000)
schedule: str | None = None
interval_seconds: int = Field(default=3600, ge=60, le=604800)
is_active: bool = True
class ParseJobUpdate(BaseModel):
source_config: dict[str, Any] | None = None
profile_id: int | None = None
channel_id: int | None = None
limit: int | None = Field(default=None, ge=1, le=1000)
interval_seconds: int | None = Field(default=None, ge=60, le=604800)
is_active: bool | None = None
class ParseJobRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
source_type: str
source_config: dict[str, Any]
profile_id: int | None = None
channel_id: int | None = None
schedule: str | None
interval_seconds: int
is_active: bool
status: str
last_run_at: datetime | None
last_error: str | None
created_at: datetime
profile_name: str | None = None
channel_name: str | None = None
channel_handle: str | None = None
class ConsumerCreate(BaseModel):
name: str
regions: list[str] | None = None
topics: list[str] | None = None
date_from: datetime | None = None
class ConsumerUpdate(BaseModel):
name: str | None = None
is_active: bool | None = None
regions: list[str] | None = None
topics: list[str] | None = None
date_from: datetime | None = None
class ConsumerRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
name: str
is_active: bool
created_at: datetime
api_key: str | None = None
regions: list[str] | None = None
topics: list[str] | None = None
date_from: datetime | None = None
class EventListResponse(BaseModel):
items: list[EventRead]
total: int
class AnalyticsSummary(BaseModel):
total_events: int
events_with_coords: int
events_last_24h: int
total_consumers: int
active_consumers: int
total_jobs: int
pending_jobs: int
class TimelinePoint(BaseModel):
date: str
count: int
class TopItem(BaseModel):
name: str
count: int
class VpnSettingsUpdate(BaseModel):
enabled: bool | None = None
mode: Literal["subscription", "socks5", "http"] | None = None
subscription_url: str | None = None
clear_subscription_url: bool | None = None
subscription_interval_seconds: int | None = Field(default=None, ge=60, le=86400)
host: str | None = None
port: int | None = Field(default=None, ge=1, le=65535)
username: str | None = None
password: str | None = None
clear_password: bool | None = None
proxied_source_types: list[str] | None = None
class VpnSettingsAdminRead(BaseModel):
enabled: bool
mode: Literal["subscription", "socks5", "http"]
subscription_url_set: bool
subscription_url_hint: str | None = None
subscription_interval_seconds: int
host: str | None = None
port: int | None = None
username: str | None = None
password_set: bool
proxied_source_types: list[str]
available_source_types: list[str]
class VpnSettingsInternalRead(BaseModel):
enabled: bool
mode: Literal["subscription", "socks5", "http"]
subscription_url: str | None = None
subscription_interval_seconds: int
host: str | None = None
port: int | None = None
username: str | None = None
password: str | None = None
proxied_source_types: list[str]