Add CP source adapter registry with multi-worker queues and LLM extract.
Replace legacy root backend/frontend with Telegram, Crawl4AI, and VIINA adapters routed by Redis job families. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Cursor
@@ -0,0 +1,5 @@
|
||||
"""CP source adapters (telegram, crawl4ai, viina, …)."""
|
||||
|
||||
from workers.adapters.registry import AdapterRegistry, build_registry
|
||||
|
||||
__all__ = ["AdapterRegistry", "build_registry"]
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Source adapter protocol and shared worker context."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkerContext:
|
||||
"""Runtime deps shared by adapters (optional Telegram client, etc.)."""
|
||||
|
||||
tg_client: Any | None = None
|
||||
extras: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class SourceAdapter(Protocol):
|
||||
source_type: str
|
||||
|
||||
async def run(
|
||||
self,
|
||||
job_id: int,
|
||||
source_config: dict,
|
||||
*,
|
||||
ctx: WorkerContext,
|
||||
) -> tuple[list[dict], str | None]:
|
||||
"""Return ingest-ready dicts (IngestEventItem-shaped) or an error string."""
|
||||
...
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Crawl4AI web adapter: crawl URLs → map fields → ingest events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from contracts.sources import Crawl4AISourceConfig
|
||||
from workers.adapters.base import WorkerContext
|
||||
from workers.llm_extract import (
|
||||
DEFAULT_INSTRUCTION,
|
||||
crawl4ai_llm_config,
|
||||
extract_event_fields,
|
||||
fields_to_ingest,
|
||||
llm_enabled,
|
||||
parse_coords,
|
||||
parse_date,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("cp-worker.crawl4ai")
|
||||
|
||||
COORDS_RE = re.compile(r"(-?\d{1,3}\.\d+)\s*,\s*(-?\d{1,3}\.\d+)")
|
||||
DATE_RE = re.compile(
|
||||
r"\b(\d{1,2}[./]\d{1,2}[./]\d{2,4}|\d{4}-\d{2}-\d{2})\b",
|
||||
)
|
||||
|
||||
|
||||
class Crawl4AIAdapter:
|
||||
source_type = "crawl4ai"
|
||||
|
||||
async def run(
|
||||
self,
|
||||
job_id: int,
|
||||
source_config: dict,
|
||||
*,
|
||||
ctx: WorkerContext,
|
||||
) -> tuple[list[dict], str | None]:
|
||||
try:
|
||||
cfg = Crawl4AISourceConfig.model_validate(source_config or {})
|
||||
except Exception as exc:
|
||||
return [], f"Invalid crawl4ai source_config: {exc}"
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler # type: ignore
|
||||
except ImportError:
|
||||
return [], (
|
||||
"crawl4ai is not installed in this worker image. "
|
||||
"Use cp-workers-web (ENABLED_ADAPTERS=crawl4ai)."
|
||||
)
|
||||
|
||||
if cfg.extract_mode == "llm" and not llm_enabled():
|
||||
return [], "extract_mode=llm requires DEEPSEEK_API_KEY in worker env"
|
||||
|
||||
events: list[dict] = []
|
||||
errors: list[str] = []
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
for url in cfg.urls:
|
||||
try:
|
||||
if cfg.extract_mode == "llm":
|
||||
item, err = await _crawl_with_llm(crawler, url, cfg)
|
||||
else:
|
||||
item, err = await _crawl_heuristic(crawler, url, cfg)
|
||||
if err:
|
||||
errors.append(err)
|
||||
if item:
|
||||
events.append(item)
|
||||
except Exception as exc:
|
||||
logger.exception("Crawl4AI failed for %s", url)
|
||||
errors.append(f"{url}: {exc}")
|
||||
|
||||
if not events and errors:
|
||||
return [], "; ".join(errors)
|
||||
return events, None
|
||||
|
||||
|
||||
async def _crawl_heuristic(crawler: Any, url: str, cfg: Crawl4AISourceConfig):
|
||||
result = await crawler.arun(url=url)
|
||||
markdown = _result_markdown(result)
|
||||
if not markdown:
|
||||
return None, f"{url}: empty crawl result"
|
||||
return (
|
||||
_markdown_to_ingest(
|
||||
url=url,
|
||||
markdown=markdown,
|
||||
extract_schema=cfg.extract_schema,
|
||||
domain_profile=cfg.domain_profile,
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
async def _crawl_with_llm(crawler: Any, url: str, cfg: Crawl4AISourceConfig):
|
||||
"""Prefer Crawl4AI LLMExtractionStrategy; fall back to DeepSeek on markdown."""
|
||||
instruction = cfg.instruction or DEFAULT_INSTRUCTION
|
||||
try:
|
||||
from crawl4ai import CacheMode, CrawlerRunConfig # type: ignore
|
||||
from crawl4ai import LLMExtractionStrategy # type: ignore
|
||||
|
||||
strategy = LLMExtractionStrategy(
|
||||
llm_config=crawl4ai_llm_config(),
|
||||
schema=_pydantic_like_schema(cfg.extract_schema),
|
||||
extraction_type="schema",
|
||||
instruction=instruction,
|
||||
input_format="markdown",
|
||||
apply_chunking=False,
|
||||
extra_args={"temperature": 0.1, "max_tokens": 1200},
|
||||
)
|
||||
run_config = CrawlerRunConfig(
|
||||
extraction_strategy=strategy,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
)
|
||||
result = await crawler.arun(url=url, config=run_config)
|
||||
markdown = _result_markdown(result)
|
||||
fields = _parse_extracted_content(getattr(result, "extracted_content", None))
|
||||
if fields:
|
||||
return (
|
||||
fields_to_ingest(
|
||||
source_type="crawl4ai",
|
||||
source_url=url,
|
||||
raw_text=markdown or json.dumps(fields, ensure_ascii=False),
|
||||
fields=fields,
|
||||
domain_profile=cfg.domain_profile,
|
||||
extra_metadata={"extractor": "crawl4ai_llm"},
|
||||
),
|
||||
None,
|
||||
)
|
||||
if markdown:
|
||||
# Fallback: same DeepSeek path as Telegram
|
||||
fields = await extract_event_fields(
|
||||
markdown,
|
||||
extract_schema=cfg.extract_schema,
|
||||
instruction=instruction,
|
||||
)
|
||||
if fields.get("is_event", True):
|
||||
return (
|
||||
fields_to_ingest(
|
||||
source_type="crawl4ai",
|
||||
source_url=url,
|
||||
raw_text=markdown,
|
||||
fields=fields,
|
||||
domain_profile=cfg.domain_profile,
|
||||
extra_metadata={"extractor": "deepseek_fallback"},
|
||||
),
|
||||
None,
|
||||
)
|
||||
return None, f"{url}: LLM marked as non-event"
|
||||
return None, f"{url}: empty LLM extraction"
|
||||
except Exception as exc:
|
||||
logger.warning("Crawl4AI LLMStrategy failed for %s: %s; trying DeepSeek on markdown", url, exc)
|
||||
result = await crawler.arun(url=url)
|
||||
markdown = _result_markdown(result)
|
||||
if not markdown:
|
||||
return None, f"{url}: empty crawl result ({exc})"
|
||||
fields = await extract_event_fields(
|
||||
markdown,
|
||||
extract_schema=cfg.extract_schema,
|
||||
instruction=instruction,
|
||||
)
|
||||
if not fields.get("is_event", True):
|
||||
return None, None
|
||||
return (
|
||||
fields_to_ingest(
|
||||
source_type="crawl4ai",
|
||||
source_url=url,
|
||||
raw_text=markdown,
|
||||
fields=fields,
|
||||
domain_profile=cfg.domain_profile,
|
||||
extra_metadata={"extractor": "deepseek_fallback", "llm_strategy_error": str(exc)},
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _pydantic_like_schema(extract_schema: dict[str, str]) -> dict:
|
||||
properties = {
|
||||
key: {"type": "string", "description": desc}
|
||||
for key, desc in extract_schema.items()
|
||||
}
|
||||
return {
|
||||
"title": "MapEvent",
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"required": list(extract_schema.keys()),
|
||||
}
|
||||
|
||||
|
||||
def _parse_extracted_content(raw: Any) -> dict[str, Any]:
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
if isinstance(data, list) and data:
|
||||
data = data[0]
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
# Unwrap common nesting
|
||||
if isinstance(data.get("fields"), dict):
|
||||
data = data["fields"]
|
||||
return {k: str(v).strip() if v is not None else "" for k, v in data.items()}
|
||||
|
||||
|
||||
def _result_markdown(result: Any) -> str:
|
||||
markdown = getattr(result, "markdown", None) or ""
|
||||
if hasattr(markdown, "raw_markdown"):
|
||||
return str(markdown.raw_markdown or "")
|
||||
if isinstance(markdown, str):
|
||||
return markdown
|
||||
fit = getattr(result, "fit_markdown", None)
|
||||
return str(fit or "")
|
||||
|
||||
|
||||
def _markdown_to_ingest(
|
||||
*,
|
||||
url: str,
|
||||
markdown: str,
|
||||
extract_schema: dict[str, str],
|
||||
domain_profile: str,
|
||||
) -> dict:
|
||||
fields = _extract_fields(markdown, extract_schema)
|
||||
lat, lng = parse_coords(fields.get("coords", ""))
|
||||
description = fields.get("description") or markdown[:4000]
|
||||
locality = fields.get("locality") or ""
|
||||
title = fields.get("title") or locality or _title_from_url(url) or description.splitlines()[0][:120]
|
||||
event_date = parse_date(fields.get("event_date", ""))
|
||||
|
||||
return {
|
||||
"source_type": "crawl4ai",
|
||||
"source_url": url,
|
||||
"raw_text": markdown[:20000],
|
||||
"title": title[:255],
|
||||
"description": description[:8000],
|
||||
"locality": locality,
|
||||
"latitude": lat,
|
||||
"longitude": lng,
|
||||
"event_date": event_date.isoformat() if event_date else None,
|
||||
"region": locality or None,
|
||||
"topic": fields.get("topic") or domain_profile,
|
||||
"tags": ["crawl4ai", domain_profile],
|
||||
"metadata": {
|
||||
"domain_profile": domain_profile,
|
||||
"extract_schema": extract_schema,
|
||||
"extracted": fields,
|
||||
"extract_mode": "heuristic",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _extract_fields(markdown: str, schema: dict[str, str]) -> dict[str, str]:
|
||||
fields: dict[str, str] = {}
|
||||
for key in schema:
|
||||
if key == "coords":
|
||||
match = COORDS_RE.search(markdown)
|
||||
fields[key] = match.group(0) if match else ""
|
||||
elif key == "event_date":
|
||||
match = DATE_RE.search(markdown)
|
||||
fields[key] = match.group(1) if match else ""
|
||||
elif key == "description":
|
||||
fields[key] = markdown.strip()[:4000]
|
||||
elif key == "locality":
|
||||
fields[key] = _guess_locality(markdown)
|
||||
elif key == "title":
|
||||
fields[key] = _guess_locality(markdown)
|
||||
else:
|
||||
fields[key] = ""
|
||||
return fields
|
||||
|
||||
|
||||
def _guess_locality(markdown: str) -> str:
|
||||
for line in markdown.splitlines():
|
||||
stripped = line.strip().lstrip("#").strip()
|
||||
if 2 <= len(stripped) <= 80 and not COORDS_RE.search(stripped):
|
||||
return stripped
|
||||
return ""
|
||||
|
||||
|
||||
def _title_from_url(url: str) -> str:
|
||||
path = urlparse(url).path.rstrip("/")
|
||||
if not path:
|
||||
return urlparse(url).netloc
|
||||
return path.rsplit("/", 1)[-1].replace("-", " ").replace("_", " ")
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Registry of source adapters enabled for this worker process."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Callable
|
||||
|
||||
from workers.adapters.base import SourceAdapter
|
||||
|
||||
logger = logging.getLogger("cp-worker.adapters")
|
||||
|
||||
AdapterFactory = Callable[[], SourceAdapter]
|
||||
|
||||
|
||||
class AdapterRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._adapters: dict[str, SourceAdapter] = {}
|
||||
|
||||
def register(self, adapter: SourceAdapter) -> None:
|
||||
self._adapters[adapter.source_type] = adapter
|
||||
logger.info("Registered adapter: %s", adapter.source_type)
|
||||
|
||||
def get(self, source_type: str) -> SourceAdapter | None:
|
||||
return self._adapters.get(source_type)
|
||||
|
||||
def enabled_types(self) -> list[str]:
|
||||
return sorted(self._adapters.keys())
|
||||
|
||||
def __contains__(self, source_type: str) -> bool:
|
||||
return source_type in self._adapters
|
||||
|
||||
|
||||
def _parse_enabled_adapters() -> set[str] | None:
|
||||
raw = os.getenv("ENABLED_ADAPTERS", "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
return {part.strip() for part in raw.split(",") if part.strip()}
|
||||
|
||||
|
||||
def _factories() -> dict[str, AdapterFactory]:
|
||||
# Lazy imports so telegram-only / web-only / nlp-only images do not need all deps
|
||||
def telegram() -> SourceAdapter:
|
||||
from workers.adapters.telegram import TelegramAdapter
|
||||
|
||||
return TelegramAdapter()
|
||||
|
||||
def crawl4ai() -> SourceAdapter:
|
||||
from workers.adapters.crawl4ai_adapter import Crawl4AIAdapter
|
||||
|
||||
return Crawl4AIAdapter()
|
||||
|
||||
def viina() -> SourceAdapter:
|
||||
from workers.adapters.viina import ViinaAdapter
|
||||
|
||||
return ViinaAdapter()
|
||||
|
||||
return {
|
||||
"telegram": telegram,
|
||||
"crawl4ai": crawl4ai,
|
||||
"viina": viina,
|
||||
}
|
||||
|
||||
|
||||
def build_registry() -> AdapterRegistry:
|
||||
"""Register adapters filtered by ENABLED_ADAPTERS (comma-separated).
|
||||
|
||||
Empty ENABLED_ADAPTERS → attempt to load every known adapter; skip import failures.
|
||||
"""
|
||||
enabled = _parse_enabled_adapters()
|
||||
factories = _factories()
|
||||
names = sorted(enabled) if enabled is not None else sorted(factories)
|
||||
|
||||
registry = AdapterRegistry()
|
||||
for name in names:
|
||||
factory = factories.get(name)
|
||||
if factory is None:
|
||||
logger.warning("Unknown adapter in ENABLED_ADAPTERS: %s", name)
|
||||
continue
|
||||
try:
|
||||
registry.register(factory())
|
||||
except Exception:
|
||||
logger.exception("Failed to load adapter %s", name)
|
||||
|
||||
if not registry.enabled_types():
|
||||
logger.warning("No adapters enabled (ENABLED_ADAPTERS=%r)", os.getenv("ENABLED_ADAPTERS"))
|
||||
else:
|
||||
logger.info("Enabled adapters: %s", ", ".join(registry.enabled_types()))
|
||||
return registry
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Telegram batch adapter (Telethon history → ingest events)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from contracts.sources import TelegramSourceConfig
|
||||
from workers.adapters.base import WorkerContext
|
||||
from workers.converter import event_record_to_ingest
|
||||
from workers.llm_extract import (
|
||||
DEFAULT_EXTRACT_SCHEMA,
|
||||
DEFAULT_INSTRUCTION,
|
||||
extract_event_fields,
|
||||
fields_to_ingest,
|
||||
llm_enabled,
|
||||
)
|
||||
from workers.parsers.telegram_events import parse_event_posts
|
||||
from workers.sources.telegram_client import (
|
||||
TelegramAuthError,
|
||||
TelegramConfigError,
|
||||
fetch_channel_posts,
|
||||
normalize_channel,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("cp-worker.telegram")
|
||||
|
||||
|
||||
class TelegramAdapter:
|
||||
source_type = "telegram"
|
||||
|
||||
async def run(
|
||||
self,
|
||||
job_id: int,
|
||||
source_config: dict,
|
||||
*,
|
||||
ctx: WorkerContext,
|
||||
) -> tuple[list[dict], str | None]:
|
||||
try:
|
||||
cfg = TelegramSourceConfig.model_validate(source_config or {})
|
||||
except Exception as exc:
|
||||
return [], f"Invalid telegram source_config: {exc}"
|
||||
|
||||
try:
|
||||
username = normalize_channel(cfg.channel)
|
||||
posts = await fetch_channel_posts(
|
||||
username,
|
||||
limit=cfg.limit,
|
||||
client=ctx.tg_client,
|
||||
)
|
||||
except (TelegramConfigError, TelegramAuthError, ValueError) as exc:
|
||||
return [], str(exc)
|
||||
except Exception as exc:
|
||||
return [], f"Telegram: {exc}"
|
||||
|
||||
if cfg.extract_mode == "llm":
|
||||
if not llm_enabled():
|
||||
return [], "extract_mode=llm requires DEEPSEEK_API_KEY in worker env"
|
||||
return await _extract_posts_with_llm(posts, cfg)
|
||||
|
||||
records = parse_event_posts(posts)
|
||||
events = [event_record_to_ingest(r) for r in records]
|
||||
return events, None
|
||||
|
||||
|
||||
async def _extract_posts_with_llm(posts, cfg: TelegramSourceConfig) -> tuple[list[dict], str | None]:
|
||||
schema = cfg.extract_schema or DEFAULT_EXTRACT_SCHEMA
|
||||
instruction = cfg.instruction or DEFAULT_INSTRUCTION
|
||||
events: list[dict] = []
|
||||
errors: list[str] = []
|
||||
|
||||
for post in posts:
|
||||
text = (post.text or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
try:
|
||||
fields = await extract_event_fields(
|
||||
text,
|
||||
extract_schema=schema,
|
||||
instruction=instruction,
|
||||
)
|
||||
if not fields.get("is_event", True):
|
||||
continue
|
||||
events.append(
|
||||
fields_to_ingest(
|
||||
source_type="telegram",
|
||||
source_url=post.url,
|
||||
raw_text=text,
|
||||
fields=fields,
|
||||
domain_profile="telegram_llm",
|
||||
extra_metadata={
|
||||
"channel": post.channel,
|
||||
"message_id": post.id,
|
||||
"post_date": post.date.isoformat() if post.date else None,
|
||||
},
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("LLM extract failed for %s", post.url)
|
||||
errors.append(f"{post.url}: {exc}")
|
||||
|
||||
if not events and errors:
|
||||
return [], "; ".join(errors[:5])
|
||||
return events, None
|
||||
@@ -0,0 +1,155 @@
|
||||
"""VIINA-style news incident adapter: fetch/text → extract → ingest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from contracts.sources import ViinaSourceConfig
|
||||
from workers.adapters.base import WorkerContext
|
||||
|
||||
logger = logging.getLogger("cp-worker.viina")
|
||||
|
||||
COORDS_RE = re.compile(r"(-?\d{1,3}\.\d+)\s*,\s*(-?\d{1,3}\.\d+)")
|
||||
DATE_RE = re.compile(
|
||||
r"\b(\d{1,2}[./]\d{1,2}[./]\d{2,4}|\d{4}-\d{2}-\d{2})\b",
|
||||
)
|
||||
# Lightweight incident cues inspired by VIINA-style violent-event coding
|
||||
INCIDENT_CUES = re.compile(
|
||||
r"\b(attack|shelling|strike|explosion|casualty|killed|wounded|"
|
||||
r"обстрел|удар|взрыв|погибли|ранены|атака)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
class ViinaAdapter:
|
||||
source_type = "viina"
|
||||
|
||||
async def run(
|
||||
self,
|
||||
job_id: int,
|
||||
source_config: dict,
|
||||
*,
|
||||
ctx: WorkerContext,
|
||||
) -> tuple[list[dict], str | None]:
|
||||
try:
|
||||
cfg = ViinaSourceConfig.model_validate(source_config or {})
|
||||
except Exception as exc:
|
||||
return [], f"Invalid viina source_config: {exc}"
|
||||
|
||||
articles: list[tuple[str, str]] = []
|
||||
errors: list[str] = []
|
||||
|
||||
if cfg.input_mode in ("urls", "mixed"):
|
||||
for url in cfg.urls:
|
||||
try:
|
||||
text = await _fetch_article_text(url)
|
||||
if text:
|
||||
articles.append((url, text))
|
||||
else:
|
||||
errors.append(f"{url}: empty article")
|
||||
except Exception as exc:
|
||||
logger.exception("VIINA fetch failed for %s", url)
|
||||
errors.append(f"{url}: {exc}")
|
||||
|
||||
if cfg.input_mode in ("texts", "mixed"):
|
||||
for idx, text in enumerate(cfg.texts):
|
||||
articles.append((f"viina:text:{job_id}:{idx}", text))
|
||||
|
||||
events = [_article_to_ingest(source_url, text) for source_url, text in articles]
|
||||
# Keep articles without strong cues — still useful raw intelligence
|
||||
if not events and errors:
|
||||
return [], "; ".join(errors)
|
||||
return events, None
|
||||
|
||||
|
||||
async def _fetch_article_text(url: str) -> str:
|
||||
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
|
||||
response = await client.get(
|
||||
url,
|
||||
headers={"User-Agent": "MapMil-CP-Viina/1.0"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if "html" in content_type or url.endswith(".html"):
|
||||
return _html_to_text(response.text)
|
||||
return response.text.strip()
|
||||
|
||||
|
||||
def _html_to_text(html: str) -> str:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
for tag in soup(["script", "style", "noscript", "nav", "footer", "header"]):
|
||||
tag.decompose()
|
||||
article = soup.find("article") or soup.find("main") or soup.body
|
||||
if article is None:
|
||||
return soup.get_text("\n", strip=True)
|
||||
return article.get_text("\n", strip=True)
|
||||
|
||||
|
||||
def _article_to_ingest(source_url: str, text: str) -> dict[str, Any]:
|
||||
lat, lng = _parse_coords(text)
|
||||
date_match = DATE_RE.search(text)
|
||||
event_date = _parse_date(date_match.group(1) if date_match else "")
|
||||
cue = INCIDENT_CUES.search(text)
|
||||
title = next((ln.strip() for ln in text.splitlines() if ln.strip()), source_url)[:120]
|
||||
locality = _guess_locality(text)
|
||||
|
||||
return {
|
||||
"source_type": "viina",
|
||||
"source_url": source_url,
|
||||
"raw_text": text[:20000],
|
||||
"title": title,
|
||||
"description": text[:8000],
|
||||
"locality": locality,
|
||||
"latitude": lat,
|
||||
"longitude": lng,
|
||||
"event_date": event_date.isoformat() if event_date else None,
|
||||
"region": locality or None,
|
||||
"topic": "violent_incident" if cue else "news",
|
||||
"tags": ["viina", "news"] + ([cue.group(0).lower()] if cue else []),
|
||||
"metadata": {
|
||||
"extractor": "viina_heuristic",
|
||||
"incident_cue": cue.group(0) if cue else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _parse_coords(raw: str) -> tuple[float | None, float | None]:
|
||||
match = COORDS_RE.search(raw or "")
|
||||
if not match:
|
||||
return None, None
|
||||
return float(match.group(1)), float(match.group(2))
|
||||
|
||||
|
||||
def _parse_date(raw: str) -> datetime | None:
|
||||
if not raw:
|
||||
return None
|
||||
raw = raw.strip()
|
||||
for fmt in ("%d.%m.%Y", "%d.%m.%y", "%d/%m/%Y", "%d/%m/%y", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(raw, fmt).replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
continue
|
||||
try:
|
||||
dt = parsedate_to_datetime(raw)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
except (TypeError, ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def _guess_locality(text: str) -> str:
|
||||
for line in text.splitlines()[:15]:
|
||||
stripped = line.strip()
|
||||
if 2 <= len(stripped) <= 60 and not DATE_RE.search(stripped):
|
||||
if INCIDENT_CUES.search(stripped):
|
||||
continue
|
||||
return stripped
|
||||
return ""
|
||||
@@ -0,0 +1,179 @@
|
||||
"""DeepSeek / OpenAI-compatible LLM extraction for unstructured text."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger("cp-worker.llm")
|
||||
|
||||
COORDS_RE = re.compile(r"(-?\d{1,3}\.\d+)\s*,\s*(-?\d{1,3}\.\d+)")
|
||||
|
||||
DEFAULT_EXTRACT_SCHEMA: dict[str, str] = {
|
||||
"title": "string — short event title",
|
||||
"locality": "string — place / settlement name",
|
||||
"event_date": "string — date as DD.MM.YYYY or YYYY-MM-DD if known",
|
||||
"description": "string — concise event summary",
|
||||
"coords": "string — latitude, longitude if present else empty",
|
||||
"topic": "string — short topic tag",
|
||||
}
|
||||
|
||||
DEFAULT_INSTRUCTION = (
|
||||
"Extract structured military/news event fields from the text. "
|
||||
"If the text is not an event, return is_event=false. "
|
||||
"Respond with a single JSON object only."
|
||||
)
|
||||
|
||||
|
||||
def llm_enabled() -> bool:
|
||||
return bool(os.getenv("DEEPSEEK_API_KEY", "").strip())
|
||||
|
||||
|
||||
def llm_settings() -> dict[str, str]:
|
||||
return {
|
||||
"api_key": os.getenv("DEEPSEEK_API_KEY", "").strip(),
|
||||
"base_url": os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com").rstrip("/"),
|
||||
"model": os.getenv("DEEPSEEK_MODEL", "deepseek-chat"),
|
||||
}
|
||||
|
||||
|
||||
async def extract_event_fields(
|
||||
text: str,
|
||||
*,
|
||||
extract_schema: dict[str, str] | None = None,
|
||||
instruction: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Ask DeepSeek to fill schema fields from free text. Returns dict (+ is_event)."""
|
||||
settings = llm_settings()
|
||||
if not settings["api_key"]:
|
||||
raise RuntimeError(
|
||||
"DEEPSEEK_API_KEY is not set. Add it to .env for LLM extract_mode."
|
||||
)
|
||||
|
||||
schema = extract_schema or DEFAULT_EXTRACT_SCHEMA
|
||||
instr = instruction or DEFAULT_INSTRUCTION
|
||||
schema_lines = "\n".join(f"- {k}: {v}" for k, v in schema.items())
|
||||
user_prompt = (
|
||||
f"{instr}\n\n"
|
||||
f"Fields to extract:\n{schema_lines}\n\n"
|
||||
'Return JSON: {"is_event": true|false, "fields": {<field>: <string>}}\n\n'
|
||||
f"Text:\n{text[:12000]}"
|
||||
)
|
||||
|
||||
payload = {
|
||||
"model": settings["model"],
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You extract structured event data for a geoint map. "
|
||||
"Output valid JSON only, no markdown."
|
||||
),
|
||||
},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
"temperature": 0.1,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
|
||||
url = f"{settings['base_url']}/chat/completions"
|
||||
async with httpx.AsyncClient(timeout=90.0) as client:
|
||||
response = await client.post(
|
||||
url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {settings['api_key']}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json=payload,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
content = data["choices"][0]["message"]["content"]
|
||||
parsed = json.loads(content)
|
||||
fields = parsed.get("fields") if isinstance(parsed.get("fields"), dict) else parsed
|
||||
if not isinstance(fields, dict):
|
||||
fields = {}
|
||||
# Normalize to strings for known keys
|
||||
result = {key: str(fields.get(key) or "").strip() for key in schema}
|
||||
result["is_event"] = bool(parsed.get("is_event", True))
|
||||
return result
|
||||
|
||||
|
||||
def fields_to_ingest(
|
||||
*,
|
||||
source_type: str,
|
||||
source_url: str,
|
||||
raw_text: str,
|
||||
fields: dict[str, Any],
|
||||
domain_profile: str = "llm",
|
||||
extra_metadata: dict | None = None,
|
||||
) -> dict:
|
||||
lat, lng = parse_coords(str(fields.get("coords") or ""))
|
||||
description = str(fields.get("description") or raw_text)[:8000]
|
||||
locality = str(fields.get("locality") or "")
|
||||
title = str(fields.get("title") or locality or description.splitlines()[0][:120])
|
||||
topic = str(fields.get("topic") or domain_profile)
|
||||
event_date = parse_date(str(fields.get("event_date") or ""))
|
||||
|
||||
meta = {
|
||||
"extract_mode": "llm",
|
||||
"extracted": {k: fields.get(k) for k in fields if k != "is_event"},
|
||||
}
|
||||
if extra_metadata:
|
||||
meta.update(extra_metadata)
|
||||
|
||||
return {
|
||||
"source_type": source_type,
|
||||
"source_url": source_url,
|
||||
"raw_text": raw_text[:20000],
|
||||
"title": title[:255],
|
||||
"description": description,
|
||||
"locality": locality,
|
||||
"latitude": lat,
|
||||
"longitude": lng,
|
||||
"event_date": event_date.isoformat() if event_date else None,
|
||||
"region": locality or None,
|
||||
"topic": topic,
|
||||
"tags": [source_type, "llm", domain_profile],
|
||||
"metadata": meta,
|
||||
}
|
||||
|
||||
|
||||
def parse_coords(raw: str) -> tuple[float | None, float | None]:
|
||||
match = COORDS_RE.search(raw or "")
|
||||
if not match:
|
||||
return None, None
|
||||
return float(match.group(1)), float(match.group(2))
|
||||
|
||||
|
||||
def parse_date(raw: str) -> datetime | None:
|
||||
if not raw:
|
||||
return None
|
||||
raw = raw.strip()
|
||||
for fmt in ("%d.%m.%Y", "%d.%m.%y", "%d/%m/%Y", "%d/%m/%y", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(raw, fmt).replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def crawl4ai_llm_config():
|
||||
"""Build Crawl4AI LLMConfig for DeepSeek (OpenAI-compatible)."""
|
||||
from crawl4ai import LLMConfig # type: ignore
|
||||
|
||||
settings = llm_settings()
|
||||
if not settings["api_key"]:
|
||||
raise RuntimeError("DEEPSEEK_API_KEY is not set")
|
||||
return LLMConfig(
|
||||
provider=f"openai/{settings['model']}",
|
||||
api_token=settings["api_key"],
|
||||
base_url=settings["base_url"],
|
||||
)
|
||||
Reference in New Issue
Block a user