Add CP→CA→PI platform foundation on MapMil monorepo.
Unify parsing workers, analytics API with PostgreSQL, map UI, and PI distribution into centers/ with Docker Compose. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Cursor
@@ -0,0 +1,13 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY workers/ ./workers/
|
||||
COPY worker.py .
|
||||
|
||||
ENV PYTHONPATH=/app
|
||||
|
||||
CMD ["python", "worker.py"]
|
||||
@@ -0,0 +1,6 @@
|
||||
httpx==0.28.1
|
||||
redis==5.2.1
|
||||
telethon==1.44.0
|
||||
python-socks[asyncio]==2.7.1
|
||||
beautifulsoup4==4.12.3
|
||||
lxml==5.3.0
|
||||
@@ -0,0 +1,132 @@
|
||||
"""CP worker: poll Redis jobs, parse Telegram, ingest to CA."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
import httpx
|
||||
import redis
|
||||
|
||||
from workers.converter import event_record_to_ingest
|
||||
from workers.parsers.telegram_events import parse_event_posts
|
||||
from workers.sources.telegram_client import (
|
||||
TelegramAuthError,
|
||||
TelegramConfigError,
|
||||
fetch_channel_posts,
|
||||
normalize_channel,
|
||||
)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger("cp-worker")
|
||||
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
||||
JOB_QUEUE_KEY = "cp:jobs"
|
||||
CA_API_URL = os.getenv("CA_API_URL", "http://ca-api:8000")
|
||||
INTERNAL_TOKEN = os.getenv("INTERNAL_TOKEN", "dev-internal-token")
|
||||
POLL_TIMEOUT = int(os.getenv("WORKER_POLL_TIMEOUT", "5"))
|
||||
|
||||
|
||||
def get_redis() -> redis.Redis:
|
||||
return redis.from_url(REDIS_URL, decode_responses=True)
|
||||
|
||||
|
||||
async def process_telegram_job(job_id: int, source_config: dict) -> tuple[list[dict], str | None]:
|
||||
channel = source_config.get("channel", "creamy_caprice")
|
||||
limit = int(source_config.get("limit", 100))
|
||||
|
||||
try:
|
||||
username = normalize_channel(channel)
|
||||
posts = await fetch_channel_posts(username, limit=limit)
|
||||
except (TelegramConfigError, TelegramAuthError, ValueError) as exc:
|
||||
return [], str(exc)
|
||||
except Exception as exc:
|
||||
return [], f"Telegram: {exc}"
|
||||
|
||||
records = parse_event_posts(posts)
|
||||
events = [event_record_to_ingest(r) for r in records]
|
||||
return events, None
|
||||
|
||||
|
||||
async def post_ingest(job_id: int, events: list[dict]) -> None:
|
||||
if not events:
|
||||
return
|
||||
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
response = await client.post(
|
||||
f"{CA_API_URL}/internal/ingest",
|
||||
json={"job_id": job_id, "events": events},
|
||||
headers={"X-Internal-Token": INTERNAL_TOKEN},
|
||||
)
|
||||
response.raise_for_status()
|
||||
logger.info("Ingested %s events for job %s: %s", len(events), job_id, response.json())
|
||||
|
||||
|
||||
async def patch_job_status(job_id: int, status: str, error: str | None = None) -> None:
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
params = {"status": status}
|
||||
if error:
|
||||
params["error"] = error
|
||||
response = await client.patch(
|
||||
f"{CA_API_URL}/internal/jobs/{job_id}",
|
||||
params=params,
|
||||
headers={"X-Internal-Token": INTERNAL_TOKEN},
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
async def handle_job(payload: dict) -> None:
|
||||
job_id = payload["job_id"]
|
||||
source_type = payload["source_type"]
|
||||
source_config = payload.get("source_config", {})
|
||||
|
||||
logger.info("Processing job %s (%s)", job_id, source_type)
|
||||
await patch_job_status(job_id, "running")
|
||||
|
||||
if source_type == "telegram":
|
||||
events, error = await process_telegram_job(job_id, source_config)
|
||||
else:
|
||||
events, error = [], f"Unsupported source_type: {source_type}"
|
||||
|
||||
if error:
|
||||
logger.error("Job %s failed: %s", job_id, error)
|
||||
await patch_job_status(job_id, "failed", error=error)
|
||||
return
|
||||
|
||||
try:
|
||||
await post_ingest(job_id, events)
|
||||
if not events:
|
||||
await patch_job_status(job_id, "completed", error="No events found")
|
||||
logger.info("Job %s completed with %s events", job_id, len(events))
|
||||
except Exception as exc:
|
||||
logger.exception("Ingest failed for job %s", job_id)
|
||||
await patch_job_status(job_id, "failed", error=str(exc))
|
||||
|
||||
|
||||
async def worker_loop() -> None:
|
||||
r = get_redis()
|
||||
logger.info("CP worker started, polling %s", JOB_QUEUE_KEY)
|
||||
|
||||
while True:
|
||||
try:
|
||||
item = r.blpop(JOB_QUEUE_KEY, timeout=POLL_TIMEOUT)
|
||||
if not item:
|
||||
continue
|
||||
_, raw = item
|
||||
payload = json.loads(raw)
|
||||
await handle_job(payload)
|
||||
except redis.RedisError as exc:
|
||||
logger.error("Redis error: %s", exc)
|
||||
time.sleep(3)
|
||||
except Exception:
|
||||
logger.exception("Unexpected worker error")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
asyncio.run(worker_loop())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Convert EventRecord to CA ingest payload items."""
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from workers.parsers.telegram_events import EventRecord
|
||||
|
||||
COORDS_RE = re.compile(
|
||||
r"(-?\d{1,3}\.\d+)\s*,\s*(-?\d{1,3}\.\d+)",
|
||||
)
|
||||
|
||||
|
||||
def parse_event_date(date_str: str) -> datetime | None:
|
||||
for fmt in ("%d.%m.%y", "%d.%m.%Y"):
|
||||
try:
|
||||
dt = datetime.strptime(date_str.strip(), fmt)
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def parse_geolocation(geo: str) -> tuple[float | None, float | None]:
|
||||
if not geo:
|
||||
return None, None
|
||||
match = COORDS_RE.search(geo)
|
||||
if not match:
|
||||
return None, None
|
||||
return float(match.group(1)), float(match.group(2))
|
||||
|
||||
|
||||
def event_record_to_ingest(record: EventRecord) -> dict:
|
||||
lat, lng = parse_geolocation(record.geolocation)
|
||||
title = record.locality or (record.event.splitlines()[0][:120] if record.event else "")
|
||||
event_date = parse_event_date(record.date)
|
||||
return {
|
||||
"source_type": "telegram",
|
||||
"source_url": record.source_url,
|
||||
"raw_text": record.event,
|
||||
"title": title,
|
||||
"description": record.event,
|
||||
"locality": record.locality,
|
||||
"latitude": lat,
|
||||
"longitude": lng,
|
||||
"event_date": event_date.isoformat() if event_date else None,
|
||||
"region": record.locality or None,
|
||||
"topic": "telegram",
|
||||
"tags": ["telegram"],
|
||||
"metadata": {"original_date": record.date, "geolocation": record.geolocation},
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Обход сайта на заданную глубину (BFS)."""
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urljoin, urlparse, urlunparse
|
||||
|
||||
import httpx
|
||||
|
||||
from app.fetcher import fetch_page
|
||||
|
||||
SKIP_EXTENSIONS = (
|
||||
".pdf", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg",
|
||||
".zip", ".rar", ".mp4", ".mp3", ".avi", ".doc", ".docx",
|
||||
".xls", ".xlsx", ".css", ".js", ".xml", ".rss",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlResult:
|
||||
pages: list[tuple[str, str]]
|
||||
errors: list[str]
|
||||
|
||||
|
||||
def _normalize_url(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
path = parsed.path or "/"
|
||||
if path != "/" and path.endswith("/"):
|
||||
path = path.rstrip("/")
|
||||
return urlunparse((parsed.scheme, parsed.netloc.lower(), path, "", parsed.query, ""))
|
||||
|
||||
|
||||
def _same_domain(start: str, candidate: str) -> bool:
|
||||
return urlparse(start).netloc.lower() == urlparse(candidate).netloc.lower()
|
||||
|
||||
|
||||
def _is_fetchable(url: str) -> bool:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
return False
|
||||
path = parsed.path.lower()
|
||||
return not any(path.endswith(ext) for ext in SKIP_EXTENSIONS)
|
||||
|
||||
|
||||
async def crawl_site(
|
||||
start_url: str,
|
||||
steps: int,
|
||||
max_pages: int = 100,
|
||||
timeout: float = 30.0,
|
||||
) -> CrawlResult:
|
||||
"""Обойти сайт от start_url на steps уровней (1 = только стартовая страница)."""
|
||||
steps = max(1, min(steps, 10))
|
||||
max_pages = max(1, min(max_pages, 200))
|
||||
|
||||
start = _normalize_url(start_url)
|
||||
visited: set[str] = set()
|
||||
queue: deque[tuple[str, int]] = deque([(start, 0)])
|
||||
pages: list[tuple[str, str]] = []
|
||||
errors: list[str] = []
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
follow_redirects=True,
|
||||
timeout=timeout,
|
||||
headers={"User-Agent": "SocialParser/1.0"},
|
||||
) as client:
|
||||
while queue and len(visited) < max_pages:
|
||||
url, depth = queue.popleft()
|
||||
if url in visited:
|
||||
continue
|
||||
visited.add(url)
|
||||
|
||||
try:
|
||||
page = await fetch_page(url, client)
|
||||
except Exception as exc:
|
||||
errors.append(f"{url}: {exc}")
|
||||
continue
|
||||
|
||||
pages.append((url, page.text))
|
||||
|
||||
if depth + 1 >= steps:
|
||||
continue
|
||||
|
||||
for link in page.links:
|
||||
normalized = _normalize_url(link)
|
||||
if normalized in visited:
|
||||
continue
|
||||
if not _same_domain(start, normalized):
|
||||
continue
|
||||
if not _is_fetchable(normalized):
|
||||
continue
|
||||
queue.append((normalized, depth + 1))
|
||||
|
||||
return CrawlResult(pages=pages, errors=errors)
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Загрузка содержимого из источников данных."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
NOISE_TAGS = ("script", "style", "noscript", "nav", "header", "footer", "aside", "menu")
|
||||
CONTENT_SELECTORS = (
|
||||
"article",
|
||||
"[role='main']",
|
||||
"main",
|
||||
".article",
|
||||
".content",
|
||||
".post",
|
||||
".entry-content",
|
||||
".news-item",
|
||||
".card-full-news",
|
||||
".box-card",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PageContent:
|
||||
text: str
|
||||
links: list[str]
|
||||
|
||||
|
||||
async def fetch_page(url: str, client: httpx.AsyncClient) -> PageContent:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if "html" not in content_type:
|
||||
return PageContent(text=response.text, links=[])
|
||||
|
||||
return _parse_html(response.text, base_url=str(response.url))
|
||||
|
||||
|
||||
def _parse_html(html: str, base_url: str) -> PageContent:
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
links = _extract_links(soup, base_url)
|
||||
|
||||
for tag in soup.find_all(NOISE_TAGS):
|
||||
tag.decompose()
|
||||
|
||||
chunks: list[str] = []
|
||||
for selector in CONTENT_SELECTORS:
|
||||
for node in soup.select(selector):
|
||||
text = node.get_text(separator=" ", strip=True)
|
||||
if len(text) > 80:
|
||||
chunks.append(text)
|
||||
|
||||
if chunks:
|
||||
return PageContent(text=" ".join(chunks), links=links)
|
||||
|
||||
body = soup.body or soup
|
||||
return PageContent(text=body.get_text(separator=" ", strip=True), links=links)
|
||||
|
||||
|
||||
def _extract_links(soup: BeautifulSoup, base_url: str) -> list[str]:
|
||||
found: list[str] = []
|
||||
seen: set[str] = set()
|
||||
base_host = urlparse(base_url).netloc.lower()
|
||||
|
||||
for tag in soup.find_all("a", href=True):
|
||||
href = tag["href"].strip()
|
||||
if not href or href.startswith(("#", "mailto:", "tel:", "javascript:")):
|
||||
continue
|
||||
|
||||
absolute = urljoin(base_url, href)
|
||||
parsed = urlparse(absolute)
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
continue
|
||||
if parsed.netloc.lower() != base_host:
|
||||
continue
|
||||
|
||||
clean = absolute.split("#", 1)[0]
|
||||
if clean and clean not in seen:
|
||||
seen.add(clean)
|
||||
found.append(clean)
|
||||
|
||||
return found
|
||||
|
||||
|
||||
async def fetch_url(url: str, timeout: float = 30.0) -> str:
|
||||
async with httpx.AsyncClient(
|
||||
follow_redirects=True,
|
||||
timeout=timeout,
|
||||
headers={"User-Agent": "SocialParser/1.0"},
|
||||
) as client:
|
||||
page = await fetch_page(url, client)
|
||||
return page.text
|
||||
|
||||
|
||||
def read_file(path: str) -> str:
|
||||
file_path = Path(path)
|
||||
if not file_path.exists():
|
||||
raise FileNotFoundError(f"Файл не найден: {path}")
|
||||
for encoding in ("utf-8", "cp1251", "latin-1"):
|
||||
try:
|
||||
return file_path.read_text(encoding=encoding)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
raise ValueError(f"Не удалось прочитать файл: {path}")
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Общие модели данных."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class TelegramPost:
|
||||
id: int
|
||||
text: str
|
||||
date: datetime
|
||||
url: str
|
||||
channel: str
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Словари имён и стоп-слов."""
|
||||
|
||||
FIRST_NAMES = frozenset(['александр', 'александра', 'алексей', 'алина', 'алиса', 'альбина', 'анастасия', 'анатолий', 'андрей', 'анжела', 'анна', 'антон', 'антонина', 'аркадий', 'арсений', 'артем', 'артём', 'богдан', 'борис', 'вадим', 'валентин', 'валентина', 'валерий', 'валерия', 'василий', 'вера', 'вероника', 'виктор', 'виктория', 'виталий', 'владимир', 'владислав', 'вова', 'вячеслав', 'галина', 'геннадий', 'георгий', 'григорий', 'даниил', 'данила', 'дарья', 'денис', 'диана', 'дима', 'дмитрий', 'евгений', 'евгения', 'екатерина', 'елена', 'елизавета', 'жанна', 'зоя', 'иван', 'игорь', 'инесса', 'инна', 'ира', 'ирина', 'катя', 'кирилл', 'коля', 'константин', 'ксения', 'лариса', 'леонид', 'лидия', 'лилия', 'любовь', 'людмила', 'максим', 'маргарита', 'марина', 'мария', 'матвей', 'маша', 'милана', 'михаил', 'надежда', 'настя', 'наталья', 'никита', 'николай', 'нина', 'олег', 'ольга', 'оля', 'павел', 'паша', 'петр', 'полина', 'пётр', 'раиса', 'регина', 'рената', 'роман', 'руслан', 'саша', 'света', 'светлана', 'сергей', 'серёжа', 'софия', 'софья', 'станислав', 'степан', 'тамара', 'таня', 'татьяна', 'тимофей', 'тимур', 'фаина', 'федор', 'фёдор', 'эдуард', 'эльвира', 'эмма', 'юлия', 'юрий', 'яна', 'ярослав'])
|
||||
|
||||
FIRST_NAMES_GENITIVE = frozenset(['александра', 'александры', 'алексея', 'анатолия', 'андрея', 'анны', 'артема', 'артёма', 'бориса', 'вадима', 'валерия', 'василия', 'виктора', 'виталия', 'владимира', 'вячеслава', 'геннадия', 'георгия', 'григория', 'данила', 'дмитрия', 'екатерины', 'елены', 'ефима', 'ивана', 'игоря', 'кирилла', 'константина', 'леонида', 'максима', 'марии', 'михаила', 'натальи', 'николая', 'олега', 'ольги', 'павла', 'петра', 'романа', 'руслана', 'сергея', 'станислава', 'степана', 'татьяны', 'тимофея', 'тимура', 'федора', 'филиппа', 'фёдора', 'эдуарда', 'юрия', 'якова', 'ярослава'])
|
||||
|
||||
STOPWORDS = frozenset(['авто', 'архив', 'афера', 'беларуси', 'блогер', 'блогерша', 'блогеры', 'брянской', 'вакансии', 'видео', 'войти', 'время', 'все', 'вчера', 'главная', 'главное', 'год', 'город', 'далее', 'единое', 'журналист', 'журналисты', 'исследования', 'источник', 'источники', 'камчатке', 'комментарии', 'контакты', 'корреспондент', 'криминал', 'круг', 'культура', 'лента', 'лучшее', 'материалы', 'меню', 'место', 'месяц', 'мир', 'москве', 'неделя', 'новости', 'область', 'офис', 'партнеров', 'партнёров', 'подписаться', 'подробнее', 'поиск', 'полиция', 'популярное', 'преступная', 'путешествия', 'район', 'редакция', 'реклама', 'россии', 'россия', 'рубрика', 'рубрики', 'свежее', 'свобода', 'северный', 'сегодня', 'силовые', 'следствие', 'смотреть', 'спецпроекты', 'спорт', 'статьи', 'техподдержка', 'топ', 'украине', 'улица', 'фото', 'ценности', 'читать', 'шурыгина', 'экономика', 'эксклюзивы'])
|
||||
|
||||
PREPOSITIONS = frozenset(['без', 'в', 'для', 'до', 'за', 'из', 'к', 'на', 'над', 'о', 'об', 'от', 'по', 'под', 'при', 'про', 'с', 'у'])
|
||||
|
||||
ADJECTIVE_ENDINGS = ("ый", "ий", "ая", "яя", "ое", "ее", "ие", "ые", "ой", "ей")
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Извлечение ФИО из текста."""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.names import ADJECTIVE_ENDINGS, FIRST_NAMES, FIRST_NAMES_GENITIVE, PREPOSITIONS, STOPWORDS
|
||||
|
||||
PATRONYMIC_SUFFIXES = (
|
||||
"ович", "евич", "овна", "евна", "ична", "инична", "оглы", "кызы",
|
||||
)
|
||||
|
||||
SURNAME_SUFFIXES = (
|
||||
"ов", "ев", "ёв", "ин", "ын", "ский", "ская", "цкий", "цкая",
|
||||
"енко", "ук", "юк", "ко", "ич", "як", "ова", "ева", "ёва", "ина",
|
||||
)
|
||||
|
||||
NON_NAME_ENDINGS = ("ция", "ство", "ение", "ание", "ура", "ика", "ник", "тель", "изм", "ист")
|
||||
|
||||
CYRILLIC_WORD = r"[А-ЯЁ][а-яё]+(?:-[А-ЯЁ][а-яё]+)?"
|
||||
INITIAL = r"[А-ЯЁ]\."
|
||||
|
||||
FULL_FIO_RE = re.compile(
|
||||
rf"\b({CYRILLIC_WORD})\s+({CYRILLIC_WORD})\s+({CYRILLIC_WORD})\b"
|
||||
)
|
||||
SHORT_FIO_RE = re.compile(
|
||||
rf"\b({CYRILLIC_WORD})\s+({INITIAL})\s*({INITIAL})\b"
|
||||
)
|
||||
TWO_WORD_RE = re.compile(
|
||||
rf"\b({CYRILLIC_WORD})\s+({CYRILLIC_WORD})\b"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Person:
|
||||
surname: str
|
||||
name: str
|
||||
patronymic: str
|
||||
source: str
|
||||
raw: str
|
||||
|
||||
@property
|
||||
def full_name(self) -> str:
|
||||
parts = [self.surname, self.name]
|
||||
if self.patronymic:
|
||||
parts.append(self.patronymic)
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _norm(word: str) -> str:
|
||||
return word.lower().replace("ё", "е")
|
||||
|
||||
|
||||
def _looks_like_patronymic(word: str) -> bool:
|
||||
lower = _norm(word)
|
||||
return any(lower.endswith(s.replace("ё", "е")) for s in PATRONYMIC_SUFFIXES)
|
||||
|
||||
|
||||
def _looks_like_surname(word: str) -> bool:
|
||||
lower = _norm(word)
|
||||
if len(lower) < 4:
|
||||
return False
|
||||
return any(lower.endswith(s.replace("ё", "е")) for s in SURNAME_SUFFIXES)
|
||||
|
||||
|
||||
def _looks_like_adjective(word: str) -> bool:
|
||||
if _is_known_first_name(word):
|
||||
return False
|
||||
lower = _norm(word)
|
||||
return any(lower.endswith(e) for e in ADJECTIVE_ENDINGS)
|
||||
|
||||
|
||||
def _is_known_first_name(word: str) -> bool:
|
||||
lower = _norm(word)
|
||||
return lower in FIRST_NAMES or lower in FIRST_NAMES_GENITIVE
|
||||
|
||||
|
||||
def _is_blocked_word(word: str) -> bool:
|
||||
lower = _norm(word)
|
||||
if len(lower) < 2:
|
||||
return True
|
||||
if lower in STOPWORDS or lower in PREPOSITIONS:
|
||||
return True
|
||||
if _looks_like_adjective(word):
|
||||
return True
|
||||
return any(lower.endswith(e) for e in NON_NAME_ENDINGS)
|
||||
|
||||
|
||||
def _is_initial(word: str) -> bool:
|
||||
return len(word) <= 3 and word.rstrip(".").isalpha() and len(word.rstrip(".")) == 1
|
||||
|
||||
|
||||
def _validate_person(surname: str, name: str, patronymic: str) -> bool:
|
||||
parts = [surname, name, patronymic]
|
||||
if any(_is_blocked_word(w) for w in parts if w):
|
||||
return False
|
||||
|
||||
if patronymic:
|
||||
if _is_initial(patronymic) or _is_initial(name):
|
||||
return _looks_like_surname(surname) and not _is_blocked_word(surname)
|
||||
if not _looks_like_patronymic(patronymic):
|
||||
return False
|
||||
if not _is_known_first_name(name):
|
||||
return False
|
||||
if not (_looks_like_surname(surname) or _is_known_first_name(surname)):
|
||||
return False
|
||||
return True
|
||||
|
||||
if _looks_like_patronymic(name):
|
||||
return _looks_like_surname(surname) and not _is_blocked_word(name)
|
||||
|
||||
if not _is_known_first_name(name):
|
||||
return False
|
||||
if not _looks_like_surname(surname):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _parse_full_fio(w1: str, w2: str, w3: str) -> Person | None:
|
||||
if _looks_like_patronymic(w3):
|
||||
person = Person(w1, w2, w3, "", f"{w1} {w2} {w3}")
|
||||
elif _looks_like_patronymic(w2) and _looks_like_surname(w3):
|
||||
person = Person(w3, w1, w2, "", f"{w1} {w2} {w3}")
|
||||
elif _looks_like_surname(w1) and _is_known_first_name(w2) and _looks_like_patronymic(w3):
|
||||
person = Person(w1, w2, w3, "", f"{w1} {w2} {w3}")
|
||||
else:
|
||||
return None
|
||||
|
||||
if _validate_person(person.surname, person.name, person.patronymic):
|
||||
return person
|
||||
return None
|
||||
|
||||
|
||||
def _parse_two_words(w1: str, w2: str) -> Person | None:
|
||||
if _looks_like_patronymic(w1) or _looks_like_patronymic(w2):
|
||||
return None
|
||||
|
||||
if _looks_like_surname(w1) and _is_known_first_name(w2):
|
||||
person = Person(w1, w2, "", "", f"{w1} {w2}")
|
||||
elif _looks_like_surname(w2) and _is_known_first_name(w1):
|
||||
person = Person(w2, w1, "", "", f"{w1} {w2}")
|
||||
else:
|
||||
return None
|
||||
|
||||
if _validate_person(person.surname, person.name, person.patronymic):
|
||||
return person
|
||||
return None
|
||||
|
||||
|
||||
def extract_fio(text: str, source: str = "") -> list[Person]:
|
||||
"""Найти все ФИО в тексте."""
|
||||
seen: set[str] = set()
|
||||
results: list[Person] = []
|
||||
occupied: list[tuple[int, int]] = []
|
||||
|
||||
def overlaps(start: int, end: int) -> bool:
|
||||
return any(not (end <= s or start >= e) for s, e in occupied)
|
||||
|
||||
def add(person: Person, start: int, end: int) -> None:
|
||||
key = person.full_name.lower()
|
||||
if key not in seen and not overlaps(start, end):
|
||||
seen.add(key)
|
||||
occupied.append((start, end))
|
||||
results.append(
|
||||
Person(
|
||||
surname=person.surname,
|
||||
name=person.name,
|
||||
patronymic=person.patronymic,
|
||||
source=source,
|
||||
raw=person.raw,
|
||||
)
|
||||
)
|
||||
|
||||
for match in FULL_FIO_RE.finditer(text):
|
||||
w1, w2, w3 = match.groups()
|
||||
person = _parse_full_fio(w1, w2, w3)
|
||||
if person:
|
||||
add(person, *match.span())
|
||||
|
||||
for match in SHORT_FIO_RE.finditer(text):
|
||||
start, end = match.span()
|
||||
if overlaps(start, end):
|
||||
continue
|
||||
surname, i1, i2 = match.groups()
|
||||
if not _looks_like_surname(surname) or _is_blocked_word(surname):
|
||||
continue
|
||||
person = Person(
|
||||
surname=surname,
|
||||
name=i1,
|
||||
patronymic=i2,
|
||||
source="",
|
||||
raw=match.group(),
|
||||
)
|
||||
if _validate_person(person.surname, person.name, person.patronymic):
|
||||
add(person, start, end)
|
||||
|
||||
for match in TWO_WORD_RE.finditer(text):
|
||||
start, end = match.span()
|
||||
if overlaps(start, end):
|
||||
continue
|
||||
w1, w2 = match.groups()
|
||||
person = _parse_two_words(w1, w2)
|
||||
if person:
|
||||
add(person, start, end)
|
||||
|
||||
return results
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Парсинг событий из постов Telegram-канала."""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
||||
from workers.models import TelegramPost
|
||||
|
||||
DATE_LINE_RE = re.compile(
|
||||
r"^(\d{2}\.\d{2}\.\d{2,4})\s+(.+)$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
COORDS_RE = re.compile(
|
||||
r"(-?\d{1,3}\.\d+)\s*,\s*(-?\d{1,3}\.\d+)",
|
||||
)
|
||||
HASHTAG_LINE_RE = re.compile(r"^#\w+", re.MULTILINE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EventRecord:
|
||||
event: str
|
||||
date: str
|
||||
geolocation: str
|
||||
locality: str
|
||||
source_url: str
|
||||
|
||||
|
||||
def _format_message_date(dt: datetime) -> str:
|
||||
return dt.strftime("%d.%m.%y")
|
||||
|
||||
|
||||
def _clean_event_text(text: str) -> str:
|
||||
lines: list[str] = []
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
if stripped.lower() == "источник":
|
||||
continue
|
||||
if stripped.startswith("#"):
|
||||
continue
|
||||
if COORDS_RE.search(stripped):
|
||||
continue
|
||||
lines.append(stripped)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def parse_event_post(post: TelegramPost) -> EventRecord:
|
||||
text = post.text
|
||||
date = _format_message_date(post.date)
|
||||
locality = ""
|
||||
geolocation = ""
|
||||
event = text
|
||||
|
||||
first_line = text.splitlines()[0].strip() if text.splitlines() else ""
|
||||
header_match = DATE_LINE_RE.match(first_line)
|
||||
if header_match:
|
||||
date = header_match.group(1)
|
||||
locality = header_match.group(2).strip()
|
||||
|
||||
coords_match = COORDS_RE.search(text)
|
||||
if coords_match:
|
||||
geolocation = f"{coords_match.group(1)}, {coords_match.group(2)}"
|
||||
|
||||
if header_match:
|
||||
body = text[len(first_line):].strip()
|
||||
else:
|
||||
body = text
|
||||
|
||||
if coords_match:
|
||||
before, after = body.split(coords_match.group(0), 1)
|
||||
event_body = before.strip()
|
||||
else:
|
||||
event_body = body
|
||||
|
||||
event = _clean_event_text(event_body) or _clean_event_text(text) or text
|
||||
event = HASHTAG_LINE_RE.sub("", event).strip()
|
||||
|
||||
return EventRecord(
|
||||
event=event,
|
||||
date=date,
|
||||
geolocation=geolocation,
|
||||
locality=locality,
|
||||
source_url=post.url,
|
||||
)
|
||||
|
||||
|
||||
def parse_event_posts(posts: list[TelegramPost]) -> list[EventRecord]:
|
||||
return [parse_event_post(post) for post in posts if post.text.strip()]
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Telegram Client API (Telethon) для чтения постов канала."""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from telethon.errors import AuthKeyUnregisteredError, SessionPasswordNeededError
|
||||
|
||||
from workers.models import TelegramPost
|
||||
from workers.sources.telegram_settings import create_client, get_api_credentials
|
||||
|
||||
DEFAULT_SESSION_PATH = "/data/telegram.session"
|
||||
MAX_POSTS = 500
|
||||
|
||||
|
||||
class TelegramConfigError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class TelegramAuthError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _get_config() -> tuple[int, str, str]:
|
||||
api_id, api_hash = get_api_credentials()
|
||||
session_path = os.environ.get("TELEGRAM_SESSION_PATH", DEFAULT_SESSION_PATH)
|
||||
return api_id, api_hash, session_path
|
||||
|
||||
|
||||
def normalize_channel(raw: str) -> str:
|
||||
value = raw.strip()
|
||||
if not value:
|
||||
raise ValueError("Укажите канал Telegram")
|
||||
|
||||
value = value.replace("https://", "").replace("http://", "")
|
||||
value = re.sub(r"^web\.telegram\.org/k/#@", "", value)
|
||||
value = re.sub(r"^t\.me/", "", value)
|
||||
value = value.lstrip("@").split("/")[0].split("?")[0]
|
||||
if not value:
|
||||
raise ValueError("Некорректное имя канала")
|
||||
return value
|
||||
|
||||
|
||||
def _build_post_url(channel: str, message_id: int) -> str:
|
||||
username = channel.lstrip("@")
|
||||
return f"https://t.me/{username}/{message_id}"
|
||||
|
||||
|
||||
async def fetch_channel_posts(channel: str, limit: int = 100) -> list[TelegramPost]:
|
||||
limit = max(1, min(limit, MAX_POSTS))
|
||||
try:
|
||||
api_id, api_hash, session_path = _get_config()
|
||||
except ValueError as exc:
|
||||
raise TelegramConfigError(str(exc)) from exc
|
||||
|
||||
username = normalize_channel(channel)
|
||||
|
||||
if not Path(session_path).exists():
|
||||
raise TelegramAuthError(
|
||||
f"Файл сессии не найден: {session_path}. "
|
||||
"Выполните: python scripts/telegram_auth.py"
|
||||
)
|
||||
|
||||
client = create_client(session_path, api_id, api_hash)
|
||||
posts: list[TelegramPost] = []
|
||||
|
||||
try:
|
||||
await client.connect()
|
||||
if not await client.is_user_authorized():
|
||||
raise TelegramAuthError(
|
||||
"Telegram-сессия не авторизована. Выполните: python scripts/telegram_auth.py"
|
||||
)
|
||||
|
||||
entity = await client.get_entity(username)
|
||||
async for message in client.iter_messages(entity, limit=limit):
|
||||
if not message.text:
|
||||
continue
|
||||
posts.append(
|
||||
TelegramPost(
|
||||
id=message.id,
|
||||
text=message.text.strip(),
|
||||
date=message.date,
|
||||
url=_build_post_url(username, message.id),
|
||||
channel=username,
|
||||
)
|
||||
)
|
||||
except AuthKeyUnregisteredError as exc:
|
||||
raise TelegramAuthError(
|
||||
"Сессия Telegram недействительна. Переавторизуйтесь: python scripts/telegram_auth.py"
|
||||
) from exc
|
||||
except SessionPasswordNeededError as exc:
|
||||
raise TelegramAuthError(
|
||||
"Для аккаунта включена двухфакторная аутентификация. "
|
||||
"Авторизуйтесь через scripts/telegram_auth.py с паролем 2FA."
|
||||
) from exc
|
||||
finally:
|
||||
await client.disconnect()
|
||||
|
||||
return posts
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Общие настройки подключения Telethon."""
|
||||
|
||||
import os
|
||||
|
||||
from telethon import TelegramClient
|
||||
|
||||
DEFAULT_SESSION_PATH = "/data/telegram.session"
|
||||
|
||||
|
||||
def get_session_path() -> str:
|
||||
return os.environ.get("TELEGRAM_SESSION_PATH", DEFAULT_SESSION_PATH)
|
||||
|
||||
|
||||
def get_api_credentials() -> tuple[int, str]:
|
||||
api_id_raw = os.environ.get("TELEGRAM_API_ID", "")
|
||||
api_hash = os.environ.get("TELEGRAM_API_HASH", "")
|
||||
|
||||
if not api_id_raw or not api_hash:
|
||||
raise ValueError(
|
||||
"Не заданы TELEGRAM_API_ID и TELEGRAM_API_HASH в .env"
|
||||
)
|
||||
|
||||
try:
|
||||
api_id = int(api_id_raw)
|
||||
except ValueError as exc:
|
||||
raise ValueError("TELEGRAM_API_ID должен быть числом") from exc
|
||||
|
||||
return api_id, api_hash
|
||||
|
||||
|
||||
def get_proxy() -> tuple | None:
|
||||
proxy_type = os.environ.get("TELEGRAM_PROXY_TYPE", "").strip().lower()
|
||||
if not proxy_type or proxy_type == "none":
|
||||
return None
|
||||
|
||||
host = os.environ.get("TELEGRAM_PROXY_HOST", "127.0.0.1").strip()
|
||||
port = int(os.environ.get("TELEGRAM_PROXY_PORT", "1080"))
|
||||
user = os.environ.get("TELEGRAM_PROXY_USER", "").strip()
|
||||
password = os.environ.get("TELEGRAM_PROXY_PASS", "").strip()
|
||||
|
||||
if proxy_type not in ("socks5", "socks4", "http"):
|
||||
raise ValueError(
|
||||
f"Неподдерживаемый TELEGRAM_PROXY_TYPE={proxy_type!r}. "
|
||||
"Используйте socks5, socks4 или http."
|
||||
)
|
||||
|
||||
if user:
|
||||
return proxy_type, host, port, True, user, password
|
||||
return proxy_type, host, port
|
||||
|
||||
|
||||
def describe_connection() -> str:
|
||||
proxy = get_proxy()
|
||||
if not proxy:
|
||||
return "напрямую (без прокси)"
|
||||
return f"через {proxy[0]}://{proxy[1]}:{proxy[2]}"
|
||||
|
||||
|
||||
def create_client(session_path: str, api_id: int, api_hash: str) -> TelegramClient:
|
||||
kwargs = {
|
||||
"connection_retries": 10,
|
||||
"retry_delay": 3,
|
||||
"timeout": 60,
|
||||
"request_retries": 5,
|
||||
}
|
||||
proxy = get_proxy()
|
||||
if proxy:
|
||||
kwargs["proxy"] = proxy
|
||||
return TelegramClient(session_path, api_id, api_hash, **kwargs)
|
||||
Reference in New Issue
Block a user