Add Telegram listener, scheduler, and extended parsers admin UI.
Introduce background scheduling and migrations for analytics ingest, expand parser management and map toolbar UX, and ignore local data/session files from version control. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Cursor
@@ -4,6 +4,7 @@ import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from telethon import TelegramClient
|
||||
from telethon.errors import AuthKeyUnregisteredError, SessionPasswordNeededError
|
||||
|
||||
from workers.models import TelegramPost
|
||||
@@ -46,54 +47,76 @@ def _build_post_url(channel: str, message_id: int) -> str:
|
||||
return f"https://t.me/{username}/{message_id}"
|
||||
|
||||
|
||||
async def fetch_channel_posts(channel: str, limit: int = 100) -> list[TelegramPost]:
|
||||
limit = max(1, min(limit, MAX_POSTS))
|
||||
try:
|
||||
api_id, api_hash, session_path = _get_config()
|
||||
except ValueError as exc:
|
||||
raise TelegramConfigError(str(exc)) from exc
|
||||
|
||||
def message_to_post(message, channel: str) -> TelegramPost | None:
|
||||
text = (message.text or message.message or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
username = normalize_channel(channel)
|
||||
return TelegramPost(
|
||||
id=message.id,
|
||||
text=text,
|
||||
date=message.date,
|
||||
url=_build_post_url(username, message.id),
|
||||
channel=username,
|
||||
)
|
||||
|
||||
if not Path(session_path).exists():
|
||||
raise TelegramAuthError(
|
||||
f"Файл сессии не найден: {session_path}. "
|
||||
"Выполните: python scripts/telegram_auth.py"
|
||||
)
|
||||
|
||||
client = create_client(session_path, api_id, api_hash)
|
||||
posts: list[TelegramPost] = []
|
||||
async def fetch_channel_posts(
|
||||
channel: str,
|
||||
limit: int = 100,
|
||||
*,
|
||||
client: TelegramClient | None = None,
|
||||
) -> list[TelegramPost]:
|
||||
limit = max(1, min(limit, MAX_POSTS))
|
||||
username = normalize_channel(channel)
|
||||
own_client = client is None
|
||||
|
||||
try:
|
||||
await client.connect()
|
||||
if not await client.is_user_authorized():
|
||||
if own_client:
|
||||
try:
|
||||
api_id, api_hash, session_path = _get_config()
|
||||
except ValueError as exc:
|
||||
raise TelegramConfigError(str(exc)) from exc
|
||||
|
||||
if not Path(session_path).exists():
|
||||
raise TelegramAuthError(
|
||||
"Telegram-сессия не авторизована. Выполните: python scripts/telegram_auth.py"
|
||||
f"Файл сессии не найден: {session_path}. "
|
||||
"Выполните: python scripts/telegram_auth.py"
|
||||
)
|
||||
|
||||
entity = await client.get_entity(username)
|
||||
async for message in client.iter_messages(entity, limit=limit):
|
||||
if not message.text:
|
||||
continue
|
||||
posts.append(
|
||||
TelegramPost(
|
||||
id=message.id,
|
||||
text=message.text.strip(),
|
||||
date=message.date,
|
||||
url=_build_post_url(username, message.id),
|
||||
channel=username,
|
||||
client = create_client(session_path, api_id, api_hash)
|
||||
posts: list[TelegramPost] = []
|
||||
|
||||
try:
|
||||
await client.connect()
|
||||
if not await client.is_user_authorized():
|
||||
raise TelegramAuthError(
|
||||
"Telegram-сессия не авторизована. Выполните: python scripts/telegram_auth.py"
|
||||
)
|
||||
)
|
||||
except AuthKeyUnregisteredError as exc:
|
||||
raise TelegramAuthError(
|
||||
"Сессия Telegram недействительна. Переавторизуйтесь: python scripts/telegram_auth.py"
|
||||
) from exc
|
||||
except SessionPasswordNeededError as exc:
|
||||
raise TelegramAuthError(
|
||||
"Для аккаунта включена двухфакторная аутентификация. "
|
||||
"Авторизуйтесь через scripts/telegram_auth.py с паролем 2FA."
|
||||
) from exc
|
||||
finally:
|
||||
await client.disconnect()
|
||||
|
||||
entity = await client.get_entity(username)
|
||||
async for message in client.iter_messages(entity, limit=limit):
|
||||
post = message_to_post(message, username)
|
||||
if post:
|
||||
posts.append(post)
|
||||
except AuthKeyUnregisteredError as exc:
|
||||
raise TelegramAuthError(
|
||||
"Сессия Telegram недействительна. Переавторизуйтесь: python scripts/telegram_auth.py"
|
||||
) from exc
|
||||
except SessionPasswordNeededError as exc:
|
||||
raise TelegramAuthError(
|
||||
"Для аккаунта включена двухфакторная аутентификация. "
|
||||
"Авторизуйтесь через scripts/telegram_auth.py с паролем 2FA."
|
||||
) from exc
|
||||
finally:
|
||||
await client.disconnect()
|
||||
|
||||
return posts
|
||||
|
||||
assert client is not None
|
||||
posts = []
|
||||
entity = await client.get_entity(username)
|
||||
async for message in client.iter_messages(entity, limit=limit):
|
||||
post = message_to_post(message, username)
|
||||
if post:
|
||||
posts.append(post)
|
||||
return posts
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Real-time Telethon listener for active Telegram parse jobs."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from telethon import TelegramClient, events
|
||||
|
||||
from workers.converter import event_record_to_ingest
|
||||
from workers.parsers.telegram_events import parse_event_post
|
||||
from workers.sources.telegram_client import message_to_post, normalize_channel
|
||||
|
||||
logger = logging.getLogger("cp-listener")
|
||||
|
||||
CA_API_URL = os.getenv("CA_API_URL", "http://ca-api:8000")
|
||||
INTERNAL_TOKEN = os.getenv("INTERNAL_TOKEN", "dev-internal-token")
|
||||
REFRESH_SECONDS = int(os.getenv("TELEGRAM_LISTENER_REFRESH_SECONDS", "60"))
|
||||
|
||||
|
||||
class TelegramListener:
|
||||
def __init__(self, client: TelegramClient) -> None:
|
||||
self.client = client
|
||||
self._channels: dict[str, int] = {}
|
||||
self._chat_ids: set[int] = set()
|
||||
self._handlers_registered = False
|
||||
|
||||
async def fetch_subscriptions(self) -> dict[str, int]:
|
||||
async with httpx.AsyncClient(timeout=30.0) as http:
|
||||
response = await http.get(
|
||||
f"{CA_API_URL}/internal/listener/subscriptions",
|
||||
headers={"X-Internal-Token": INTERNAL_TOKEN},
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
channels: dict[str, int] = {}
|
||||
for item in data:
|
||||
raw = item.get("channel")
|
||||
job_id = item.get("job_id")
|
||||
if not raw or job_id is None:
|
||||
continue
|
||||
try:
|
||||
key = normalize_channel(str(raw))
|
||||
except ValueError:
|
||||
logger.warning("Skip invalid channel in subscription: %r", raw)
|
||||
continue
|
||||
channels.setdefault(key, int(job_id))
|
||||
return channels
|
||||
|
||||
async def refresh_subscriptions(self) -> None:
|
||||
try:
|
||||
channels = await self.fetch_subscriptions()
|
||||
except Exception:
|
||||
logger.exception("Failed to load listener subscriptions")
|
||||
return
|
||||
|
||||
if channels == self._channels:
|
||||
return
|
||||
|
||||
self._channels = channels
|
||||
self._chat_ids = set()
|
||||
for username in channels:
|
||||
try:
|
||||
entity = await self.client.get_entity(username)
|
||||
self._chat_ids.add(entity.id)
|
||||
except Exception:
|
||||
logger.exception("Cannot resolve channel entity: %s", username)
|
||||
|
||||
logger.info(
|
||||
"Listener subscriptions updated: %s",
|
||||
", ".join(sorted(channels)) or "(none)",
|
||||
)
|
||||
|
||||
async def _ingest_post(self, channel: str, post) -> None:
|
||||
record = parse_event_post(post)
|
||||
event = event_record_to_ingest(record)
|
||||
job_id = self._channels.get(normalize_channel(channel))
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as http:
|
||||
response = await http.post(
|
||||
f"{CA_API_URL}/internal/ingest",
|
||||
json={
|
||||
"job_id": job_id,
|
||||
"events": [event],
|
||||
"listener": True,
|
||||
},
|
||||
headers={"X-Internal-Token": INTERNAL_TOKEN},
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
logger.info(
|
||||
"Listener ingest %s: new=%s skipped=%s",
|
||||
post.url,
|
||||
result.get("ingested", 0),
|
||||
result.get("skipped", 0),
|
||||
)
|
||||
|
||||
def _channel_from_event(self, event: events.common.EventCommon) -> str | None:
|
||||
chat = event.chat
|
||||
if chat is None:
|
||||
return None
|
||||
username = getattr(chat, "username", None)
|
||||
if username:
|
||||
return normalize_channel(username)
|
||||
return None
|
||||
|
||||
async def _process_messages(self, event: events.common.EventCommon, messages: list[Any]) -> None:
|
||||
channel = self._channel_from_event(event)
|
||||
if not channel or channel not in self._channels:
|
||||
return
|
||||
if event.chat_id not in self._chat_ids:
|
||||
return
|
||||
|
||||
for message in messages:
|
||||
post = message_to_post(message, channel)
|
||||
if not post:
|
||||
continue
|
||||
try:
|
||||
await self._ingest_post(channel, post)
|
||||
except Exception:
|
||||
logger.exception("Listener ingest failed for %s", post.url)
|
||||
|
||||
def register_handlers(self) -> None:
|
||||
if self._handlers_registered:
|
||||
return
|
||||
|
||||
@self.client.on(events.NewMessage(incoming=True))
|
||||
async def on_new_message(event: events.NewMessage.Event) -> None:
|
||||
if event.grouped_id:
|
||||
return
|
||||
await self._process_messages(event, [event.message])
|
||||
|
||||
@self.client.on(events.Album)
|
||||
async def on_album(event: events.Album.Event) -> None:
|
||||
await self._process_messages(event, list(event.messages))
|
||||
|
||||
self._handlers_registered = True
|
||||
logger.info("Telegram NewMessage/Album handlers registered")
|
||||
|
||||
async def refresh_loop(self) -> None:
|
||||
while True:
|
||||
await self.refresh_subscriptions()
|
||||
await asyncio.sleep(REFRESH_SECONDS)
|
||||
|
||||
async def run(self) -> None:
|
||||
self.register_handlers()
|
||||
await self.refresh_subscriptions()
|
||||
await self.refresh_loop()
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Единое подключение Telethon для listener и batch-заданий."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from telethon import TelegramClient
|
||||
from telethon.errors import AuthKeyUnregisteredError, SessionPasswordNeededError
|
||||
|
||||
from workers.sources.telegram_settings import create_client, get_api_credentials, get_session_path
|
||||
from workers.sources.telegram_client import TelegramAuthError, TelegramConfigError
|
||||
|
||||
_shared_client: TelegramClient | None = None
|
||||
|
||||
|
||||
async def get_shared_client() -> TelegramClient:
|
||||
global _shared_client
|
||||
if _shared_client is not None and _shared_client.is_connected():
|
||||
return _shared_client
|
||||
|
||||
try:
|
||||
api_id, api_hash = get_api_credentials()
|
||||
except ValueError as exc:
|
||||
raise TelegramConfigError(str(exc)) from exc
|
||||
|
||||
session_path = get_session_path()
|
||||
if not Path(session_path).exists():
|
||||
raise TelegramAuthError(
|
||||
f"Файл сессии не найден: {session_path}. "
|
||||
"Выполните: python scripts/telegram_auth.py"
|
||||
)
|
||||
|
||||
client = create_client(session_path, api_id, api_hash)
|
||||
try:
|
||||
await client.connect()
|
||||
if not await client.is_user_authorized():
|
||||
raise TelegramAuthError(
|
||||
"Telegram-сессия не авторизована. Выполните: python scripts/telegram_auth.py"
|
||||
)
|
||||
except AuthKeyUnregisteredError as exc:
|
||||
raise TelegramAuthError(
|
||||
"Сессия Telegram недействительна. Переавторизуйтесь: python scripts/telegram_auth.py"
|
||||
) from exc
|
||||
except SessionPasswordNeededError as exc:
|
||||
raise TelegramAuthError(
|
||||
"Для аккаунта включена 2FA. Авторизуйтесь через scripts/telegram_auth.py."
|
||||
) from exc
|
||||
|
||||
_shared_client = client
|
||||
return client
|
||||
|
||||
|
||||
async def close_shared_client() -> None:
|
||||
global _shared_client
|
||||
if _shared_client is not None:
|
||||
await _shared_client.disconnect()
|
||||
_shared_client = None
|
||||
Reference in New Issue
Block a user