Add Telegram listener, scheduler, and extended parsers admin UI.

Introduce background scheduling and migrations for analytics ingest, expand parser management and map toolbar UX, and ignore local data/session files from version control. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-02 20:47:56 +03:00
parent 1d6d54ab9f
commit 1492576fd9
23 changed files with 1124 additions and 267 deletions
@@ -1,10 +1,9 @@
-"""CP worker: poll Redis jobs, parse Telegram, ingest to CA."""
+"""CP worker: Redis jobs + real-time Telethon listener."""

 import asyncio
 import json
 import logging
 import os
-import time

 import httpx
 import redis
@@ -17,6 +16,8 @@ from workers.sources.telegram_client import (
    fetch_channel_posts,
    normalize_channel,
 )
+from workers.sources.telegram_listener import TelegramListener
+from workers.sources.telegram_session import close_shared_client, get_shared_client

 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 logger = logging.getLogger("cp-worker")
@@ -26,19 +27,30 @@ JOB_QUEUE_KEY = "cp:jobs"
 CA_API_URL = os.getenv("CA_API_URL", "http://ca-api:8000")
 INTERNAL_TOKEN = os.getenv("INTERNAL_TOKEN", "dev-internal-token")
 POLL_TIMEOUT = int(os.getenv("WORKER_POLL_TIMEOUT", "5"))
+LISTENER_ENABLED = os.getenv("TELEGRAM_LISTENER_ENABLED", "true").lower() not in (
+    "0",
+    "false",
+    "no",
+    "off",
+)


 def get_redis() -> redis.Redis:
    return redis.from_url(REDIS_URL, decode_responses=True)


-async def process_telegram_job(job_id: int, source_config: dict) -> tuple[list[dict], str | None]:
+async def process_telegram_job(
+    job_id: int,
+    source_config: dict,
+    *,
+    client=None,
+) -> tuple[list[dict], str | None]:
    channel = source_config.get("channel", "creamy_caprice")
    limit = int(source_config.get("limit", 100))

    try:
        username = normalize_channel(channel)
-        posts = await fetch_channel_posts(username, limit=limit)
+        posts = await fetch_channel_posts(username, limit=limit, client=client)
    except (TelegramConfigError, TelegramAuthError, ValueError) as exc:
        return [], str(exc)
    except Exception as exc:
@@ -60,7 +72,14 @@ async def post_ingest(job_id: int, events: list[dict]) -> None:
            headers={"X-Internal-Token": INTERNAL_TOKEN},
        )
        response.raise_for_status()
-        logger.info("Ingested %s events for job %s: %s", len(events), job_id, response.json())
+        result = response.json()
+        logger.info(
+            "Ingested job %s: new=%s skipped=%s map=%s",
+            job_id,
+            result.get("ingested", 0),
+            result.get("skipped", 0),
+            result.get("map_objects_synced", 0),
+        )


 async def patch_job_status(job_id: int, status: str, error: str | None = None) -> None:
@@ -76,7 +95,7 @@ async def patch_job_status(job_id: int, status: str, error: str | None = None) -
        response.raise_for_status()


-async def handle_job(payload: dict) -> None:
+async def handle_job(payload: dict, *, tg_client=None) -> None:
    job_id = payload["job_id"]
    source_type = payload["source_type"]
    source_config = payload.get("source_config", {})
@@ -85,7 +104,7 @@ async def handle_job(payload: dict) -> None:
    await patch_job_status(job_id, "running")

    if source_type == "telegram":
-        events, error = await process_telegram_job(job_id, source_config)
+        events, error = await process_telegram_job(job_id, source_config, client=tg_client)
    else:
        events, error = [], f"Unsupported source_type: {source_type}"

@@ -104,28 +123,54 @@ async def handle_job(payload: dict) -> None:
        await patch_job_status(job_id, "failed", error=str(exc))


-async def worker_loop() -> None:
+async def worker_loop(*, tg_client=None) -> None:
    r = get_redis()
    logger.info("CP worker started, polling %s", JOB_QUEUE_KEY)

    while True:
        try:
-            item = r.blpop(JOB_QUEUE_KEY, timeout=POLL_TIMEOUT)
+            item = await asyncio.to_thread(r.blpop, JOB_QUEUE_KEY, POLL_TIMEOUT)
            if not item:
                continue
            _, raw = item
            payload = json.loads(raw)
-            await handle_job(payload)
+            await handle_job(payload, tg_client=tg_client)
        except redis.RedisError as exc:
            logger.error("Redis error: %s", exc)
-            time.sleep(3)
+            await asyncio.sleep(3)
        except Exception:
            logger.exception("Unexpected worker error")
-            time.sleep(1)
+            await asyncio.sleep(1)
+
+
+async def run_with_listener() -> None:
+    client = await get_shared_client()
+    listener = TelegramListener(client)
+    worker_task = asyncio.create_task(worker_loop(tg_client=client))
+    listener_task = asyncio.create_task(listener.run())
+
+    logger.info("Telegram listener enabled (shared session with batch worker)")
+    try:
+        await client.run_until_disconnected()
+    finally:
+        worker_task.cancel()
+        listener_task.cancel()
+        await close_shared_client()
+
+
+async def run_batch_only() -> None:
+    logger.info("Telegram listener disabled")
+    await worker_loop(tg_client=None)


 def main() -> None:
-    asyncio.run(worker_loop())
+    try:
+        if LISTENER_ENABLED:
+            asyncio.run(run_with_listener())
+        else:
+            asyncio.run(run_batch_only())
+    except KeyboardInterrupt:
+        pass


 if __name__ == "__main__":
@@ -4,6 +4,7 @@ import os
 import re
 from pathlib import Path

+from telethon import TelegramClient
 from telethon.errors import AuthKeyUnregisteredError, SessionPasswordNeededError

 from workers.models import TelegramPost
@@ -46,54 +47,76 @@ def _build_post_url(channel: str, message_id: int) -> str:
    return f"https://t.me/{username}/{message_id}"


-async def fetch_channel_posts(channel: str, limit: int = 100) -> list[TelegramPost]:
-    limit = max(1, min(limit, MAX_POSTS))
-    try:
-        api_id, api_hash, session_path = _get_config()
-    except ValueError as exc:
-        raise TelegramConfigError(str(exc)) from exc
-
+def message_to_post(message, channel: str) -> TelegramPost | None:
+    text = (message.text or message.message or "").strip()
+    if not text:
+        return None
    username = normalize_channel(channel)
+    return TelegramPost(
+        id=message.id,
+        text=text,
+        date=message.date,
+        url=_build_post_url(username, message.id),
+        channel=username,
+    )

-    if not Path(session_path).exists():
-        raise TelegramAuthError(
-            f"Файл сессии не найден: {session_path}. "
-            "Выполните: python scripts/telegram_auth.py"
-        )

-    client = create_client(session_path, api_id, api_hash)
-    posts: list[TelegramPost] = []
+async def fetch_channel_posts(
+    channel: str,
+    limit: int = 100,
+    *,
+    client: TelegramClient | None = None,
+) -> list[TelegramPost]:
+    limit = max(1, min(limit, MAX_POSTS))
+    username = normalize_channel(channel)
+    own_client = client is None

-    try:
-        await client.connect()
-        if not await client.is_user_authorized():
+    if own_client:
+        try:
+            api_id, api_hash, session_path = _get_config()
+        except ValueError as exc:
+            raise TelegramConfigError(str(exc)) from exc
+
+        if not Path(session_path).exists():
            raise TelegramAuthError(
-                "Telegram-сессия не авторизована. Выполните: python scripts/telegram_auth.py"
+                f"Файл сессии не найден: {session_path}. "
+                "Выполните: python scripts/telegram_auth.py"
            )

-        entity = await client.get_entity(username)
-        async for message in client.iter_messages(entity, limit=limit):
-            if not message.text:
-                continue
-            posts.append(
-                TelegramPost(
-                    id=message.id,
-                    text=message.text.strip(),
-                    date=message.date,
-                    url=_build_post_url(username, message.id),
-                    channel=username,
+        client = create_client(session_path, api_id, api_hash)
+        posts: list[TelegramPost] = []
+
+        try:
+            await client.connect()
+            if not await client.is_user_authorized():
+                raise TelegramAuthError(
+                    "Telegram-сессия не авторизована. Выполните: python scripts/telegram_auth.py"
                )
-            )
-    except AuthKeyUnregisteredError as exc:
-        raise TelegramAuthError(
-            "Сессия Telegram недействительна. Переавторизуйтесь: python scripts/telegram_auth.py"
-        ) from exc
-    except SessionPasswordNeededError as exc:
-        raise TelegramAuthError(
-            "Для аккаунта включена двухфакторная аутентификация. "
-            "Авторизуйтесь через scripts/telegram_auth.py с паролем 2FA."
-        ) from exc
-    finally:
-        await client.disconnect()

+            entity = await client.get_entity(username)
+            async for message in client.iter_messages(entity, limit=limit):
+                post = message_to_post(message, username)
+                if post:
+                    posts.append(post)
+        except AuthKeyUnregisteredError as exc:
+            raise TelegramAuthError(
+                "Сессия Telegram недействительна. Переавторизуйтесь: python scripts/telegram_auth.py"
+            ) from exc
+        except SessionPasswordNeededError as exc:
+            raise TelegramAuthError(
+                "Для аккаунта включена двухфакторная аутентификация. "
+                "Авторизуйтесь через scripts/telegram_auth.py с паролем 2FA."
+            ) from exc
+        finally:
+            await client.disconnect()
+
+        return posts
+
+    assert client is not None
+    posts = []
+    entity = await client.get_entity(username)
+    async for message in client.iter_messages(entity, limit=limit):
+        post = message_to_post(message, username)
+        if post:
+            posts.append(post)
    return posts
@@ -0,0 +1,150 @@
+"""Real-time Telethon listener for active Telegram parse jobs."""
+
+import asyncio
+import logging
+import os
+from typing import Any
+
+import httpx
+from telethon import TelegramClient, events
+
+from workers.converter import event_record_to_ingest
+from workers.parsers.telegram_events import parse_event_post
+from workers.sources.telegram_client import message_to_post, normalize_channel
+
+logger = logging.getLogger("cp-listener")
+
+CA_API_URL = os.getenv("CA_API_URL", "http://ca-api:8000")
+INTERNAL_TOKEN = os.getenv("INTERNAL_TOKEN", "dev-internal-token")
+REFRESH_SECONDS = int(os.getenv("TELEGRAM_LISTENER_REFRESH_SECONDS", "60"))
+
+
+class TelegramListener:
+    def __init__(self, client: TelegramClient) -> None:
+        self.client = client
+        self._channels: dict[str, int] = {}
+        self._chat_ids: set[int] = set()
+        self._handlers_registered = False
+
+    async def fetch_subscriptions(self) -> dict[str, int]:
+        async with httpx.AsyncClient(timeout=30.0) as http:
+            response = await http.get(
+                f"{CA_API_URL}/internal/listener/subscriptions",
+                headers={"X-Internal-Token": INTERNAL_TOKEN},
+            )
+            response.raise_for_status()
+            data = response.json()
+
+        channels: dict[str, int] = {}
+        for item in data:
+            raw = item.get("channel")
+            job_id = item.get("job_id")
+            if not raw or job_id is None:
+                continue
+            try:
+                key = normalize_channel(str(raw))
+            except ValueError:
+                logger.warning("Skip invalid channel in subscription: %r", raw)
+                continue
+            channels.setdefault(key, int(job_id))
+        return channels
+
+    async def refresh_subscriptions(self) -> None:
+        try:
+            channels = await self.fetch_subscriptions()
+        except Exception:
+            logger.exception("Failed to load listener subscriptions")
+            return
+
+        if channels == self._channels:
+            return
+
+        self._channels = channels
+        self._chat_ids = set()
+        for username in channels:
+            try:
+                entity = await self.client.get_entity(username)
+                self._chat_ids.add(entity.id)
+            except Exception:
+                logger.exception("Cannot resolve channel entity: %s", username)
+
+        logger.info(
+            "Listener subscriptions updated: %s",
+            ", ".join(sorted(channels)) or "(none)",
+        )
+
+    async def _ingest_post(self, channel: str, post) -> None:
+        record = parse_event_post(post)
+        event = event_record_to_ingest(record)
+        job_id = self._channels.get(normalize_channel(channel))
+
+        async with httpx.AsyncClient(timeout=60.0) as http:
+            response = await http.post(
+                f"{CA_API_URL}/internal/ingest",
+                json={
+                    "job_id": job_id,
+                    "events": [event],
+                    "listener": True,
+                },
+                headers={"X-Internal-Token": INTERNAL_TOKEN},
+            )
+            response.raise_for_status()
+            result = response.json()
+            logger.info(
+                "Listener ingest %s: new=%s skipped=%s",
+                post.url,
+                result.get("ingested", 0),
+                result.get("skipped", 0),
+            )
+
+    def _channel_from_event(self, event: events.common.EventCommon) -> str | None:
+        chat = event.chat
+        if chat is None:
+            return None
+        username = getattr(chat, "username", None)
+        if username:
+            return normalize_channel(username)
+        return None
+
+    async def _process_messages(self, event: events.common.EventCommon, messages: list[Any]) -> None:
+        channel = self._channel_from_event(event)
+        if not channel or channel not in self._channels:
+            return
+        if event.chat_id not in self._chat_ids:
+            return
+
+        for message in messages:
+            post = message_to_post(message, channel)
+            if not post:
+                continue
+            try:
+                await self._ingest_post(channel, post)
+            except Exception:
+                logger.exception("Listener ingest failed for %s", post.url)
+
+    def register_handlers(self) -> None:
+        if self._handlers_registered:
+            return
+
+        @self.client.on(events.NewMessage(incoming=True))
+        async def on_new_message(event: events.NewMessage.Event) -> None:
+            if event.grouped_id:
+                return
+            await self._process_messages(event, [event.message])
+
+        @self.client.on(events.Album)
+        async def on_album(event: events.Album.Event) -> None:
+            await self._process_messages(event, list(event.messages))
+
+        self._handlers_registered = True
+        logger.info("Telegram NewMessage/Album handlers registered")
+
+    async def refresh_loop(self) -> None:
+        while True:
+            await self.refresh_subscriptions()
+            await asyncio.sleep(REFRESH_SECONDS)
+
+    async def run(self) -> None:
+        self.register_handlers()
+        await self.refresh_subscriptions()
+        await self.refresh_loop()
@@ -0,0 +1,55 @@
+"""Единое подключение Telethon для listener и batch-заданий."""
+
+from pathlib import Path
+
+from telethon import TelegramClient
+from telethon.errors import AuthKeyUnregisteredError, SessionPasswordNeededError
+
+from workers.sources.telegram_settings import create_client, get_api_credentials, get_session_path
+from workers.sources.telegram_client import TelegramAuthError, TelegramConfigError
+
+_shared_client: TelegramClient | None = None
+
+
+async def get_shared_client() -> TelegramClient:
+    global _shared_client
+    if _shared_client is not None and _shared_client.is_connected():
+        return _shared_client
+
+    try:
+        api_id, api_hash = get_api_credentials()
+    except ValueError as exc:
+        raise TelegramConfigError(str(exc)) from exc
+
+    session_path = get_session_path()
+    if not Path(session_path).exists():
+        raise TelegramAuthError(
+            f"Файл сессии не найден: {session_path}. "
+            "Выполните: python scripts/telegram_auth.py"
+        )
+
+    client = create_client(session_path, api_id, api_hash)
+    try:
+        await client.connect()
+        if not await client.is_user_authorized():
+            raise TelegramAuthError(
+                "Telegram-сессия не авторизована. Выполните: python scripts/telegram_auth.py"
+            )
+    except AuthKeyUnregisteredError as exc:
+        raise TelegramAuthError(
+            "Сессия Telegram недействительна. Переавторизуйтесь: python scripts/telegram_auth.py"
+        ) from exc
+    except SessionPasswordNeededError as exc:
+        raise TelegramAuthError(
+            "Для аккаунта включена 2FA. Авторизуйтесь через scripts/telegram_auth.py."
+        ) from exc
+
+    _shared_client = client
+    return client
+
+
+async def close_shared_client() -> None:
+    global _shared_client
+    if _shared_client is not None:
+        await _shared_client.disconnect()
+        _shared_client = None