Add Telegram listener, scheduler, and extended parsers admin UI.

Introduce background scheduling and migrations for analytics ingest, expand parser management and map toolbar UX, and ignore local data/session files from version control.

Co-authored-by: Cursor <cursoragent@cursor.com>

centers/parsing/workers/worker.py

@@ -1,10 +1,9 @@
-"""CP worker: poll Redis jobs, parse Telegram, ingest to CA."""
+"""CP worker: Redis jobs + real-time Telethon listener."""
 
 import asyncio
 import json
 import logging
 import os
-import time
 
 import httpx
 import redis
@@ -17,6 +16,8 @@ from workers.sources.telegram_client import (
     fetch_channel_posts,
     normalize_channel,
 )
+from workers.sources.telegram_listener import TelegramListener
+from workers.sources.telegram_session import close_shared_client, get_shared_client
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
 logger = logging.getLogger("cp-worker")
@@ -26,19 +27,30 @@ JOB_QUEUE_KEY = "cp:jobs"
 CA_API_URL = os.getenv("CA_API_URL", "http://ca-api:8000")
 INTERNAL_TOKEN = os.getenv("INTERNAL_TOKEN", "dev-internal-token")
 POLL_TIMEOUT = int(os.getenv("WORKER_POLL_TIMEOUT", "5"))
+LISTENER_ENABLED = os.getenv("TELEGRAM_LISTENER_ENABLED", "true").lower() not in (
+    "0",
+    "false",
+    "no",
+    "off",
+)
 
 
 def get_redis() -> redis.Redis:
     return redis.from_url(REDIS_URL, decode_responses=True)
 
 
-async def process_telegram_job(job_id: int, source_config: dict) -> tuple[list[dict], str | None]:
+async def process_telegram_job(
+    job_id: int,
+    source_config: dict,
+    *,
+    client=None,
+) -> tuple[list[dict], str | None]:
     channel = source_config.get("channel", "creamy_caprice")
     limit = int(source_config.get("limit", 100))
 
     try:
         username = normalize_channel(channel)
-        posts = await fetch_channel_posts(username, limit=limit)
+        posts = await fetch_channel_posts(username, limit=limit, client=client)
     except (TelegramConfigError, TelegramAuthError, ValueError) as exc:
         return [], str(exc)
     except Exception as exc:
@@ -60,7 +72,14 @@ async def post_ingest(job_id: int, events: list[dict]) -> None:
             headers={"X-Internal-Token": INTERNAL_TOKEN},
         )
         response.raise_for_status()
-        logger.info("Ingested %s events for job %s: %s", len(events), job_id, response.json())
+        result = response.json()
+        logger.info(
+            "Ingested job %s: new=%s skipped=%s map=%s",
+            job_id,
+            result.get("ingested", 0),
+            result.get("skipped", 0),
+            result.get("map_objects_synced", 0),
+        )
 
 
 async def patch_job_status(job_id: int, status: str, error: str | None = None) -> None:
@@ -76,7 +95,7 @@ async def patch_job_status(job_id: int, status: str, error: str | None = None) -
         response.raise_for_status()
 
 
-async def handle_job(payload: dict) -> None:
+async def handle_job(payload: dict, *, tg_client=None) -> None:
     job_id = payload["job_id"]
     source_type = payload["source_type"]
     source_config = payload.get("source_config", {})
@@ -85,7 +104,7 @@ async def handle_job(payload: dict) -> None:
     await patch_job_status(job_id, "running")
 
     if source_type == "telegram":
-        events, error = await process_telegram_job(job_id, source_config)
+        events, error = await process_telegram_job(job_id, source_config, client=tg_client)
     else:
         events, error = [], f"Unsupported source_type: {source_type}"
 
@@ -104,28 +123,54 @@ async def handle_job(payload: dict) -> None:
         await patch_job_status(job_id, "failed", error=str(exc))
 
 
-async def worker_loop() -> None:
+async def worker_loop(*, tg_client=None) -> None:
     r = get_redis()
     logger.info("CP worker started, polling %s", JOB_QUEUE_KEY)
 
     while True:
         try:
-            item = r.blpop(JOB_QUEUE_KEY, timeout=POLL_TIMEOUT)
+            item = await asyncio.to_thread(r.blpop, JOB_QUEUE_KEY, POLL_TIMEOUT)
             if not item:
                 continue
             _, raw = item
             payload = json.loads(raw)
-            await handle_job(payload)
+            await handle_job(payload, tg_client=tg_client)
         except redis.RedisError as exc:
             logger.error("Redis error: %s", exc)
-            time.sleep(3)
+            await asyncio.sleep(3)
         except Exception:
             logger.exception("Unexpected worker error")
-            time.sleep(1)
+            await asyncio.sleep(1)
+
+
+async def run_with_listener() -> None:
+    client = await get_shared_client()
+    listener = TelegramListener(client)
+    worker_task = asyncio.create_task(worker_loop(tg_client=client))
+    listener_task = asyncio.create_task(listener.run())
+
+    logger.info("Telegram listener enabled (shared session with batch worker)")
+    try:
+        await client.run_until_disconnected()
+    finally:
+        worker_task.cancel()
+        listener_task.cancel()
+        await close_shared_client()
+
+
+async def run_batch_only() -> None:
+    logger.info("Telegram listener disabled")
+    await worker_loop(tg_client=None)
 
 
 def main() -> None:
-    asyncio.run(worker_loop())
+    try:
+        if LISTENER_ENABLED:
+            asyncio.run(run_with_listener())
+        else:
+            asyncio.run(run_batch_only())
+    except KeyboardInterrupt:
+        pass
 
 
 if __name__ == "__main__":