Add Telegram listener, scheduler, and extended parsers admin UI.

Introduce background scheduling and migrations for analytics ingest, expand parser management and map toolbar UX, and ignore local data/session files from version control.

Co-authored-by: Cursor <cursoragent@cursor.com>

centers/analytics/api/app/services/ingest.py

@@ -9,9 +9,12 @@ def ingest_events(
     db: Session,
     items: list[IngestEventItem],
     job_id: int | None = None,
-) -> tuple[int, int, int]:
+    *,
+    touch_job: bool = True,
+) -> tuple[int, int, int, int]:
     ingested = 0
     updated = 0
+    skipped = 0
     map_synced = 0
 
     for item in items:
@@ -22,44 +25,32 @@ def ingest_events(
         )
 
         if existing:
-            existing.source_type = item.source_type
-            existing.raw_text = item.raw_text
-            existing.title = item.title
-            existing.description = item.description
-            existing.locality = item.locality
-            existing.latitude = item.latitude
-            existing.longitude = item.longitude
-            existing.event_date = item.event_date
-            existing.region = item.region
-            existing.topic = item.topic
-            existing.tags = item.tags
-            existing.metadata_ = item.metadata
-            event = existing
-            updated += 1
-        else:
-            event = Event(
-                source_type=item.source_type,
-                source_url=item.source_url,
-                raw_text=item.raw_text,
-                title=item.title,
-                description=item.description,
-                locality=item.locality,
-                latitude=item.latitude,
-                longitude=item.longitude,
-                event_date=item.event_date,
-                region=item.region,
-                topic=item.topic,
-                tags=item.tags,
-                metadata_=item.metadata,
-            )
-            db.add(event)
-            ingested += 1
+            skipped += 1
+            continue
+
+        event = Event(
+            source_type=item.source_type,
+            source_url=item.source_url,
+            raw_text=item.raw_text,
+            title=item.title,
+            description=item.description,
+            locality=item.locality,
+            latitude=item.latitude,
+            longitude=item.longitude,
+            event_date=item.event_date,
+            region=item.region,
+            topic=item.topic,
+            tags=item.tags,
+            metadata_=item.metadata,
+        )
+        db.add(event)
+        ingested += 1
 
         db.flush()
         if sync_event_to_map_object(db, event):
             map_synced += 1
 
-    if job_id is not None:
+    if touch_job and job_id is not None:
         job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
         if job:
             from datetime import datetime, timezone
@@ -69,4 +60,4 @@ def ingest_events(
             job.last_error = None
 
     db.commit()
-    return ingested, updated, map_synced
+    return ingested, updated, skipped, map_synced

centers/analytics/api/app/services/migrations.py

@@ -0,0 +1,28 @@
+from sqlalchemy import inspect, text
+from sqlalchemy.engine import Engine
+
+
+def migrate_schema(engine: Engine) -> None:
+    """Apply lightweight schema updates for existing deployments."""
+    inspector = inspect(engine)
+    if "parse_jobs" not in inspector.get_table_names():
+        return
+
+    columns = {col["name"] for col in inspector.get_columns("parse_jobs")}
+    statements: list[str] = []
+
+    if "interval_seconds" not in columns:
+        statements.append(
+            "ALTER TABLE parse_jobs ADD COLUMN interval_seconds INTEGER NOT NULL DEFAULT 3600"
+        )
+    if "is_active" not in columns:
+        statements.append(
+            "ALTER TABLE parse_jobs ADD COLUMN is_active BOOLEAN NOT NULL DEFAULT TRUE"
+        )
+
+    if not statements:
+        return
+
+    with engine.begin() as conn:
+        for stmt in statements:
+            conn.execute(text(stmt))

centers/analytics/api/app/services/scheduler.py

@@ -0,0 +1,58 @@
+import logging
+import threading
+import time
+from datetime import datetime, timezone
+
+from ..database import SessionLocal
+from ..models import ParseJob
+from .jobs import enqueue_job
+
+logger = logging.getLogger(__name__)
+
+TICK_SECONDS = 30
+RECURRING_STATUSES = ("completed", "failed")
+
+
+def run_scheduler_tick() -> None:
+    db = SessionLocal()
+    try:
+        now = datetime.now(timezone.utc)
+        jobs = (
+            db.query(ParseJob)
+            .filter(
+                ParseJob.is_active.is_(True),
+                ParseJob.interval_seconds > 0,
+                ParseJob.status.in_(RECURRING_STATUSES),
+            )
+            .all()
+        )
+        for job in jobs:
+            if job.last_run_at is None:
+                continue
+            elapsed = (now - job.last_run_at).total_seconds()
+            if elapsed < job.interval_seconds:
+                continue
+
+            job.status = "queued"
+            job.last_error = None
+            db.commit()
+            enqueue_job(job.id, job.source_type, job.source_config)
+            logger.info("Re-queued recurring job %s (interval %ss)", job.id, job.interval_seconds)
+    except Exception:
+        logger.exception("Scheduler tick failed")
+        db.rollback()
+    finally:
+        db.close()
+
+
+def _scheduler_loop(stop_event: threading.Event) -> None:
+    while not stop_event.wait(TICK_SECONDS):
+        run_scheduler_tick()
+
+
+def start_scheduler() -> threading.Event:
+    stop_event = threading.Event()
+    thread = threading.Thread(target=_scheduler_loop, args=(stop_event,), daemon=True)
+    thread.start()
+    logger.info("Parse job scheduler started (tick every %ss)", TICK_SECONDS)
+    return stop_event