gitrusprus/MapMil
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Telegram listener, scheduler, and extended parsers admin UI.

Browse Source 
Introduce background scheduling and migrations for analytics ingest, expand parser management and map toolbar UX, and ignore local data/session files from version control.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
gitrusprus
2026-07-02 20:47:56 +03:00
co-authored by Cursor
parent 1d6d54ab9f
commit 1492576fd9
23 changed files with 1124 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files
centers/analytics/api/app/main.py
+5
View File
@@ -6,6 +6,8 @@ from fastapi.middleware.cors import CORSMiddleware
from .database import Base, engine, get_db
from .routers import admin, internal, map, objects, v1
from .seed import seed_objects, seed_test_consumer
from .services.migrations import migrate_schema
from .services.scheduler import start_scheduler
from .storage import ensure_upload_dir
@@ -13,6 +15,8 @@ from .storage import ensure_upload_dir
async def lifespan(_: FastAPI):
ensure_upload_dir()
Base.metadata.create_all(bind=engine)
migrate_schema(engine)
scheduler_stop = start_scheduler()
db = next(get_db())
try:
seed_objects(db)
@@ -20,6 +24,7 @@ async def lifespan(_: FastAPI):
finally:
db.close()
yield
scheduler_stop.set()
app = FastAPI(title="CA API (Analytics Center)", lifespan=lifespan)
centers/analytics/api/app/models.py
+2
View File
@@ -100,6 +100,8 @@ class ParseJob(Base):
source_type: Mapped[str] = mapped_column(String(50), nullable=False)
source_config: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
schedule: Mapped[str | None] = mapped_column(String(100), nullable=True)
interval_seconds: Mapped[int] = mapped_column(Integer, default=3600, nullable=False)
is_active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
status: Mapped[str] = mapped_column(String(50), default="pending", index=True)
last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
centers/analytics/api/app/routers/admin.py
+39
View File
@@ -14,6 +14,7 @@ from ..schemas import (
EventRead,
ParseJobCreate,
ParseJobRead,
ParseJobUpdate,
TimelinePoint,
TopItem,
)
@@ -42,6 +43,8 @@ def create_parse_job(payload: ParseJobCreate, db: Session = Depends(get_db)):
source_type=payload.source_type,
source_config=payload.source_config,
schedule=payload.schedule,
interval_seconds=payload.interval_seconds,
is_active=payload.is_active,
status="queued",
)
db.add(job)
@@ -70,6 +73,8 @@ def retry_parse_job(job_id: int, db: Session = Depends(get_db)):
job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
if not job:
raise HTTPException(status_code=404, detail="Job not found")
if job.status in ("queued", "running"):
raise HTTPException(status_code=409, detail="Job is already running or queued")
job.status = "queued"
job.last_error = None
@@ -80,6 +85,40 @@ def retry_parse_job(job_id: int, db: Session = Depends(get_db)):
return job
@router.patch("/jobs/{job_id}", response_model=ParseJobRead)
def update_parse_job(
job_id: int,
payload: ParseJobUpdate,
db: Session = Depends(get_db),
):
job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
if not job:
raise HTTPException(status_code=404, detail="Job not found")
if payload.source_config is not None:
job.source_config = payload.source_config
if payload.interval_seconds is not None:
job.interval_seconds = payload.interval_seconds
if payload.is_active is not None:
job.is_active = payload.is_active
db.commit()
db.refresh(job)
return job
@router.delete("/jobs/{job_id}", status_code=204)
def delete_parse_job(job_id: int, db: Session = Depends(get_db)):
job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
if not job:
raise HTTPException(status_code=404, detail="Job not found")
if job.status == "running":
raise HTTPException(status_code=409, detail="Cannot delete a running job")
db.delete(job)
db.commit()
@router.get("/events", response_model=EventListResponse)
def list_events(
limit: int = Query(default=50, ge=1, le=1000),
centers/analytics/api/app/routers/internal.py
+33 -3
View File
@@ -6,7 +6,7 @@ from sqlalchemy.orm import Session
from ..database import get_db
from ..deps import verify_internal_token
from ..models import ParseJob
from ..schemas import IngestRequest, IngestResponse
from ..schemas import IngestRequest, IngestResponse, ListenerSubscription
from ..services.ingest import ingest_events
router = APIRouter(prefix="/internal", tags=["internal"])
@@ -18,10 +18,16 @@ def internal_ingest(
_: None = Depends(verify_internal_token),
db: Session = Depends(get_db),
):
ingested, updated, map_synced = ingest_events(db, payload.events, payload.job_id)
ingested, updated, skipped, map_synced = ingest_events(
db,
payload.events,
payload.job_id,
touch_job=not payload.listener,
)
return IngestResponse(
ingested=ingested,
updated=updated,
skipped=skipped,
map_objects_synced=map_synced,
)
@@ -39,7 +45,31 @@ def update_job_status(
raise HTTPException(status_code=404, detail="Job not found")
job.status = status
job.last_run_at = datetime.now(timezone.utc)
if status in ("completed", "failed"):
job.last_run_at = datetime.now(timezone.utc)
job.last_error = error
db.commit()
return {"id": job.id, "status": job.status}
@router.get("/listener/subscriptions", response_model=list[ListenerSubscription])
def listener_subscriptions(
_: None = Depends(verify_internal_token),
db: Session = Depends(get_db),
):
jobs = (
db.query(ParseJob)
.filter(
ParseJob.is_active.is_(True),
ParseJob.source_type == "telegram",
)
.order_by(ParseJob.id.asc())
.all()
)
result: list[ListenerSubscription] = []
for job in jobs:
channel = job.source_config.get("channel") if job.source_config else None
if not channel:
continue
result.append(ListenerSubscription(job_id=job.id, channel=str(channel)))
return result
centers/analytics/api/app/schemas.py
+17
View File
@@ -111,11 +111,18 @@ class IngestEventItem(BaseModel):
class IngestRequest(BaseModel):
job_id: int | None = None
events: list[IngestEventItem] = Field(default_factory=list)
listener: bool = False
class ListenerSubscription(BaseModel):
job_id: int
channel: str
class IngestResponse(BaseModel):
ingested: int
updated: int
skipped: int = 0
map_objects_synced: int
@@ -123,6 +130,14 @@ class ParseJobCreate(BaseModel):
source_type: str = "telegram"
source_config: dict[str, Any] = Field(default_factory=dict)
schedule: str | None = None
interval_seconds: int = Field(default=3600, ge=60, le=604800)
is_active: bool = True
class ParseJobUpdate(BaseModel):
source_config: dict[str, Any] | None = None
interval_seconds: int | None = Field(default=None, ge=60, le=604800)
is_active: bool | None = None
class ParseJobRead(BaseModel):
@@ -132,6 +147,8 @@ class ParseJobRead(BaseModel):
source_type: str
source_config: dict[str, Any]
schedule: str | None
interval_seconds: int
is_active: bool
status: str
last_run_at: datetime | None
last_error: str | None
centers/analytics/api/app/services/ingest.py
+26 -35
View File
@@ -9,9 +9,12 @@ def ingest_events(
db: Session,
items: list[IngestEventItem],
job_id: int | None = None,
) -> tuple[int, int, int]:
*,
touch_job: bool = True,
) -> tuple[int, int, int, int]:
ingested = 0
updated = 0
skipped = 0
map_synced = 0
for item in items:
@@ -22,44 +25,32 @@ def ingest_events(
)
if existing:
existing.source_type = item.source_type
existing.raw_text = item.raw_text
existing.title = item.title
existing.description = item.description
existing.locality = item.locality
existing.latitude = item.latitude
existing.longitude = item.longitude
existing.event_date = item.event_date
existing.region = item.region
existing.topic = item.topic
existing.tags = item.tags
existing.metadata_ = item.metadata
event = existing
updated += 1
else:
event = Event(
source_type=item.source_type,
source_url=item.source_url,
raw_text=item.raw_text,
title=item.title,
description=item.description,
locality=item.locality,
latitude=item.latitude,
longitude=item.longitude,
event_date=item.event_date,
region=item.region,
topic=item.topic,
tags=item.tags,
metadata_=item.metadata,
)
db.add(event)
ingested += 1
skipped += 1
continue
event = Event(
source_type=item.source_type,
source_url=item.source_url,
raw_text=item.raw_text,
title=item.title,
description=item.description,
locality=item.locality,
latitude=item.latitude,
longitude=item.longitude,
event_date=item.event_date,
region=item.region,
topic=item.topic,
tags=item.tags,
metadata_=item.metadata,
)
db.add(event)
ingested += 1
db.flush()
if sync_event_to_map_object(db, event):
map_synced += 1
if job_id is not None:
if touch_job and job_id is not None:
job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
if job:
from datetime import datetime, timezone
@@ -69,4 +60,4 @@ def ingest_events(
job.last_error = None
db.commit()
return ingested, updated, map_synced
return ingested, updated, skipped, map_synced
centers/analytics/api/app/services/migrations.py
+28
View File
@@ -0,0 +1,28 @@
from sqlalchemy import inspect, text
from sqlalchemy.engine import Engine
def migrate_schema(engine: Engine) -> None:
"""Apply lightweight schema updates for existing deployments."""
inspector = inspect(engine)
if "parse_jobs" not in inspector.get_table_names():
return
columns = {col["name"] for col in inspector.get_columns("parse_jobs")}
statements: list[str] = []
if "interval_seconds" not in columns:
statements.append(
"ALTER TABLE parse_jobs ADD COLUMN interval_seconds INTEGER NOT NULL DEFAULT 3600"
)
if "is_active" not in columns:
statements.append(
"ALTER TABLE parse_jobs ADD COLUMN is_active BOOLEAN NOT NULL DEFAULT TRUE"
)
if not statements:
return
with engine.begin() as conn:
for stmt in statements:
conn.execute(text(stmt))
centers/analytics/api/app/services/scheduler.py
+58
View File
@@ -0,0 +1,58 @@
import logging
import threading
import time
from datetime import datetime, timezone
from ..database import SessionLocal
from ..models import ParseJob
from .jobs import enqueue_job
logger = logging.getLogger(__name__)
TICK_SECONDS = 30
RECURRING_STATUSES = ("completed", "failed")
def run_scheduler_tick() -> None:
db = SessionLocal()
try:
now = datetime.now(timezone.utc)
jobs = (
db.query(ParseJob)
.filter(
ParseJob.is_active.is_(True),
ParseJob.interval_seconds > 0,
ParseJob.status.in_(RECURRING_STATUSES),
)
.all()
)
for job in jobs:
if job.last_run_at is None:
continue
elapsed = (now - job.last_run_at).total_seconds()
if elapsed < job.interval_seconds:
continue
job.status = "queued"
job.last_error = None
db.commit()
enqueue_job(job.id, job.source_type, job.source_config)
logger.info("Re-queued recurring job %s (interval %ss)", job.id, job.interval_seconds)
except Exception:
logger.exception("Scheduler tick failed")
db.rollback()
finally:
db.close()
def _scheduler_loop(stop_event: threading.Event) -> None:
while not stop_event.wait(TICK_SECONDS):
run_scheduler_tick()
def start_scheduler() -> threading.Event:
stop_event = threading.Event()
thread = threading.Thread(target=_scheduler_loop, args=(stop_event,), daemon=True)
thread.start()
logger.info("Parse job scheduler started (tick every %ss)", TICK_SECONDS)
return stop_event