Add CP source adapter registry with multi-worker queues and LLM extract.

Replace legacy root backend/frontend with Telegram, Crawl4AI, and VIINA adapters routed by Redis job families.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-08-14 11:34:28 +03:00
co-authored by Cursor
parent 1492576fd9
commit 8fbabd3c11
54 changed files with 1625 additions and 2521 deletions
+14 -2
View File
@@ -37,11 +37,23 @@ from ..services.jobs import enqueue_job
router = APIRouter(prefix="/admin", tags=["admin"])
def _validated_source_config(source_type: str, source_config: dict) -> dict:
try:
from contracts.queues import family_for_source
from contracts.sources import parse_source_config
family_for_source(source_type)
return parse_source_config(source_type, source_config or {}).model_dump()
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.post("/jobs", response_model=ParseJobRead, status_code=201)
def create_parse_job(payload: ParseJobCreate, db: Session = Depends(get_db)):
source_config = _validated_source_config(payload.source_type, payload.source_config)
job = ParseJob(
source_type=payload.source_type,
source_config=payload.source_config,
source_config=source_config,
schedule=payload.schedule,
interval_seconds=payload.interval_seconds,
is_active=payload.is_active,
@@ -96,7 +108,7 @@ def update_parse_job(
raise HTTPException(status_code=404, detail="Job not found")
if payload.source_config is not None:
job.source_config = payload.source_config
job.source_config = _validated_source_config(job.source_type, payload.source_config)
if payload.interval_seconds is not None:
job.interval_seconds = payload.interval_seconds
if payload.is_active is not None: