Add CP source adapter registry with multi-worker queues and LLM extract.
Replace legacy root backend/frontend with Telegram, Crawl4AI, and VIINA adapters routed by Redis job families. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -37,11 +37,23 @@ from ..services.jobs import enqueue_job
|
||||
router = APIRouter(prefix="/admin", tags=["admin"])
|
||||
|
||||
|
||||
def _validated_source_config(source_type: str, source_config: dict) -> dict:
|
||||
try:
|
||||
from contracts.queues import family_for_source
|
||||
from contracts.sources import parse_source_config
|
||||
|
||||
family_for_source(source_type)
|
||||
return parse_source_config(source_type, source_config or {}).model_dump()
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/jobs", response_model=ParseJobRead, status_code=201)
|
||||
def create_parse_job(payload: ParseJobCreate, db: Session = Depends(get_db)):
|
||||
source_config = _validated_source_config(payload.source_type, payload.source_config)
|
||||
job = ParseJob(
|
||||
source_type=payload.source_type,
|
||||
source_config=payload.source_config,
|
||||
source_config=source_config,
|
||||
schedule=payload.schedule,
|
||||
interval_seconds=payload.interval_seconds,
|
||||
is_active=payload.is_active,
|
||||
@@ -96,7 +108,7 @@ def update_parse_job(
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
if payload.source_config is not None:
|
||||
job.source_config = payload.source_config
|
||||
job.source_config = _validated_source_config(job.source_type, payload.source_config)
|
||||
if payload.interval_seconds is not None:
|
||||
job.interval_seconds = payload.interval_seconds
|
||||
if payload.is_active is not None:
|
||||
|
||||
Reference in New Issue
Block a user