Add CP source adapter registry with multi-worker queues and LLM extract.

Replace legacy root backend/frontend with Telegram, Crawl4AI, and VIINA adapters routed by Redis job families.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-08-14 11:34:28 +03:00
co-authored by Cursor
parent 1492576fd9
commit 8fbabd3c11
54 changed files with 1625 additions and 2521 deletions
+32
View File
@@ -0,0 +1,32 @@
"""Redis queue routing for CP adapter families."""
from __future__ import annotations
# source_type → worker family (separate Redis list + Docker image)
SOURCE_FAMILY: dict[str, str] = {
"telegram": "telegram",
"crawl4ai": "web",
"viina": "nlp",
}
LEGACY_JOB_QUEUE_KEY = "cp:jobs"
QUEUE_PREFIX = "cp:jobs"
def family_for_source(source_type: str) -> str:
try:
return SOURCE_FAMILY[source_type]
except KeyError as exc:
raise ValueError(f"Unknown source_type: {source_type}") from exc
def queue_key_for_source(source_type: str) -> str:
return f"{QUEUE_PREFIX}:{family_for_source(source_type)}"
def queue_key_for_family(family: str) -> str:
return f"{QUEUE_PREFIX}:{family}"
def known_source_types() -> list[str]:
return sorted(SOURCE_FAMILY.keys())