Add CP source adapter registry with multi-worker queues and LLM extract.

Replace legacy root backend/frontend with Telegram, Crawl4AI, and VIINA adapters routed by Redis job families.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-08-14 11:34:28 +03:00
co-authored by Cursor
parent 1492576fd9
commit 8fbabd3c11
54 changed files with 1625 additions and 2521 deletions
+51 -2
View File
@@ -19,7 +19,9 @@ services:
restart: unless-stopped
ca-api:
build: ./centers/analytics/api
build:
context: .
dockerfile: centers/analytics/api/Dockerfile
environment:
DATABASE_URL: postgresql://mapmil:mapmil@ca-db:5432/mapmil
REDIS_URL: redis://redis:6379/0
@@ -43,13 +45,20 @@ services:
restart: unless-stopped
cp-workers:
build: ./centers/parsing/workers
build:
context: .
dockerfile: centers/parsing/workers/Dockerfile
args:
REQUIREMENTS_FILE: requirements.txt
INSTALL_PLAYWRIGHT: "0"
env_file:
- .env
environment:
TELEGRAM_SESSION_PATH: /data/telegram.session
TELEGRAM_LISTENER_ENABLED: "true"
TELEGRAM_LISTENER_REFRESH_SECONDS: "60"
ENABLED_ADAPTERS: telegram
WORKER_FAMILIES: telegram
CA_API_URL: http://ca-api:8000
REDIS_URL: redis://redis:6379/0
INTERNAL_TOKEN: dev-internal-token
@@ -60,6 +69,46 @@ services:
- ca-api
restart: unless-stopped
cp-workers-web:
build:
context: .
dockerfile: centers/parsing/workers/Dockerfile
args:
REQUIREMENTS_FILE: requirements-web.txt
INSTALL_PLAYWRIGHT: "1"
env_file:
- .env
environment:
TELEGRAM_LISTENER_ENABLED: "false"
ENABLED_ADAPTERS: crawl4ai
WORKER_FAMILIES: web
CA_API_URL: http://ca-api:8000
REDIS_URL: redis://redis:6379/0
INTERNAL_TOKEN: dev-internal-token
depends_on:
- redis
- ca-api
restart: unless-stopped
cp-workers-nlp:
build:
context: .
dockerfile: centers/parsing/workers/Dockerfile
args:
REQUIREMENTS_FILE: requirements-nlp.txt
INSTALL_PLAYWRIGHT: "0"
environment:
TELEGRAM_LISTENER_ENABLED: "false"
ENABLED_ADAPTERS: viina
WORKER_FAMILIES: nlp
CA_API_URL: http://ca-api:8000
REDIS_URL: redis://redis:6379/0
INTERNAL_TOKEN: dev-internal-token
depends_on:
- redis
- ca-api
restart: unless-stopped
volumes:
pgdata:
ca_uploads: