Add CP source adapter registry with multi-worker queues and LLM extract.
Replace legacy root backend/frontend with Telegram, Crawl4AI, and VIINA adapters routed by Redis job families. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+51
-2
@@ -19,7 +19,9 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
ca-api:
|
||||
build: ./centers/analytics/api
|
||||
build:
|
||||
context: .
|
||||
dockerfile: centers/analytics/api/Dockerfile
|
||||
environment:
|
||||
DATABASE_URL: postgresql://mapmil:mapmil@ca-db:5432/mapmil
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
@@ -43,13 +45,20 @@ services:
|
||||
restart: unless-stopped
|
||||
|
||||
cp-workers:
|
||||
build: ./centers/parsing/workers
|
||||
build:
|
||||
context: .
|
||||
dockerfile: centers/parsing/workers/Dockerfile
|
||||
args:
|
||||
REQUIREMENTS_FILE: requirements.txt
|
||||
INSTALL_PLAYWRIGHT: "0"
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
TELEGRAM_SESSION_PATH: /data/telegram.session
|
||||
TELEGRAM_LISTENER_ENABLED: "true"
|
||||
TELEGRAM_LISTENER_REFRESH_SECONDS: "60"
|
||||
ENABLED_ADAPTERS: telegram
|
||||
WORKER_FAMILIES: telegram
|
||||
CA_API_URL: http://ca-api:8000
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
INTERNAL_TOKEN: dev-internal-token
|
||||
@@ -60,6 +69,46 @@ services:
|
||||
- ca-api
|
||||
restart: unless-stopped
|
||||
|
||||
cp-workers-web:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: centers/parsing/workers/Dockerfile
|
||||
args:
|
||||
REQUIREMENTS_FILE: requirements-web.txt
|
||||
INSTALL_PLAYWRIGHT: "1"
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
TELEGRAM_LISTENER_ENABLED: "false"
|
||||
ENABLED_ADAPTERS: crawl4ai
|
||||
WORKER_FAMILIES: web
|
||||
CA_API_URL: http://ca-api:8000
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
INTERNAL_TOKEN: dev-internal-token
|
||||
depends_on:
|
||||
- redis
|
||||
- ca-api
|
||||
restart: unless-stopped
|
||||
|
||||
cp-workers-nlp:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: centers/parsing/workers/Dockerfile
|
||||
args:
|
||||
REQUIREMENTS_FILE: requirements-nlp.txt
|
||||
INSTALL_PLAYWRIGHT: "0"
|
||||
environment:
|
||||
TELEGRAM_LISTENER_ENABLED: "false"
|
||||
ENABLED_ADAPTERS: viina
|
||||
WORKER_FAMILIES: nlp
|
||||
CA_API_URL: http://ca-api:8000
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
INTERNAL_TOKEN: dev-internal-token
|
||||
depends_on:
|
||||
- redis
|
||||
- ca-api
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
ca_uploads:
|
||||
|
||||
Reference in New Issue
Block a user