Add VPN admin tab with subscription proxy via cp-vpn for selected sources.

Persist settings in CA, expose /admin/vpn and /internal/vpn, and route Telegram (and optional web/nlp) traffic through mihomo SOCKS when enabled.

Co-authored-by: Cursor <cursoragent@cursor.com>

centers/parsing/workers/workers/adapters/crawl4ai_adapter.py

@@ -57,7 +57,25 @@ class Crawl4AIAdapter:
         events: list[dict] = []
         errors: list[str] = []
 
-        async with AsyncWebCrawler(verbose=False) as crawler:
+        crawler_kwargs: dict[str, Any] = {"verbose": False}
+        try:
+            from workers.sources.vpn_config import http_proxy_url
+
+            proxy = http_proxy_url("crawl4ai")
+            if proxy:
+                try:
+                    from crawl4ai import BrowserConfig  # type: ignore
+
+                    crawler_kwargs["config"] = BrowserConfig(proxy=proxy)
+                    logger.info("Crawl4AI via proxy %s", proxy.split("@")[-1])
+                except Exception:
+                    logger.exception(
+                        "Could not apply VPN proxy to Crawl4AI; continuing without"
+                    )
+        except Exception:
+            logger.exception("VPN config lookup failed for crawl4ai")
+
+        async with AsyncWebCrawler(**crawler_kwargs) as crawler:
             for url in cfg.urls:
                 try:
                     if cfg.extract_mode == "llm":

centers/parsing/workers/workers/adapters/viina.py

@@ -70,7 +70,14 @@ class ViinaAdapter:
 
 
 async def _fetch_article_text(url: str) -> str:
-    async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
+    from workers.sources.vpn_config import http_proxy_url
+
+    proxy = http_proxy_url("viina")
+    kwargs: dict = {"timeout": 60.0, "follow_redirects": True}
+    if proxy:
+        kwargs["proxy"] = proxy
+        logger.info("VIINA fetch via proxy %s", proxy.split("@")[-1])
+    async with httpx.AsyncClient(**kwargs) as client:
         response = await client.get(
             url,
             headers={"User-Agent": "MapMil-CP-Viina/1.0"},

centers/parsing/workers/workers/sources/telegram_session.py

@@ -1,20 +1,39 @@
 """Единое подключение Telethon для listener и batch-заданий."""
 
+import logging
 from pathlib import Path
 
 from telethon import TelegramClient
 from telethon.errors import AuthKeyUnregisteredError, SessionPasswordNeededError
 
-from workers.sources.telegram_settings import create_client, get_api_credentials, get_session_path
+from workers.sources.telegram_settings import (
+    create_client,
+    describe_connection,
+    get_api_credentials,
+    get_session_path,
+)
 from workers.sources.telegram_client import TelegramAuthError, TelegramConfigError
+from workers.sources.vpn_config import proxy_fingerprint
+
+logger = logging.getLogger("cp-worker.telegram-session")
 
 _shared_client: TelegramClient | None = None
+_proxy_fingerprint: str | None = None
 
 
 async def get_shared_client() -> TelegramClient:
-    global _shared_client
+    global _shared_client, _proxy_fingerprint
+
+    fp = proxy_fingerprint()
     if _shared_client is not None and _shared_client.is_connected():
-        return _shared_client
+        if fp == _proxy_fingerprint:
+            return _shared_client
+        logger.info(
+            "VPN proxy changed (%s → %s); reconnecting Telethon",
+            _proxy_fingerprint,
+            fp,
+        )
+        await close_shared_client()
 
     try:
         api_id, api_hash = get_api_credentials()
@@ -29,6 +48,7 @@ async def get_shared_client() -> TelegramClient:
         )
 
     client = create_client(session_path, api_id, api_hash)
+    logger.info("Connecting Telethon %s", describe_connection())
     try:
         await client.connect()
         if not await client.is_user_authorized():
@@ -45,11 +65,13 @@ async def get_shared_client() -> TelegramClient:
         ) from exc
 
     _shared_client = client
+    _proxy_fingerprint = fp
     return client
 
 
 async def close_shared_client() -> None:
-    global _shared_client
+    global _shared_client, _proxy_fingerprint
     if _shared_client is not None:
         await _shared_client.disconnect()
         _shared_client = None
+    _proxy_fingerprint = None

centers/parsing/workers/workers/sources/telegram_settings.py

@@ -28,7 +28,8 @@ def get_api_credentials() -> tuple[int, str]:
     return api_id, api_hash
 
 
-def get_proxy() -> tuple | None:
+def get_env_proxy() -> tuple | None:
+    """Legacy TELEGRAM_PROXY_* env fallback (used when CA VPN is off)."""
     proxy_type = os.environ.get("TELEGRAM_PROXY_TYPE", "").strip().lower()
     if not proxy_type or proxy_type == "none":
         return None
@@ -49,6 +50,15 @@ def get_proxy() -> tuple | None:
     return proxy_type, host, port
 
 
+def get_proxy() -> tuple | None:
+    try:
+        from workers.sources.vpn_config import telethon_proxy_tuple
+
+        return telethon_proxy_tuple()
+    except Exception:
+        return get_env_proxy()
+
+
 def describe_connection() -> str:
     proxy = get_proxy()
     if not proxy:

centers/parsing/workers/workers/sources/vpn_config.py

@@ -0,0 +1,124 @@
+"""Fetch VPN settings from CA /internal/vpn (cached)."""
+
+from __future__ import annotations
+
+import logging
+import os
+import time
+from typing import Any
+
+import httpx
+
+from contracts.vpn import VpnSettings
+
+logger = logging.getLogger("cp-worker.vpn")
+
+CA_API_URL = os.getenv("CA_API_URL", "http://ca-api:8000").rstrip("/")
+INTERNAL_TOKEN = os.getenv("INTERNAL_TOKEN", "dev-internal-token")
+CACHE_TTL = float(os.getenv("VPN_CONFIG_CACHE_SECONDS", "30"))
+CP_VPN_HOST = os.getenv("CP_VPN_HOST", "cp-vpn")
+CP_VPN_PORT = int(os.getenv("CP_VPN_PORT", "1080"))
+
+_cache: VpnSettings | None = None
+_cache_at: float = 0.0
+_cache_failed: bool = False
+
+
+def invalidate_vpn_cache() -> None:
+    global _cache, _cache_at, _cache_failed
+    _cache = None
+    _cache_at = 0.0
+    _cache_failed = False
+
+
+def fetch_vpn_settings(*, force: bool = False) -> VpnSettings | None:
+    """Return VPN settings from CA, or None if unreachable / unset."""
+    global _cache, _cache_at, _cache_failed
+
+    now = time.monotonic()
+    if (
+        not force
+        and _cache is not None
+        and (now - _cache_at) < CACHE_TTL
+        and not _cache_failed
+    ):
+        return _cache
+
+    try:
+        with httpx.Client(timeout=10.0) as client:
+            response = client.get(
+                f"{CA_API_URL}/internal/vpn",
+                headers={"X-Internal-Token": INTERNAL_TOKEN},
+            )
+            response.raise_for_status()
+            raw: dict[str, Any] = response.json()
+        settings = VpnSettings.model_validate(raw)
+        _cache = settings
+        _cache_at = now
+        _cache_failed = False
+        return settings
+    except Exception:
+        logger.exception("Failed to load VPN settings from CA")
+        _cache_failed = True
+        _cache_at = now
+        # Keep stale cache if present
+        return _cache
+
+
+def proxy_fingerprint(settings: VpnSettings | None = None) -> str:
+    cfg = settings if settings is not None else fetch_vpn_settings()
+    if cfg is None or not cfg.proxies_source("telegram"):
+        # Env fallback fingerprint
+        from workers.sources.telegram_settings import get_env_proxy
+
+        env = get_env_proxy()
+        return f"env:{env!r}"
+    if cfg.mode == "subscription":
+        return f"sub:{CP_VPN_HOST}:{CP_VPN_PORT}:{cfg.subscription_url}"
+    return (
+        f"{cfg.mode}:{cfg.host}:{cfg.port}:"
+        f"{cfg.username or ''}:{'*' if cfg.password else ''}"
+    )
+
+
+def telethon_proxy_tuple(settings: VpnSettings | None = None) -> tuple | None:
+    """Telethon-compatible proxy tuple for telegram traffic, or None."""
+    cfg = settings if settings is not None else fetch_vpn_settings()
+    if cfg is not None and cfg.proxies_source("telegram"):
+        if cfg.mode == "subscription":
+            return ("socks5", CP_VPN_HOST, CP_VPN_PORT)
+        if cfg.mode in ("socks5", "http") and cfg.host and cfg.port:
+            if cfg.username:
+                return (
+                    cfg.mode,
+                    cfg.host,
+                    int(cfg.port),
+                    True,
+                    cfg.username,
+                    cfg.password or "",
+                )
+            return (cfg.mode, cfg.host, int(cfg.port))
+        return None
+
+    from workers.sources.telegram_settings import get_env_proxy
+
+    return get_env_proxy()
+
+
+def http_proxy_url(source_type: str, settings: VpnSettings | None = None) -> str | None:
+    """HTTP(S)/SOCKS proxy URL for httpx / browsers, or None."""
+    cfg = settings if settings is not None else fetch_vpn_settings()
+    if cfg is None or not cfg.proxies_source(source_type):
+        return None
+    if cfg.mode == "subscription":
+        return f"socks5://{CP_VPN_HOST}:{CP_VPN_PORT}"
+    if cfg.mode in ("socks5", "http") and cfg.host and cfg.port:
+        auth = ""
+        if cfg.username:
+            from urllib.parse import quote
+
+            user = quote(cfg.username, safe="")
+            password = quote(cfg.password or "", safe="")
+            auth = f"{user}:{password}@"
+        return f"{cfg.mode}://{auth}{cfg.host}:{int(cfg.port)}"
+    return None