Add CP→CA→PI platform foundation on MapMil monorepo.

Unify parsing workers, analytics API with PostgreSQL, map UI, and PI distribution into centers/ with Docker Compose.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-06-30 11:43:27 +03:00
co-authored by Cursor
commit 9dfe713668
88 changed files with 6587 additions and 0 deletions
+14
View File
@@ -0,0 +1,14 @@
FROM python:3.12-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app/ ./app/
RUN mkdir -p /data
EXPOSE 8000
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+28
View File
@@ -0,0 +1,28 @@
import os
from sqlalchemy import create_engine
from sqlalchemy.orm import DeclarativeBase, sessionmaker
DATABASE_URL = os.getenv(
"DATABASE_URL",
"postgresql://mapmil:mapmil@ca-db:5432/mapmil",
)
connect_args: dict = {}
if DATABASE_URL.startswith("sqlite"):
connect_args = {"check_same_thread": False}
engine = create_engine(DATABASE_URL, connect_args=connect_args)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
class Base(DeclarativeBase):
pass
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
+11
View File
@@ -0,0 +1,11 @@
import os
from fastapi import Header, HTTPException
def verify_internal_token(
x_internal_token: str | None = Header(default=None, alias="X-Internal-Token"),
) -> None:
expected = os.getenv("INTERNAL_TOKEN", "dev-internal-token")
if not x_internal_token or x_internal_token != expected:
raise HTTPException(status_code=401, detail="Invalid internal token")
+38
View File
@@ -0,0 +1,38 @@
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from .database import Base, engine, get_db
from .routers import admin, internal, objects, v1
from .seed import seed_objects, seed_test_consumer
from .storage import ensure_upload_dir
@asynccontextmanager
async def lifespan(_: FastAPI):
ensure_upload_dir()
Base.metadata.create_all(bind=engine)
db = next(get_db())
try:
seed_objects(db)
seed_test_consumer(db)
finally:
db.close()
yield
app = FastAPI(title="CA API (Analytics Center)", lifespan=lifespan)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
app.include_router(objects.router)
app.include_router(internal.router)
app.include_router(admin.router)
app.include_router(v1.router)
+145
View File
@@ -0,0 +1,145 @@
from datetime import datetime, timezone
from sqlalchemy import (
JSON,
Boolean,
DateTime,
Float,
ForeignKey,
Integer,
String,
Text,
UniqueConstraint,
)
from sqlalchemy.orm import Mapped, mapped_column, relationship
from .database import Base
class MapObject(Base):
__tablename__ = "map_objects"
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
name: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str] = mapped_column(Text, default="")
type: Mapped[str] = mapped_column(String(50), nullable=False)
latitude: Mapped[float] = mapped_column(Float, nullable=False)
longitude: Mapped[float] = mapped_column(Float, nullable=False)
event_id: Mapped[int | None] = mapped_column(
ForeignKey("events.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
)
media: Mapped[list["ObjectMedia"]] = relationship(
back_populates="object",
cascade="all, delete-orphan",
)
event: Mapped["Event | None"] = relationship(back_populates="map_object")
class ObjectMedia(Base):
__tablename__ = "object_media"
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
object_id: Mapped[int] = mapped_column(
ForeignKey("map_objects.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
stored_name: Mapped[str] = mapped_column(String(255), nullable=False)
original_name: Mapped[str] = mapped_column(String(255), nullable=False)
content_type: Mapped[str] = mapped_column(String(100), nullable=False)
size: Mapped[int] = mapped_column(Integer, nullable=False)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
)
object: Mapped["MapObject"] = relationship(back_populates="media")
class Event(Base):
__tablename__ = "events"
__table_args__ = (UniqueConstraint("source_url", name="uq_events_source_url"),)
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
source_type: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
source_url: Mapped[str] = mapped_column(String(512), nullable=False)
raw_text: Mapped[str] = mapped_column(Text, default="")
title: Mapped[str] = mapped_column(String(512), default="")
description: Mapped[str] = mapped_column(Text, default="")
locality: Mapped[str] = mapped_column(String(255), default="")
latitude: Mapped[float | None] = mapped_column(Float, nullable=True)
longitude: Mapped[float | None] = mapped_column(Float, nullable=True)
event_date: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
ingested_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
index=True,
)
region: Mapped[str | None] = mapped_column(String(100), nullable=True, index=True)
topic: Mapped[str | None] = mapped_column(String(100), nullable=True, index=True)
tags: Mapped[list | None] = mapped_column(JSON, nullable=True)
metadata_: Mapped[dict | None] = mapped_column("metadata", JSON, nullable=True)
map_object: Mapped["MapObject | None"] = relationship(
back_populates="event",
uselist=False,
)
class ParseJob(Base):
__tablename__ = "parse_jobs"
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
source_type: Mapped[str] = mapped_column(String(50), nullable=False)
source_config: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
schedule: Mapped[str | None] = mapped_column(String(100), nullable=True)
status: Mapped[str] = mapped_column(String(50), default="pending", index=True)
last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
)
class Consumer(Base):
__tablename__ = "consumers"
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
api_key_hash: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
)
filter: Mapped["ConsumerFilter | None"] = relationship(
back_populates="consumer",
uselist=False,
cascade="all, delete-orphan",
)
class ConsumerFilter(Base):
__tablename__ = "consumer_filters"
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
consumer_id: Mapped[int] = mapped_column(
ForeignKey("consumers.id", ondelete="CASCADE"),
nullable=False,
unique=True,
)
regions: Mapped[list | None] = mapped_column(JSON, nullable=True)
topics: Mapped[list | None] = mapped_column(JSON, nullable=True)
min_access_level: Mapped[int] = mapped_column(Integer, default=0)
date_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
consumer: Mapped["Consumer"] = relationship(back_populates="filter")
@@ -0,0 +1,78 @@
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
from ..database import get_db
from ..models import Event, ParseJob
from ..schemas import (
ConsumerCreate,
ConsumerRead,
EventRead,
ParseJobCreate,
ParseJobRead,
)
from ..services.filtering import create_consumer
from ..services.jobs import enqueue_job
router = APIRouter(prefix="/admin", tags=["admin"])
@router.post("/jobs", response_model=ParseJobRead, status_code=201)
def create_parse_job(payload: ParseJobCreate, db: Session = Depends(get_db)):
job = ParseJob(
source_type=payload.source_type,
source_config=payload.source_config,
schedule=payload.schedule,
status="queued",
)
db.add(job)
db.commit()
db.refresh(job)
enqueue_job(job.id, job.source_type, job.source_config)
return job
@router.get("/jobs", response_model=list[ParseJobRead])
def list_parse_jobs(db: Session = Depends(get_db)):
return db.query(ParseJob).order_by(ParseJob.id.desc()).all()
@router.get("/jobs/{job_id}", response_model=ParseJobRead)
def get_parse_job(job_id: int, db: Session = Depends(get_db)):
job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
if not job:
raise HTTPException(status_code=404, detail="Job not found")
return job
@router.get("/events", response_model=list[EventRead])
def list_events(
limit: int = Query(default=100, ge=1, le=1000),
offset: int = Query(default=0, ge=0),
db: Session = Depends(get_db),
):
return (
db.query(Event)
.order_by(Event.ingested_at.desc())
.offset(offset)
.limit(limit)
.all()
)
@router.post("/consumers", response_model=ConsumerRead, status_code=201)
def create_pi_consumer(payload: ConsumerCreate, db: Session = Depends(get_db)):
consumer, api_key = create_consumer(
db,
name=payload.name,
regions=payload.regions,
topics=payload.topics,
date_from=payload.date_from,
)
return ConsumerRead(
id=consumer.id,
name=consumer.name,
is_active=consumer.is_active,
created_at=consumer.created_at,
api_key=api_key,
)
@@ -0,0 +1,45 @@
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
from ..database import get_db
from ..deps import verify_internal_token
from ..models import ParseJob
from ..schemas import IngestRequest, IngestResponse
from ..services.ingest import ingest_events
router = APIRouter(prefix="/internal", tags=["internal"])
@router.post("/ingest", response_model=IngestResponse)
def internal_ingest(
payload: IngestRequest,
_: None = Depends(verify_internal_token),
db: Session = Depends(get_db),
):
ingested, updated, map_synced = ingest_events(db, payload.events, payload.job_id)
return IngestResponse(
ingested=ingested,
updated=updated,
map_objects_synced=map_synced,
)
@router.patch("/jobs/{job_id}")
def update_job_status(
job_id: int,
status: str,
error: str | None = None,
_: None = Depends(verify_internal_token),
db: Session = Depends(get_db),
):
job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
if not job:
raise HTTPException(status_code=404, detail="Job not found")
job.status = status
job.last_run_at = datetime.now(timezone.utc)
job.last_error = error
db.commit()
return {"id": job.id, "status": job.status}
@@ -0,0 +1,169 @@
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from fastapi.responses import FileResponse
from sqlalchemy.orm import Session
from ..database import get_db
from ..models import MapObject, ObjectMedia
from ..schemas import MapObjectCreate, MapObjectRead, MapObjectUpdate, ObjectMediaRead
from ..storage import (
ALLOWED_CONTENT_TYPES,
MAX_FILE_SIZE,
build_stored_name,
is_allowed_content_type,
media_file_path,
remove_media_file,
)
router = APIRouter(tags=["objects"])
def media_to_read(media: ObjectMedia) -> ObjectMediaRead:
return ObjectMediaRead(
id=media.id,
object_id=media.object_id,
original_name=media.original_name,
content_type=media.content_type,
size=media.size,
created_at=media.created_at,
url=f"/api/media/{media.id}/file",
)
@router.get("/api/health")
def health():
return {"status": "ok"}
@router.get("/api/objects", response_model=list[MapObjectRead])
def list_objects(db: Session = Depends(get_db)):
return db.query(MapObject).order_by(MapObject.id).all()
@router.get("/api/objects/{object_id}", response_model=MapObjectRead)
def get_object(object_id: int, db: Session = Depends(get_db)):
obj = db.query(MapObject).filter(MapObject.id == object_id).first()
if not obj:
raise HTTPException(status_code=404, detail="Объект не найден")
return obj
@router.post("/api/objects", response_model=MapObjectRead, status_code=201)
def create_object(payload: MapObjectCreate, db: Session = Depends(get_db)):
data = payload.model_dump()
created_at = data.pop("created_at", None)
obj = MapObject(**data)
if created_at is not None:
obj.created_at = created_at
db.add(obj)
db.commit()
db.refresh(obj)
return obj
@router.patch("/api/objects/{object_id}", response_model=MapObjectRead)
def update_object(
object_id: int,
payload: MapObjectUpdate,
db: Session = Depends(get_db),
):
obj = db.query(MapObject).filter(MapObject.id == object_id).first()
if not obj:
raise HTTPException(status_code=404, detail="Объект не найден")
for field, value in payload.model_dump(exclude_unset=True).items():
setattr(obj, field, value)
db.commit()
db.refresh(obj)
return obj
@router.delete("/api/objects/{object_id}", status_code=204)
def delete_object(object_id: int, db: Session = Depends(get_db)):
obj = db.query(MapObject).filter(MapObject.id == object_id).first()
if not obj:
raise HTTPException(status_code=404, detail="Объект не найден")
for media in obj.media:
remove_media_file(media.stored_name)
db.delete(obj)
db.commit()
@router.get("/api/objects/{object_id}/media", response_model=list[ObjectMediaRead])
def list_object_media(object_id: int, db: Session = Depends(get_db)):
obj = db.query(MapObject).filter(MapObject.id == object_id).first()
if not obj:
raise HTTPException(status_code=404, detail="Объект не найден")
media_items = (
db.query(ObjectMedia)
.filter(ObjectMedia.object_id == object_id)
.order_by(ObjectMedia.id)
.all()
)
return [media_to_read(item) for item in media_items]
@router.post("/api/objects/{object_id}/media", response_model=ObjectMediaRead, status_code=201)
async def upload_object_media(
object_id: int,
file: UploadFile = File(...),
db: Session = Depends(get_db),
):
obj = db.query(MapObject).filter(MapObject.id == object_id).first()
if not obj:
raise HTTPException(status_code=404, detail="Объект не найден")
content_type = file.content_type or "application/octet-stream"
if not is_allowed_content_type(content_type):
raise HTTPException(
status_code=400,
detail=f"Неподдерживаемый тип файла. Разрешены: {', '.join(sorted(ALLOWED_CONTENT_TYPES))}",
)
content = await file.read()
if len(content) > MAX_FILE_SIZE:
raise HTTPException(status_code=400, detail="Файл слишком большой (макс. 20 МБ)")
original_name = file.filename or "file"
stored_name = build_stored_name(original_name)
path = media_file_path(stored_name)
path.write_bytes(content)
media = ObjectMedia(
object_id=object_id,
stored_name=stored_name,
original_name=original_name,
content_type=content_type,
size=len(content),
)
db.add(media)
db.commit()
db.refresh(media)
return media_to_read(media)
@router.get("/api/media/{media_id}/file")
def get_media_file(media_id: int, db: Session = Depends(get_db)):
media = db.query(ObjectMedia).filter(ObjectMedia.id == media_id).first()
if not media:
raise HTTPException(status_code=404, detail="Медиафайл не найден")
path = media_file_path(media.stored_name)
if not path.exists():
raise HTTPException(status_code=404, detail="Файл не найден на диске")
return FileResponse(path, media_type=media.content_type, filename=media.original_name)
@router.delete("/api/media/{media_id}", status_code=204)
def delete_media(media_id: int, db: Session = Depends(get_db)):
media = db.query(ObjectMedia).filter(ObjectMedia.id == media_id).first()
if not media:
raise HTTPException(status_code=404, detail="Медиафайл не найден")
remove_media_file(media.stored_name)
db.delete(media)
db.commit()
+33
View File
@@ -0,0 +1,33 @@
from fastapi import APIRouter, Depends, Header, HTTPException, Query
from sqlalchemy.orm import Session
from ..database import get_db
from ..schemas import EventRead
from ..services.filtering import apply_consumer_filter, find_consumer_by_api_key
router = APIRouter(prefix="/api/v1", tags=["distribution"])
def get_consumer_from_api_key(
authorization: str | None = Header(default=None),
db: Session = Depends(get_db),
):
if not authorization or not authorization.startswith("Bearer "):
raise HTTPException(status_code=401, detail="Missing or invalid Authorization header")
api_key = authorization.removeprefix("Bearer ").strip()
consumer = find_consumer_by_api_key(db, api_key)
if not consumer:
raise HTTPException(status_code=401, detail="Invalid API key")
return consumer
@router.get("/events", response_model=list[EventRead])
def list_filtered_events(
limit: int = Query(default=100, ge=1, le=1000),
offset: int = Query(default=0, ge=0),
consumer=Depends(get_consumer_from_api_key),
db: Session = Depends(get_db),
):
events = apply_consumer_filter(db, consumer)
return events[offset : offset + limit]
+132
View File
@@ -0,0 +1,132 @@
from datetime import datetime
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
ObjectType = Literal["point", "marker", "zone", "other"]
class MapObjectCreate(BaseModel):
name: str = Field(min_length=1, max_length=255)
description: str = ""
type: ObjectType
latitude: float = Field(ge=-90, le=90)
longitude: float = Field(ge=-180, le=180)
created_at: datetime | None = None
class MapObjectRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
name: str
description: str
type: ObjectType
latitude: float
longitude: float
event_id: int | None = None
created_at: datetime
class MapObjectUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=255)
description: str | None = None
type: ObjectType | None = None
latitude: float | None = Field(default=None, ge=-90, le=90)
longitude: float | None = Field(default=None, ge=-180, le=180)
created_at: datetime | None = None
class ObjectMediaRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
object_id: int
original_name: str
content_type: str
size: int
created_at: datetime
url: str
class EventRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
source_type: str
source_url: str
raw_text: str
title: str
description: str
locality: str
latitude: float | None
longitude: float | None
event_date: datetime | None
ingested_at: datetime
region: str | None
topic: str | None
tags: list[str] | None
metadata: dict[str, Any] | None = Field(validation_alias="metadata_")
class IngestEventItem(BaseModel):
source_type: str = "telegram"
source_url: str
raw_text: str = ""
title: str = ""
description: str = ""
locality: str = ""
latitude: float | None = None
longitude: float | None = None
event_date: datetime | None = None
region: str | None = None
topic: str | None = None
tags: list[str] | None = None
metadata: dict[str, Any] | None = None
class IngestRequest(BaseModel):
job_id: int | None = None
events: list[IngestEventItem] = Field(default_factory=list)
class IngestResponse(BaseModel):
ingested: int
updated: int
map_objects_synced: int
class ParseJobCreate(BaseModel):
source_type: str = "telegram"
source_config: dict[str, Any] = Field(default_factory=dict)
schedule: str | None = None
class ParseJobRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
source_type: str
source_config: dict[str, Any]
schedule: str | None
status: str
last_run_at: datetime | None
last_error: str | None
created_at: datetime
class ConsumerCreate(BaseModel):
name: str
regions: list[str] | None = None
topics: list[str] | None = None
date_from: datetime | None = None
class ConsumerRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: int
name: str
is_active: bool
created_at: datetime
api_key: str | None = None
+68
View File
@@ -0,0 +1,68 @@
import os
from datetime import datetime, timezone
from sqlalchemy.orm import Session
from .models import Consumer, ConsumerFilter, MapObject
from .services.filtering import hash_api_key
SEED_OBJECTS = [
{
"name": "Красная площадь",
"description": "Главная площадь Москвы, исторический центр города.",
"type": "marker",
"latitude": 55.7539,
"longitude": 37.6208,
"created_at": datetime(2018, 5, 9, 12, 0, tzinfo=timezone.utc),
},
{
"name": "ВДНХ",
"description": "Выставка достижений народного хозяйства — крупный выставочный комплекс.",
"type": "zone",
"latitude": 55.8298,
"longitude": 37.6361,
"created_at": datetime(2020, 8, 15, 10, 30, tzinfo=timezone.utc),
},
{
"name": "МГУ",
"description": "Московский государственный университет имени М.В. Ломоносова.",
"type": "point",
"latitude": 55.7033,
"longitude": 37.5307,
"created_at": datetime(2022, 2, 12, 14, 0, tzinfo=timezone.utc),
},
{
"name": "Парк Горького",
"description": "Центральный парк культуры и отдыха имени М. Горького.",
"type": "other",
"latitude": 55.7312,
"longitude": 37.6013,
"created_at": datetime(2024, 6, 1, 9, 0, tzinfo=timezone.utc),
},
]
def seed_objects(db: Session) -> None:
if db.query(MapObject).count() > 0:
return
for item in SEED_OBJECTS:
db.add(MapObject(**item))
db.commit()
def seed_test_consumer(db: Session) -> None:
if db.query(Consumer).filter(Consumer.name == "test-pi").first():
return
api_key = os.getenv("TEST_PI_API_KEY", "test-pi-api-key-change-me")
consumer = Consumer(
name="test-pi",
api_key_hash=hash_api_key(api_key),
is_active=True,
)
db.add(consumer)
db.flush()
db.add(ConsumerFilter(consumer_id=consumer.id, regions=None, topics=None))
db.commit()
@@ -0,0 +1,109 @@
import hashlib
import secrets
from datetime import datetime
from sqlalchemy.orm import Session
from ..models import Consumer, ConsumerFilter, Event, MapObject
def hash_api_key(api_key: str) -> str:
return hashlib.sha256(api_key.encode()).hexdigest()
def generate_api_key() -> str:
return secrets.token_urlsafe(32)
def create_consumer(
db: Session,
name: str,
regions: list[str] | None = None,
topics: list[str] | None = None,
date_from: datetime | None = None,
) -> tuple[Consumer, str]:
api_key = generate_api_key()
consumer = Consumer(name=name, api_key_hash=hash_api_key(api_key), is_active=True)
db.add(consumer)
db.flush()
consumer_filter = ConsumerFilter(
consumer_id=consumer.id,
regions=regions,
topics=topics,
date_from=date_from,
)
db.add(consumer_filter)
db.commit()
db.refresh(consumer)
return consumer, api_key
def find_consumer_by_api_key(db: Session, api_key: str) -> Consumer | None:
key_hash = hash_api_key(api_key)
return (
db.query(Consumer)
.filter(Consumer.api_key_hash == key_hash, Consumer.is_active.is_(True))
.first()
)
def apply_consumer_filter(db: Session, consumer: Consumer) -> list[Event]:
query = db.query(Event).order_by(Event.ingested_at.desc())
consumer_filter = consumer.filter
if consumer_filter:
if consumer_filter.regions:
query = query.filter(Event.region.in_(consumer_filter.regions))
if consumer_filter.topics:
query = query.filter(Event.topic.in_(consumer_filter.topics))
if consumer_filter.date_from:
query = query.filter(Event.event_date >= consumer_filter.date_from)
return query.all()
def sync_event_to_map_object(db: Session, event: Event) -> MapObject | None:
if event.latitude is None or event.longitude is None:
return None
name = event.title or event.locality or f"Событие #{event.id}"
description = event.description or event.raw_text or ""
existing = (
db.query(MapObject)
.filter(MapObject.event_id == event.id)
.first()
)
if existing:
existing.name = name
existing.description = description
existing.latitude = event.latitude
existing.longitude = event.longitude
return existing
by_url = (
db.query(MapObject)
.join(Event, MapObject.event_id == Event.id)
.filter(Event.source_url == event.source_url)
.first()
)
if by_url:
by_url.event_id = event.id
by_url.name = name
by_url.description = description
by_url.latitude = event.latitude
by_url.longitude = event.longitude
return by_url
obj = MapObject(
name=name,
description=description,
type="marker",
latitude=event.latitude,
longitude=event.longitude,
event_id=event.id,
created_at=event.event_date or event.ingested_at,
)
db.add(obj)
return obj
@@ -0,0 +1,72 @@
from sqlalchemy.orm import Session
from ..models import Event, ParseJob
from ..schemas import IngestEventItem
from .filtering import sync_event_to_map_object
def ingest_events(
db: Session,
items: list[IngestEventItem],
job_id: int | None = None,
) -> tuple[int, int, int]:
ingested = 0
updated = 0
map_synced = 0
for item in items:
existing = (
db.query(Event)
.filter(Event.source_url == item.source_url)
.first()
)
if existing:
existing.source_type = item.source_type
existing.raw_text = item.raw_text
existing.title = item.title
existing.description = item.description
existing.locality = item.locality
existing.latitude = item.latitude
existing.longitude = item.longitude
existing.event_date = item.event_date
existing.region = item.region
existing.topic = item.topic
existing.tags = item.tags
existing.metadata_ = item.metadata
event = existing
updated += 1
else:
event = Event(
source_type=item.source_type,
source_url=item.source_url,
raw_text=item.raw_text,
title=item.title,
description=item.description,
locality=item.locality,
latitude=item.latitude,
longitude=item.longitude,
event_date=item.event_date,
region=item.region,
topic=item.topic,
tags=item.tags,
metadata_=item.metadata,
)
db.add(event)
ingested += 1
db.flush()
if sync_event_to_map_object(db, event):
map_synced += 1
if job_id is not None:
job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
if job:
from datetime import datetime, timezone
job.status = "completed"
job.last_run_at = datetime.now(timezone.utc)
job.last_error = None
db.commit()
return ingested, updated, map_synced
@@ -0,0 +1,20 @@
import json
import os
import redis
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
JOB_QUEUE_KEY = "cp:jobs"
def get_redis() -> redis.Redis:
return redis.from_url(REDIS_URL, decode_responses=True)
def enqueue_job(job_id: int, source_type: str, source_config: dict) -> None:
payload = {
"job_id": job_id,
"source_type": source_type,
"source_config": source_config,
}
get_redis().rpush(JOB_QUEUE_KEY, json.dumps(payload))
+39
View File
@@ -0,0 +1,39 @@
import os
import uuid
from pathlib import Path
UPLOAD_DIR = Path(os.getenv("UPLOAD_DIR", "/data/uploads"))
ALLOWED_CONTENT_TYPES = {
"image/jpeg",
"image/png",
"image/gif",
"image/webp",
"video/mp4",
"video/webm",
}
MAX_FILE_SIZE = 20 * 1024 * 1024
def ensure_upload_dir() -> None:
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
def is_allowed_content_type(content_type: str) -> bool:
return content_type in ALLOWED_CONTENT_TYPES
def build_stored_name(original_name: str) -> str:
suffix = Path(original_name).suffix.lower()
return f"{uuid.uuid4().hex}{suffix}"
def media_file_path(stored_name: str) -> Path:
return UPLOAD_DIR / stored_name
def remove_media_file(stored_name: str) -> None:
path = media_file_path(stored_name)
if path.exists():
path.unlink()
+7
View File
@@ -0,0 +1,7 @@
fastapi==0.115.6
uvicorn[standard]==0.34.0
sqlalchemy==2.0.36
pydantic==2.10.3
python-multipart==0.0.20
psycopg2-binary==2.9.10
redis==5.2.1
+18
View File
@@ -0,0 +1,18 @@
FROM node:22-alpine AS build
WORKDIR /app
COPY package.json ./
RUN npm install
COPY . .
RUN npm run build
FROM nginx:alpine
COPY nginx.conf /etc/nginx/conf.d/default.conf
COPY --from=build /app/dist /usr/share/nginx/html
EXPOSE 80
CMD ["nginx", "-g", "daemon off;"]
+18
View File
@@ -0,0 +1,18 @@
<!DOCTYPE html>
<html lang="ru">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>MapMil</title>
<link
rel="stylesheet"
href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css"
integrity="sha256-p4NxAoJBhIIN+hmNHrzRCf9tD/miZyoHS5obTRR9BMY="
crossorigin=""
/>
</head>
<body>
<div id="app"></div>
<script type="module" src="/src/main.ts"></script>
</body>
</html>
+35
View File
@@ -0,0 +1,35 @@
server {
listen 80;
server_name localhost;
root /usr/share/nginx/html;
index index.html;
location /api/ {
client_max_body_size 50M;
proxy_pass http://ca-api:8000/api/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
location /admin/ {
proxy_pass http://ca-api:8000/admin/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
location /internal/ {
proxy_pass http://ca-api:8000/internal/;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
location / {
try_files $uri $uri/ /index.html;
}
}
+22
View File
@@ -0,0 +1,22 @@
{
"name": "mapmil-frontend",
"private": true,
"version": "1.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build",
"preview": "vite preview"
},
"dependencies": {
"leaflet": "^1.9.4",
"vue": "^3.5.13"
},
"devDependencies": {
"@types/leaflet": "^1.9.15",
"@vitejs/plugin-vue": "^5.2.1",
"typescript": "~5.6.3",
"vite": "^6.0.3",
"vue-tsc": "^2.1.10"
}
}
+349
View File
@@ -0,0 +1,349 @@
<script setup lang="ts">
import { computed, onMounted, ref, watch } from "vue";
import {
createObject,
deleteObject,
fetchObjects,
updateObject,
uploadObjectMedia,
} from "./api/objects";
import ContextMenu from "./components/ContextMenu.vue";
import CreateObjectModal from "./components/CreateObjectModal.vue";
import EditObjectModal from "./components/EditObjectModal.vue";
import MapView from "./components/MapView.vue";
import ObjectPanel from "./components/ObjectPanel.vue";
import TimelineBar from "./components/TimelineBar.vue";
import type { MapObject, MapObjectCreate, ObjectType } from "./types/object";
const objects = ref<MapObject[]>([]);
const selectedObject = ref<MapObject | null>(null);
const loading = ref(true);
const error = ref("");
const timelinePosition = ref(Date.now());
const contextMenu = ref<{
visible: boolean;
x: number;
y: number;
latitude: number;
longitude: number;
target: "map" | "object";
object: MapObject | null;
}>({
visible: false,
x: 0,
y: 0,
latitude: 0,
longitude: 0,
target: "map",
object: null,
});
const createModal = ref({
visible: false,
latitude: 0,
longitude: 0,
});
const editModal = ref({
visible: false,
object: null as MapObject | null,
});
const selectedId = computed(() => selectedObject.value?.id ?? null);
function objectTime(obj: MapObject): number {
return new Date(obj.created_at).getTime();
}
function replaceObject(updated: MapObject) {
objects.value = objects.value.map((obj) => (obj.id === updated.id ? updated : obj));
if (selectedObject.value?.id === updated.id) {
selectedObject.value = updated;
}
}
const timelineBounds = computed(() => {
if (objects.value.length === 0) {
const now = Date.now();
return { min: now, max: now };
}
const times = objects.value.map(objectTime);
return {
min: Math.min(...times),
max: Math.max(...times),
};
});
const visibleObjects = computed(() =>
objects.value.filter((obj) => objectTime(obj) <= timelinePosition.value),
);
function syncTimelineToMax() {
timelinePosition.value = timelineBounds.value.max;
}
async function loadObjects() {
loading.value = true;
error.value = "";
try {
objects.value = await fetchObjects();
syncTimelineToMax();
} catch (err) {
error.value = err instanceof Error ? err.message : "Не удалось загрузить объекты";
} finally {
loading.value = false;
}
}
function handleSelectObject(obj: MapObject) {
selectedObject.value = obj;
}
function handleMapContextMenu(payload: {
x: number;
y: number;
latitude: number;
longitude: number;
object: MapObject | null;
}) {
contextMenu.value = {
visible: true,
x: payload.x,
y: payload.y,
latitude: payload.latitude,
longitude: payload.longitude,
target: payload.object ? "object" : "map",
object: payload.object,
};
if (payload.object) {
selectedObject.value = payload.object;
}
}
function closeContextMenu() {
contextMenu.value.visible = false;
}
function openCreateModal() {
createModal.value = {
visible: true,
latitude: contextMenu.value.latitude,
longitude: contextMenu.value.longitude,
};
closeContextMenu();
}
function closeCreateModal() {
createModal.value.visible = false;
}
function openEditModal() {
if (!contextMenu.value.object) return;
editModal.value = {
visible: true,
object: contextMenu.value.object,
};
closeContextMenu();
}
function closeEditModal() {
editModal.value.visible = false;
editModal.value.object = null;
}
async function handleCreateObject(payload: {
name: string;
description: string;
type: ObjectType;
created_at?: string;
files: File[];
}) {
const data: MapObjectCreate = {
name: payload.name,
description: payload.description,
type: payload.type,
latitude: createModal.value.latitude,
longitude: createModal.value.longitude,
created_at: payload.created_at,
};
const created = await createObject(data);
for (const file of payload.files) {
await uploadObjectMedia(created.id, file);
}
objects.value = [...objects.value, created];
selectedObject.value = created;
timelinePosition.value = objectTime(created);
closeCreateModal();
}
async function handleEditObject(payload: {
name: string;
description: string;
type: ObjectType;
created_at: string;
}) {
if (!editModal.value.object) return;
const updated = await updateObject(editModal.value.object.id, payload);
replaceObject(updated);
timelinePosition.value = objectTime(updated);
closeEditModal();
}
async function handleDeleteObject() {
const object = contextMenu.value.object;
if (!object) return;
const confirmed = window.confirm(`Удалить объект «${object.name}»?`);
if (!confirmed) return;
closeContextMenu();
try {
await deleteObject(object.id);
objects.value = objects.value.filter((item) => item.id !== object.id);
if (selectedObject.value?.id === object.id) {
selectedObject.value = null;
}
} catch (err) {
error.value = err instanceof Error ? err.message : "Не удалось удалить объект";
}
}
async function handleMoveObject(payload: {
object: MapObject;
latitude: number;
longitude: number;
}) {
try {
const updated = await updateObject(payload.object.id, {
latitude: payload.latitude,
longitude: payload.longitude,
});
replaceObject(updated);
} catch (err) {
error.value = err instanceof Error ? err.message : "Не удалось переместить объект";
}
}
watch(visibleObjects, (visible) => {
if (
selectedObject.value &&
!visible.some((obj) => obj.id === selectedObject.value?.id)
) {
selectedObject.value = null;
}
});
onMounted(loadObjects);
</script>
<template>
<div class="app" @click="closeContextMenu">
<header class="header">
<h1>MapMil</h1>
<span v-if="loading" class="status">Загрузка...</span>
<span v-else-if="error" class="status error">{{ error }}</span>
<span v-else class="status">
{{ visibleObjects.length }} / {{ objects.length }} объектов на карте
</span>
</header>
<main class="main">
<MapView
:objects="visibleObjects"
:selected-id="selectedId"
@select="handleSelectObject"
@contextmenu="handleMapContextMenu"
@move="handleMoveObject"
/>
<ObjectPanel :object="selectedObject" />
</main>
<TimelineBar
v-if="!loading && objects.length > 0"
v-model="timelinePosition"
:min="timelineBounds.min"
:max="timelineBounds.max"
:objects="objects"
:visible-count="visibleObjects.length"
/>
<ContextMenu
v-if="contextMenu.visible"
:x="contextMenu.x"
:y="contextMenu.y"
:target="contextMenu.target"
:object-name="contextMenu.object?.name"
@create="openCreateModal"
@edit="openEditModal"
@delete="handleDeleteObject"
/>
<CreateObjectModal
v-if="createModal.visible"
:latitude="createModal.latitude"
:longitude="createModal.longitude"
@close="closeCreateModal"
@submit="handleCreateObject"
/>
<EditObjectModal
v-if="editModal.visible && editModal.object"
:object="editModal.object"
@close="closeEditModal"
@submit="handleEditObject"
/>
</div>
</template>
<style scoped>
.app {
display: flex;
flex-direction: column;
height: 100%;
}
.header {
position: relative;
z-index: 5;
display: flex;
align-items: center;
gap: 1rem;
padding: 0.75rem 1.25rem;
background: #fff;
border-bottom: 1px solid #e0e0e0;
}
.header h1 {
margin: 0;
font-size: 1.25rem;
font-weight: 600;
}
.status {
font-size: 0.875rem;
color: #666;
}
.status.error {
color: #c62828;
}
.main {
position: relative;
z-index: 1;
display: flex;
flex: 1;
min-height: 0;
overflow: hidden;
}
</style>
@@ -0,0 +1,82 @@
import type { MapObject, MapObjectCreate, MapObjectUpdate, ObjectMedia } from "../types/object";
const API_BASE = "/api";
async function request<T>(url: string, options?: RequestInit): Promise<T> {
const headers = new Headers(options?.headers);
const isFormData = options?.body instanceof FormData;
if (!isFormData && !headers.has("Content-Type")) {
headers.set("Content-Type", "application/json");
}
const response = await fetch(`${API_BASE}${url}`, {
...options,
headers,
});
if (!response.ok) {
const message = await response.text();
throw new Error(message || `Ошибка запроса: ${response.status}`);
}
if (response.status === 204) {
return undefined as T;
}
return response.json() as Promise<T>;
}
export function fetchObjects(): Promise<MapObject[]> {
return request<MapObject[]>("/objects");
}
export function fetchObject(id: number): Promise<MapObject> {
return request<MapObject>(`/objects/${id}`);
}
export function createObject(payload: MapObjectCreate): Promise<MapObject> {
return request<MapObject>("/objects", {
method: "POST",
body: JSON.stringify(payload),
});
}
export function updateObject(id: number, payload: MapObjectUpdate): Promise<MapObject> {
return request<MapObject>(`/objects/${id}`, {
method: "PATCH",
body: JSON.stringify(payload),
});
}
export function deleteObject(id: number): Promise<void> {
return request<void>(`/objects/${id}`, {
method: "DELETE",
});
}
export function fetchObjectMedia(objectId: number): Promise<ObjectMedia[]> {
return request<ObjectMedia[]>(`/objects/${objectId}/media`);
}
export function uploadObjectMedia(objectId: number, file: File): Promise<ObjectMedia> {
const form = new FormData();
form.append("file", file);
return request<ObjectMedia>(`/objects/${objectId}/media`, {
method: "POST",
body: form,
});
}
export function deleteObjectMedia(mediaId: number): Promise<void> {
return request<void>(`/media/${mediaId}`, {
method: "DELETE",
});
}
export function formatFileSize(bytes: number): string {
if (bytes < 1024) return `${bytes} Б`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} КБ`;
return `${(bytes / (1024 * 1024)).toFixed(1)} МБ`;
}
@@ -0,0 +1,81 @@
<script setup lang="ts">
defineProps<{
x: number;
y: number;
target: "map" | "object";
objectName?: string;
}>();
const emit = defineEmits<{
create: [];
edit: [];
delete: [];
}>();
</script>
<template>
<div
class="context-menu"
:style="{ left: `${x}px`, top: `${y}px` }"
@click.stop
>
<p v-if="target === 'object' && objectName" class="title">{{ objectName }}</p>
<template v-if="target === 'map'">
<button type="button" @click="emit('create')">Создать объект</button>
</template>
<template v-else>
<button type="button" @click="emit('edit')">Редактировать</button>
<button type="button" class="danger" @click="emit('delete')">Удалить</button>
</template>
</div>
</template>
<style scoped>
.context-menu {
position: fixed;
z-index: 1000;
min-width: 200px;
background: #fff;
border: 1px solid #d0d0d0;
border-radius: 6px;
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.12);
overflow: hidden;
}
.title {
margin: 0;
padding: 0.5rem 1rem;
font-size: 0.75rem;
font-weight: 600;
color: #666;
background: #f8f8f8;
border-bottom: 1px solid #eee;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
button {
display: block;
width: 100%;
padding: 0.625rem 1rem;
border: none;
background: #fff;
text-align: left;
cursor: pointer;
}
button:hover {
background: #f0f4ff;
}
button.danger {
color: #b91c1c;
}
button.danger:hover {
background: #fee2e2;
}
</style>
@@ -0,0 +1,253 @@
<script setup lang="ts">
import { computed, ref } from "vue";
import { MEDIA_ACCEPT, OBJECT_TYPE_LABELS, OBJECT_TYPES, type ObjectType } from "../types/object";
const props = defineProps<{
latitude: number;
longitude: number;
}>();
const emit = defineEmits<{
close: [];
submit: [payload: {
name: string;
description: string;
type: ObjectType;
created_at?: string;
files: File[];
}];
}>();
const name = ref("");
const description = ref("");
const type = ref<ObjectType>("marker");
const createdAt = ref(toLocalDateTimeValue(new Date()));
const selectedFiles = ref<File[]>([]);
const error = ref("");
const submitting = ref(false);
const fileLabel = computed(() => {
if (selectedFiles.value.length === 0) return "Файлы не выбраны";
return selectedFiles.value.map((file) => file.name).join(", ");
});
function toLocalDateTimeValue(date: Date): string {
const pad = (value: number) => String(value).padStart(2, "0");
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}T${pad(date.getHours())}:${pad(date.getMinutes())}`;
}
function handleFileChange(event: Event) {
const input = event.target as HTMLInputElement;
selectedFiles.value = input.files ? Array.from(input.files) : [];
}
async function handleSubmit() {
if (!name.value.trim()) {
error.value = "Введите название объекта";
return;
}
if (!createdAt.value) {
error.value = "Укажите дату создания";
return;
}
error.value = "";
submitting.value = true;
try {
emit("submit", {
name: name.value.trim(),
description: description.value.trim(),
type: type.value,
created_at: new Date(createdAt.value).toISOString(),
files: selectedFiles.value,
});
} catch (err) {
error.value = err instanceof Error ? err.message : "Не удалось создать объект";
} finally {
submitting.value = false;
}
}
</script>
<template>
<div class="overlay" @click.self="emit('close')">
<div class="modal" role="dialog" aria-labelledby="create-title">
<header>
<h2 id="create-title">Создать объект</h2>
<button type="button" class="close" aria-label="Закрыть" @click="emit('close')">
×
</button>
</header>
<form @submit.prevent="handleSubmit">
<p class="coords">
Координаты: {{ props.latitude.toFixed(6) }}, {{ props.longitude.toFixed(6) }}
</p>
<label>
Название *
<input v-model="name" type="text" placeholder="Название объекта" required />
</label>
<label>
Тип *
<select v-model="type" required>
<option v-for="item in OBJECT_TYPES" :key="item" :value="item">
{{ OBJECT_TYPE_LABELS[item] }}
</option>
</select>
</label>
<label>
Дата создания *
<input v-model="createdAt" type="datetime-local" required />
</label>
<label>
Описание
<textarea
v-model="description"
rows="4"
placeholder="Описание объекта"
/>
</label>
<label>
Медиафайлы
<input
type="file"
multiple
:accept="MEDIA_ACCEPT"
@change="handleFileChange"
/>
<span class="hint">{{ fileLabel }}</span>
</label>
<p v-if="error" class="error">{{ error }}</p>
<footer>
<button type="button" class="secondary" @click="emit('close')">Отмена</button>
<button type="submit" :disabled="submitting">
{{ submitting ? "Создание..." : "Создать" }}
</button>
</footer>
</form>
</div>
</div>
</template>
<style scoped>
.overlay {
position: fixed;
inset: 0;
z-index: 1100;
display: flex;
align-items: center;
justify-content: center;
background: rgba(0, 0, 0, 0.4);
}
.modal {
width: min(480px, calc(100vw - 2rem));
max-height: calc(100vh - 2rem);
overflow-y: auto;
background: #fff;
border-radius: 8px;
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2);
}
header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 1rem 1.25rem;
border-bottom: 1px solid #e8e8e8;
}
header h2 {
margin: 0;
font-size: 1.125rem;
}
.close {
border: none;
background: none;
font-size: 1.5rem;
line-height: 1;
cursor: pointer;
color: #666;
}
form {
padding: 1.25rem;
}
.coords {
margin: 0 0 1rem;
font-size: 0.875rem;
color: #666;
}
label {
display: block;
margin-bottom: 1rem;
font-size: 0.875rem;
font-weight: 500;
}
input,
select,
textarea {
display: block;
width: 100%;
margin-top: 0.375rem;
padding: 0.5rem 0.75rem;
border: 1px solid #ccc;
border-radius: 4px;
}
textarea {
resize: vertical;
}
.hint {
display: block;
margin-top: 0.375rem;
font-size: 0.75rem;
font-weight: 400;
color: #666;
}
.error {
margin: 0 0 1rem;
color: #c62828;
font-size: 0.875rem;
}
footer {
display: flex;
justify-content: flex-end;
gap: 0.5rem;
}
button {
padding: 0.5rem 1rem;
border: none;
border-radius: 4px;
background: #2563eb;
color: #fff;
cursor: pointer;
}
button.secondary {
background: #e8e8e8;
color: #333;
}
button:disabled {
opacity: 0.6;
cursor: not-allowed;
}
</style>
@@ -0,0 +1,239 @@
<script setup lang="ts">
import { ref, watch } from "vue";
import { OBJECT_TYPE_LABELS, OBJECT_TYPES, type MapObject, type ObjectType } from "../types/object";
const props = defineProps<{
object: MapObject;
}>();
const emit = defineEmits<{
close: [];
submit: [payload: {
name: string;
description: string;
type: ObjectType;
created_at: string;
}];
}>();
const name = ref("");
const description = ref("");
const type = ref<ObjectType>("marker");
const createdAt = ref("");
const error = ref("");
const submitting = ref(false);
function toLocalDateTimeValue(iso: string): string {
const date = new Date(iso);
const pad = (value: number) => String(value).padStart(2, "0");
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}T${pad(date.getHours())}:${pad(date.getMinutes())}`;
}
function resetForm() {
name.value = props.object.name;
description.value = props.object.description;
type.value = props.object.type;
createdAt.value = toLocalDateTimeValue(props.object.created_at);
error.value = "";
}
async function handleSubmit() {
if (!name.value.trim()) {
error.value = "Введите название объекта";
return;
}
if (!createdAt.value) {
error.value = "Укажите дату создания";
return;
}
error.value = "";
submitting.value = true;
try {
emit("submit", {
name: name.value.trim(),
description: description.value.trim(),
type: type.value,
created_at: new Date(createdAt.value).toISOString(),
});
} catch (err) {
error.value = err instanceof Error ? err.message : "Не удалось сохранить изменения";
} finally {
submitting.value = false;
}
}
watch(() => props.object, resetForm, { immediate: true });
</script>
<template>
<div class="overlay" @click.self="emit('close')">
<div class="modal" role="dialog" aria-labelledby="edit-title">
<header>
<h2 id="edit-title">Редактировать объект</h2>
<button type="button" class="close" aria-label="Закрыть" @click="emit('close')">
×
</button>
</header>
<form @submit.prevent="handleSubmit">
<p class="coords">
Координаты: {{ object.latitude.toFixed(6) }}, {{ object.longitude.toFixed(6) }}
<span class="hint">(измените через «Переместить» на карте)</span>
</p>
<label>
Название *
<input v-model="name" type="text" placeholder="Название объекта" required />
</label>
<label>
Тип *
<select v-model="type" required>
<option v-for="item in OBJECT_TYPES" :key="item" :value="item">
{{ OBJECT_TYPE_LABELS[item] }}
</option>
</select>
</label>
<label>
Дата создания *
<input v-model="createdAt" type="datetime-local" required />
</label>
<label>
Описание
<textarea
v-model="description"
rows="4"
placeholder="Описание объекта"
/>
</label>
<p v-if="error" class="error">{{ error }}</p>
<footer>
<button type="button" class="secondary" @click="emit('close')">Отмена</button>
<button type="submit" :disabled="submitting">
{{ submitting ? "Сохранение..." : "Сохранить" }}
</button>
</footer>
</form>
</div>
</div>
</template>
<style scoped>
.overlay {
position: fixed;
inset: 0;
z-index: 1100;
display: flex;
align-items: center;
justify-content: center;
background: rgba(0, 0, 0, 0.4);
}
.modal {
width: min(480px, calc(100vw - 2rem));
max-height: calc(100vh - 2rem);
overflow-y: auto;
background: #fff;
border-radius: 8px;
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.2);
}
header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 1rem 1.25rem;
border-bottom: 1px solid #e8e8e8;
}
header h2 {
margin: 0;
font-size: 1.125rem;
}
.close {
border: none;
background: none;
font-size: 1.5rem;
line-height: 1;
cursor: pointer;
color: #666;
}
form {
padding: 1.25rem;
}
.coords {
margin: 0 0 1rem;
font-size: 0.875rem;
color: #666;
line-height: 1.5;
}
.hint {
display: block;
font-size: 0.75rem;
color: #999;
}
label {
display: block;
margin-bottom: 1rem;
font-size: 0.875rem;
font-weight: 500;
}
input,
select,
textarea {
display: block;
width: 100%;
margin-top: 0.375rem;
padding: 0.5rem 0.75rem;
border: 1px solid #ccc;
border-radius: 4px;
}
textarea {
resize: vertical;
}
.error {
margin: 0 0 1rem;
color: #c62828;
font-size: 0.875rem;
}
footer {
display: flex;
justify-content: flex-end;
gap: 0.5rem;
}
button {
padding: 0.5rem 1rem;
border: none;
border-radius: 4px;
background: #2563eb;
color: #fff;
cursor: pointer;
}
button.secondary {
background: #e8e8e8;
color: #333;
}
button:disabled {
opacity: 0.6;
cursor: not-allowed;
}
</style>
@@ -0,0 +1,232 @@
<script setup lang="ts">
import L from "leaflet";
import { onMounted, onUnmounted, ref, watch } from "vue";
import type { MapObject } from "../types/object";
const props = defineProps<{
objects: MapObject[];
selectedId: number | null;
}>();
const emit = defineEmits<{
select: [object: MapObject];
contextmenu: [payload: {
x: number;
y: number;
latitude: number;
longitude: number;
object: MapObject | null;
}];
move: [payload: { object: MapObject; latitude: number; longitude: number }];
}>();
const mapContainer = ref<HTMLElement | null>(null);
let map: L.Map | null = null;
let markersLayer: L.LayerGroup | null = null;
const markerById = new Map<number, L.Marker>();
const defaultIcon = L.icon({
iconUrl: "https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon.png",
iconRetinaUrl: "https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon-2x.png",
shadowUrl: "https://unpkg.com/leaflet@1.9.4/dist/images/marker-shadow.png",
iconSize: [25, 41],
iconAnchor: [12, 41],
popupAnchor: [1, -34],
shadowSize: [41, 41],
});
const selectedIcon = L.icon({
iconUrl: "https://raw.githubusercontent.com/pointhi/leaflet-color-markers/master/img/marker-icon-orange.png",
iconRetinaUrl: "https://raw.githubusercontent.com/pointhi/leaflet-color-markers/master/img/marker-icon-2x-orange.png",
shadowUrl: "https://unpkg.com/leaflet@1.9.4/dist/images/marker-shadow.png",
iconSize: [25, 41],
iconAnchor: [12, 41],
popupAnchor: [1, -34],
shadowSize: [41, 41],
});
function isDraggable(obj: MapObject): boolean {
return obj.id === props.selectedId;
}
function getMarkerIcon(obj: MapObject): L.Icon {
if (obj.id === props.selectedId) return selectedIcon;
return defaultIcon;
}
function bindMarker(marker: L.Marker, obj: MapObject) {
marker.off("click");
marker.off("contextmenu");
marker.off("dragend");
marker.on("click", () => emit("select", obj));
marker.on("contextmenu", (event: L.LeafletMouseEvent) => {
L.DomEvent.stopPropagation(event.originalEvent);
L.DomEvent.preventDefault(event.originalEvent);
emit("contextmenu", {
x: event.originalEvent.clientX,
y: event.originalEvent.clientY,
latitude: obj.latitude,
longitude: obj.longitude,
object: obj,
});
});
const draggable = isDraggable(obj);
if (marker.dragging) {
if (draggable) {
marker.dragging.enable();
} else {
marker.dragging.disable();
}
}
if (draggable) {
marker.on("dragend", () => {
const latlng = marker.getLatLng();
emit("move", {
object: obj,
latitude: latlng.lat,
longitude: latlng.lng,
});
});
}
}
function syncMarkers() {
if (!map || !markersLayer) return;
const currentIds = new Set(props.objects.map((obj) => obj.id));
for (const [id, marker] of markerById) {
if (!currentIds.has(id)) {
markersLayer.removeLayer(marker);
markerById.delete(id);
}
}
for (const obj of props.objects) {
const draggable = isDraggable(obj);
let marker = markerById.get(obj.id);
if (!marker) {
marker = L.marker([obj.latitude, obj.longitude], {
icon: getMarkerIcon(obj),
draggable,
});
marker.addTo(markersLayer);
markerById.set(obj.id, marker);
} else {
const latlng = marker.getLatLng();
const positionChanged =
Math.abs(latlng.lat - obj.latitude) > 1e-8 ||
Math.abs(latlng.lng - obj.longitude) > 1e-8;
if (positionChanged) {
marker.setLatLng([obj.latitude, obj.longitude]);
}
marker.setIcon(getMarkerIcon(obj));
if (marker.dragging) {
if (draggable) {
marker.dragging.enable();
} else {
marker.dragging.disable();
}
}
}
bindMarker(marker, obj);
}
}
function handleMapContextMenu(event: MouseEvent) {
if (!map || !mapContainer.value) return;
event.preventDefault();
const rect = mapContainer.value.getBoundingClientRect();
const point = map.containerPointToLatLng(
L.point(event.clientX - rect.left, event.clientY - rect.top),
);
emit("contextmenu", {
x: event.clientX,
y: event.clientY,
latitude: point.lat,
longitude: point.lng,
object: null,
});
}
onMounted(() => {
if (!mapContainer.value) return;
map = L.map(mapContainer.value).setView([55.7558, 37.6173], 11);
L.tileLayer("https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", {
attribution: '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>',
maxZoom: 19,
}).addTo(map);
markersLayer = L.layerGroup().addTo(map);
mapContainer.value.addEventListener("contextmenu", handleMapContextMenu);
syncMarkers();
});
onUnmounted(() => {
mapContainer.value?.removeEventListener("contextmenu", handleMapContextMenu);
map?.remove();
map = null;
markersLayer = null;
markerById.clear();
});
watch(
() => [props.objects, props.selectedId] as const,
() => syncMarkers(),
{ deep: true },
);
</script>
<template>
<div class="map-wrapper">
<div
ref="mapContainer"
class="map"
:class="{ 'has-selection': selectedId !== null }"
/>
</div>
</template>
<style scoped>
.map-wrapper {
position: relative;
z-index: 0;
flex: 1;
min-width: 0;
min-height: 0;
overflow: hidden;
isolation: isolate;
}
.map {
width: 100%;
height: 100%;
}
.map-wrapper :deep(.leaflet-container) {
z-index: 0;
}
.map.has-selection :deep(.leaflet-marker-draggable) {
cursor: grab;
}
.map.has-selection :deep(.leaflet-marker-draggable:active) {
cursor: grabbing;
}
</style>
@@ -0,0 +1,343 @@
<script setup lang="ts">
import { computed, ref, watch } from "vue";
import {
deleteObjectMedia,
fetchObjectMedia,
formatFileSize,
uploadObjectMedia,
} from "../api/objects";
import { MEDIA_ACCEPT, OBJECT_TYPE_LABELS, type MapObject, type ObjectMedia } from "../types/object";
const props = defineProps<{
object: MapObject | null;
}>();
const mediaItems = ref<ObjectMedia[]>([]);
const loadingMedia = ref(false);
const mediaError = ref("");
const uploading = ref(false);
const typeLabel = computed(() =>
props.object ? OBJECT_TYPE_LABELS[props.object.type] : "",
);
const formattedDate = computed(() => {
if (!props.object) return "";
return new Date(props.object.created_at).toLocaleString("ru-RU");
});
function isImage(media: ObjectMedia): boolean {
return media.content_type.startsWith("image/");
}
function isVideo(media: ObjectMedia): boolean {
return media.content_type.startsWith("video/");
}
async function loadMedia() {
if (!props.object) {
mediaItems.value = [];
return;
}
loadingMedia.value = true;
mediaError.value = "";
try {
mediaItems.value = await fetchObjectMedia(props.object.id);
} catch (err) {
mediaError.value = err instanceof Error ? err.message : "Не удалось загрузить медиа";
mediaItems.value = [];
} finally {
loadingMedia.value = false;
}
}
async function handleUpload(event: Event) {
if (!props.object) return;
const input = event.target as HTMLInputElement;
const files = input.files ? Array.from(input.files) : [];
input.value = "";
if (files.length === 0) return;
uploading.value = true;
mediaError.value = "";
try {
for (const file of files) {
const uploaded = await uploadObjectMedia(props.object.id, file);
mediaItems.value = [...mediaItems.value, uploaded];
}
} catch (err) {
mediaError.value = err instanceof Error ? err.message : "Не удалось загрузить файл";
} finally {
uploading.value = false;
}
}
async function handleDelete(mediaId: number) {
mediaError.value = "";
try {
await deleteObjectMedia(mediaId);
mediaItems.value = mediaItems.value.filter((item) => item.id !== mediaId);
} catch (err) {
mediaError.value = err instanceof Error ? err.message : "Не удалось удалить файл";
}
}
watch(
() => props.object?.id,
() => loadMedia(),
{ immediate: true },
);
</script>
<template>
<aside class="panel">
<h2>Объект</h2>
<div v-if="!object" class="empty">
<p>Выберите объект на карте, чтобы увидеть описание.</p>
</div>
<div v-else class="content">
<h3>{{ object.name }}</h3>
<dl>
<dt>Тип</dt>
<dd>{{ typeLabel }}</dd>
<dt>Описание</dt>
<dd>{{ object.description || "—" }}</dd>
<dt>Координаты</dt>
<dd>
{{ object.latitude.toFixed(6) }}, {{ object.longitude.toFixed(6) }}
<span class="drag-hint">Перетащите маркер на карте для перемещения</span>
</dd>
<dt>Создан</dt>
<dd>{{ formattedDate }}</dd>
</dl>
<section class="media-section">
<div class="media-header">
<h4>Медиа</h4>
<label class="upload-btn">
{{ uploading ? "Загрузка..." : "Добавить" }}
<input
type="file"
multiple
:accept="MEDIA_ACCEPT"
:disabled="uploading"
hidden
@change="handleUpload"
/>
</label>
</div>
<p v-if="loadingMedia" class="media-status">Загрузка медиа...</p>
<p v-else-if="mediaError" class="media-error">{{ mediaError }}</p>
<p v-else-if="mediaItems.length === 0" class="media-status">Медиафайлы не прикреплены</p>
<ul v-else class="media-list">
<li v-for="item in mediaItems" :key="item.id" class="media-item">
<img v-if="isImage(item)" :src="item.url" :alt="item.original_name" class="preview" />
<video
v-else-if="isVideo(item)"
:src="item.url"
class="preview"
controls
preload="metadata"
/>
<a v-else :href="item.url" class="file-link" target="_blank" rel="noopener">
{{ item.original_name }}
</a>
<div class="media-meta">
<span class="name" :title="item.original_name">{{ item.original_name }}</span>
<span class="size">{{ formatFileSize(item.size) }}</span>
</div>
<button type="button" class="delete-btn" @click="handleDelete(item.id)">
Удалить
</button>
</li>
</ul>
</section>
</div>
</aside>
</template>
<style scoped>
.panel {
position: relative;
z-index: 2;
width: 320px;
flex-shrink: 0;
background: #fff;
border-left: 1px solid #e0e0e0;
padding: 1.25rem;
overflow-y: auto;
}
.panel h2 {
margin: 0 0 1rem;
font-size: 1rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.04em;
color: #666;
}
.empty {
color: #888;
font-size: 0.9rem;
line-height: 1.5;
}
.content h3 {
margin: 0 0 1rem;
font-size: 1.25rem;
}
dl {
margin: 0;
}
dt {
margin-top: 0.75rem;
font-size: 0.75rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.04em;
color: #888;
}
dt:first-child {
margin-top: 0;
}
dd {
margin: 0.25rem 0 0;
line-height: 1.5;
color: #333;
}
.drag-hint {
display: block;
margin-top: 0.25rem;
font-size: 0.75rem;
color: #2563eb;
}
.media-section {
margin-top: 1.5rem;
padding-top: 1rem;
border-top: 1px solid #eee;
}
.media-header {
display: flex;
align-items: center;
justify-content: space-between;
margin-bottom: 0.75rem;
}
.media-header h4 {
margin: 0;
font-size: 0.75rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.04em;
color: #888;
}
.upload-btn {
font-size: 0.8125rem;
font-weight: 500;
color: #2563eb;
cursor: pointer;
}
.upload-btn input:disabled + span,
.upload-btn:has(input:disabled) {
opacity: 0.6;
cursor: not-allowed;
}
.media-status {
margin: 0;
font-size: 0.875rem;
color: #888;
}
.media-error {
margin: 0;
font-size: 0.875rem;
color: #c62828;
}
.media-list {
list-style: none;
margin: 0;
padding: 0;
display: flex;
flex-direction: column;
gap: 0.75rem;
}
.media-item {
border: 1px solid #e8e8e8;
border-radius: 6px;
overflow: hidden;
background: #fafafa;
}
.preview {
display: block;
width: 100%;
max-height: 160px;
object-fit: cover;
background: #000;
}
.file-link {
display: block;
padding: 0.75rem;
color: #2563eb;
text-decoration: none;
word-break: break-all;
}
.media-meta {
display: flex;
justify-content: space-between;
gap: 0.5rem;
padding: 0.5rem 0.75rem 0;
font-size: 0.75rem;
color: #666;
}
.name {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.delete-btn {
display: block;
width: calc(100% - 1rem);
margin: 0.5rem auto 0.5rem;
padding: 0.375rem 0.5rem;
border: none;
border-radius: 4px;
background: #fee2e2;
color: #b91c1c;
font-size: 0.75rem;
cursor: pointer;
}
</style>
@@ -0,0 +1,199 @@
<script setup lang="ts">
import { computed } from "vue";
import type { MapObject } from "../types/object";
const props = defineProps<{
modelValue: number;
min: number;
max: number;
objects: MapObject[];
visibleCount: number;
}>();
const emit = defineEmits<{
"update:modelValue": [value: number];
}>();
const range = computed({
get: () => props.modelValue,
set: (value) => emit("update:modelValue", value),
});
const formattedTime = computed(() =>
new Date(props.modelValue).toLocaleString("ru-RU", {
day: "2-digit",
month: "short",
year: "numeric",
hour: "2-digit",
minute: "2-digit",
}),
);
const formattedMin = computed(() =>
new Date(props.min).toLocaleDateString("ru-RU", {
day: "2-digit",
month: "short",
year: "numeric",
}),
);
const formattedMax = computed(() =>
new Date(props.max).toLocaleDateString("ru-RU", {
day: "2-digit",
month: "short",
year: "numeric",
}),
);
const markers = computed(() => {
if (props.max <= props.min) return [];
const span = props.max - props.min;
return props.objects.map((obj) => {
const time = new Date(obj.created_at).getTime();
const percent = ((time - props.min) / span) * 100;
return {
id: obj.id,
name: obj.name,
percent: Math.min(100, Math.max(0, percent)),
visible: time <= props.modelValue,
};
});
});
const hasRange = computed(() => props.max > props.min);
</script>
<template>
<footer class="timeline">
<div class="timeline-header">
<span class="label">Таймлайн появления</span>
<span class="current-time">{{ formattedTime }}</span>
<span class="count">{{ visibleCount }} / {{ objects.length }} объектов</span>
</div>
<div class="slider-wrap">
<span class="edge-label">{{ formattedMin }}</span>
<div class="slider-track">
<div
v-for="marker in markers"
:key="marker.id"
class="object-marker"
:class="{ visible: marker.visible }"
:style="{ left: `${marker.percent}%` }"
:title="marker.name"
/>
<input
v-model.number="range"
class="slider"
type="range"
:min="min"
:max="max"
:step="hasRange ? Math.max(1, Math.floor((max - min) / 500)) : 1"
:disabled="!hasRange"
/>
</div>
<span class="edge-label">{{ formattedMax }}</span>
</div>
</footer>
</template>
<style scoped>
.timeline {
position: relative;
z-index: 3;
flex-shrink: 0;
padding: 0.75rem 1.25rem 1rem;
background: #fff;
border-top: 1px solid #e0e0e0;
}
.timeline-header {
display: flex;
align-items: center;
gap: 1rem;
margin-bottom: 0.625rem;
font-size: 0.8125rem;
}
.label {
font-weight: 600;
color: #333;
}
.current-time {
color: #2563eb;
font-weight: 500;
}
.count {
margin-left: auto;
color: #666;
}
.slider-wrap {
display: flex;
align-items: center;
gap: 0.75rem;
}
.edge-label {
flex-shrink: 0;
width: 5.5rem;
font-size: 0.75rem;
color: #888;
text-align: center;
}
.edge-label:first-child {
text-align: left;
}
.edge-label:last-child {
text-align: right;
}
.slider-track {
position: relative;
flex: 1;
height: 2rem;
display: flex;
align-items: center;
}
.object-marker {
position: absolute;
top: 50%;
width: 8px;
height: 8px;
margin-left: -4px;
margin-top: -4px;
border-radius: 50%;
background: #bbb;
pointer-events: none;
z-index: 1;
transition: background 0.15s, transform 0.15s;
}
.object-marker.visible {
background: #2563eb;
transform: scale(1.25);
}
.slider {
position: relative;
z-index: 2;
width: 100%;
margin: 0;
cursor: pointer;
accent-color: #2563eb;
}
.slider:disabled {
cursor: default;
opacity: 0.5;
}
</style>
+5
View File
@@ -0,0 +1,5 @@
import { createApp } from "vue";
import App from "./App.vue";
import "./style.css";
createApp(App).mount("#app");
+22
View File
@@ -0,0 +1,22 @@
*,
*::before,
*::after {
box-sizing: border-box;
}
html,
body,
#app {
margin: 0;
height: 100%;
font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
color: #1a1a1a;
background: #f5f5f5;
}
button,
input,
select,
textarea {
font: inherit;
}
@@ -0,0 +1,50 @@
export type ObjectType = "point" | "marker" | "zone" | "other";
export interface MapObject {
id: number;
name: string;
description: string;
type: ObjectType;
latitude: number;
longitude: number;
created_at: string;
}
export interface MapObjectCreate {
name: string;
description: string;
type: ObjectType;
latitude: number;
longitude: number;
created_at?: string;
}
export interface MapObjectUpdate {
name?: string;
description?: string;
type?: ObjectType;
latitude?: number;
longitude?: number;
created_at?: string;
}
export interface ObjectMedia {
id: number;
object_id: number;
original_name: string;
content_type: string;
size: number;
created_at: string;
url: string;
}
export const MEDIA_ACCEPT = "image/jpeg,image/png,image/gif,image/webp,video/mp4,video/webm";
export const OBJECT_TYPE_LABELS: Record<ObjectType, string> = {
point: "Точка",
marker: "Метка",
zone: "Зона",
other: "Другое",
};
export const OBJECT_TYPES: ObjectType[] = ["point", "marker", "zone", "other"];
+7
View File
@@ -0,0 +1,7 @@
/// <reference types="vite/client" />
declare module "*.vue" {
import type { DefineComponent } from "vue";
const component: DefineComponent<Record<string, unknown>, Record<string, unknown>, unknown>;
export default component;
}
@@ -0,0 +1,8 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"composite": true,
"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo"
},
"include": ["src/**/*.ts", "src/**/*.vue"]
}
+21
View File
@@ -0,0 +1,21 @@
{
"compilerOptions": {
"target": "ES2020",
"useDefineForClassFields": true,
"module": "ESNext",
"lib": ["ES2020", "DOM", "DOM.Iterable"],
"skipLibCheck": true,
"moduleResolution": "bundler",
"allowImportingTsExtensions": true,
"isolatedModules": true,
"moduleDetection": "force",
"noEmit": true,
"jsx": "preserve",
"strict": true,
"noUnusedLocals": true,
"noUnusedParameters": true,
"noFallthroughCasesInSwitch": true,
"noUncheckedSideEffectImports": true
},
"include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.vue"]
}
@@ -0,0 +1,10 @@
{
"compilerOptions": {
"composite": true,
"skipLibCheck": true,
"module": "ESNext",
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true
},
"include": ["vite.config.ts"]
}
+11
View File
@@ -0,0 +1,11 @@
import { defineConfig } from "vite";
import vue from "@vitejs/plugin-vue";
export default defineConfig({
plugins: [vue()],
server: {
proxy: {
"/api": "http://localhost:8000",
},
},
});
+13
View File
@@ -0,0 +1,13 @@
FROM python:3.12-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY workers/ ./workers/
COPY worker.py .
ENV PYTHONPATH=/app
CMD ["python", "worker.py"]
+6
View File
@@ -0,0 +1,6 @@
httpx==0.28.1
redis==5.2.1
telethon==1.44.0
python-socks[asyncio]==2.7.1
beautifulsoup4==4.12.3
lxml==5.3.0
+132
View File
@@ -0,0 +1,132 @@
"""CP worker: poll Redis jobs, parse Telegram, ingest to CA."""
import asyncio
import json
import logging
import os
import time
import httpx
import redis
from workers.converter import event_record_to_ingest
from workers.parsers.telegram_events import parse_event_posts
from workers.sources.telegram_client import (
TelegramAuthError,
TelegramConfigError,
fetch_channel_posts,
normalize_channel,
)
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logger = logging.getLogger("cp-worker")
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
JOB_QUEUE_KEY = "cp:jobs"
CA_API_URL = os.getenv("CA_API_URL", "http://ca-api:8000")
INTERNAL_TOKEN = os.getenv("INTERNAL_TOKEN", "dev-internal-token")
POLL_TIMEOUT = int(os.getenv("WORKER_POLL_TIMEOUT", "5"))
def get_redis() -> redis.Redis:
return redis.from_url(REDIS_URL, decode_responses=True)
async def process_telegram_job(job_id: int, source_config: dict) -> tuple[list[dict], str | None]:
channel = source_config.get("channel", "creamy_caprice")
limit = int(source_config.get("limit", 100))
try:
username = normalize_channel(channel)
posts = await fetch_channel_posts(username, limit=limit)
except (TelegramConfigError, TelegramAuthError, ValueError) as exc:
return [], str(exc)
except Exception as exc:
return [], f"Telegram: {exc}"
records = parse_event_posts(posts)
events = [event_record_to_ingest(r) for r in records]
return events, None
async def post_ingest(job_id: int, events: list[dict]) -> None:
if not events:
return
async with httpx.AsyncClient(timeout=120.0) as client:
response = await client.post(
f"{CA_API_URL}/internal/ingest",
json={"job_id": job_id, "events": events},
headers={"X-Internal-Token": INTERNAL_TOKEN},
)
response.raise_for_status()
logger.info("Ingested %s events for job %s: %s", len(events), job_id, response.json())
async def patch_job_status(job_id: int, status: str, error: str | None = None) -> None:
async with httpx.AsyncClient(timeout=30.0) as client:
params = {"status": status}
if error:
params["error"] = error
response = await client.patch(
f"{CA_API_URL}/internal/jobs/{job_id}",
params=params,
headers={"X-Internal-Token": INTERNAL_TOKEN},
)
response.raise_for_status()
async def handle_job(payload: dict) -> None:
job_id = payload["job_id"]
source_type = payload["source_type"]
source_config = payload.get("source_config", {})
logger.info("Processing job %s (%s)", job_id, source_type)
await patch_job_status(job_id, "running")
if source_type == "telegram":
events, error = await process_telegram_job(job_id, source_config)
else:
events, error = [], f"Unsupported source_type: {source_type}"
if error:
logger.error("Job %s failed: %s", job_id, error)
await patch_job_status(job_id, "failed", error=error)
return
try:
await post_ingest(job_id, events)
if not events:
await patch_job_status(job_id, "completed", error="No events found")
logger.info("Job %s completed with %s events", job_id, len(events))
except Exception as exc:
logger.exception("Ingest failed for job %s", job_id)
await patch_job_status(job_id, "failed", error=str(exc))
async def worker_loop() -> None:
r = get_redis()
logger.info("CP worker started, polling %s", JOB_QUEUE_KEY)
while True:
try:
item = r.blpop(JOB_QUEUE_KEY, timeout=POLL_TIMEOUT)
if not item:
continue
_, raw = item
payload = json.loads(raw)
await handle_job(payload)
except redis.RedisError as exc:
logger.error("Redis error: %s", exc)
time.sleep(3)
except Exception:
logger.exception("Unexpected worker error")
time.sleep(1)
def main() -> None:
asyncio.run(worker_loop())
if __name__ == "__main__":
main()
@@ -0,0 +1,50 @@
"""Convert EventRecord to CA ingest payload items."""
import re
from datetime import datetime, timezone
from workers.parsers.telegram_events import EventRecord
COORDS_RE = re.compile(
r"(-?\d{1,3}\.\d+)\s*,\s*(-?\d{1,3}\.\d+)",
)
def parse_event_date(date_str: str) -> datetime | None:
for fmt in ("%d.%m.%y", "%d.%m.%Y"):
try:
dt = datetime.strptime(date_str.strip(), fmt)
return dt.replace(tzinfo=timezone.utc)
except ValueError:
continue
return None
def parse_geolocation(geo: str) -> tuple[float | None, float | None]:
if not geo:
return None, None
match = COORDS_RE.search(geo)
if not match:
return None, None
return float(match.group(1)), float(match.group(2))
def event_record_to_ingest(record: EventRecord) -> dict:
lat, lng = parse_geolocation(record.geolocation)
title = record.locality or (record.event.splitlines()[0][:120] if record.event else "")
event_date = parse_event_date(record.date)
return {
"source_type": "telegram",
"source_url": record.source_url,
"raw_text": record.event,
"title": title,
"description": record.event,
"locality": record.locality,
"latitude": lat,
"longitude": lng,
"event_date": event_date.isoformat() if event_date else None,
"region": record.locality or None,
"topic": "telegram",
"tags": ["telegram"],
"metadata": {"original_date": record.date, "geolocation": record.geolocation},
}
@@ -0,0 +1,92 @@
"""Обход сайта на заданную глубину (BFS)."""
from collections import deque
from dataclasses import dataclass
from urllib.parse import urljoin, urlparse, urlunparse
import httpx
from app.fetcher import fetch_page
SKIP_EXTENSIONS = (
".pdf", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg",
".zip", ".rar", ".mp4", ".mp3", ".avi", ".doc", ".docx",
".xls", ".xlsx", ".css", ".js", ".xml", ".rss",
)
@dataclass
class CrawlResult:
pages: list[tuple[str, str]]
errors: list[str]
def _normalize_url(url: str) -> str:
parsed = urlparse(url)
path = parsed.path or "/"
if path != "/" and path.endswith("/"):
path = path.rstrip("/")
return urlunparse((parsed.scheme, parsed.netloc.lower(), path, "", parsed.query, ""))
def _same_domain(start: str, candidate: str) -> bool:
return urlparse(start).netloc.lower() == urlparse(candidate).netloc.lower()
def _is_fetchable(url: str) -> bool:
parsed = urlparse(url)
if parsed.scheme not in ("http", "https"):
return False
path = parsed.path.lower()
return not any(path.endswith(ext) for ext in SKIP_EXTENSIONS)
async def crawl_site(
start_url: str,
steps: int,
max_pages: int = 100,
timeout: float = 30.0,
) -> CrawlResult:
"""Обойти сайт от start_url на steps уровней (1 = только стартовая страница)."""
steps = max(1, min(steps, 10))
max_pages = max(1, min(max_pages, 200))
start = _normalize_url(start_url)
visited: set[str] = set()
queue: deque[tuple[str, int]] = deque([(start, 0)])
pages: list[tuple[str, str]] = []
errors: list[str] = []
async with httpx.AsyncClient(
follow_redirects=True,
timeout=timeout,
headers={"User-Agent": "SocialParser/1.0"},
) as client:
while queue and len(visited) < max_pages:
url, depth = queue.popleft()
if url in visited:
continue
visited.add(url)
try:
page = await fetch_page(url, client)
except Exception as exc:
errors.append(f"{url}: {exc}")
continue
pages.append((url, page.text))
if depth + 1 >= steps:
continue
for link in page.links:
normalized = _normalize_url(link)
if normalized in visited:
continue
if not _same_domain(start, normalized):
continue
if not _is_fetchable(normalized):
continue
queue.append((normalized, depth + 1))
return CrawlResult(pages=pages, errors=errors)
+106
View File
@@ -0,0 +1,106 @@
"""Загрузка содержимого из источников данных."""
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urljoin, urlparse
import httpx
from bs4 import BeautifulSoup
NOISE_TAGS = ("script", "style", "noscript", "nav", "header", "footer", "aside", "menu")
CONTENT_SELECTORS = (
"article",
"[role='main']",
"main",
".article",
".content",
".post",
".entry-content",
".news-item",
".card-full-news",
".box-card",
)
@dataclass
class PageContent:
text: str
links: list[str]
async def fetch_page(url: str, client: httpx.AsyncClient) -> PageContent:
response = await client.get(url)
response.raise_for_status()
content_type = response.headers.get("content-type", "")
if "html" not in content_type:
return PageContent(text=response.text, links=[])
return _parse_html(response.text, base_url=str(response.url))
def _parse_html(html: str, base_url: str) -> PageContent:
soup = BeautifulSoup(html, "lxml")
links = _extract_links(soup, base_url)
for tag in soup.find_all(NOISE_TAGS):
tag.decompose()
chunks: list[str] = []
for selector in CONTENT_SELECTORS:
for node in soup.select(selector):
text = node.get_text(separator=" ", strip=True)
if len(text) > 80:
chunks.append(text)
if chunks:
return PageContent(text=" ".join(chunks), links=links)
body = soup.body or soup
return PageContent(text=body.get_text(separator=" ", strip=True), links=links)
def _extract_links(soup: BeautifulSoup, base_url: str) -> list[str]:
found: list[str] = []
seen: set[str] = set()
base_host = urlparse(base_url).netloc.lower()
for tag in soup.find_all("a", href=True):
href = tag["href"].strip()
if not href or href.startswith(("#", "mailto:", "tel:", "javascript:")):
continue
absolute = urljoin(base_url, href)
parsed = urlparse(absolute)
if parsed.scheme not in ("http", "https"):
continue
if parsed.netloc.lower() != base_host:
continue
clean = absolute.split("#", 1)[0]
if clean and clean not in seen:
seen.add(clean)
found.append(clean)
return found
async def fetch_url(url: str, timeout: float = 30.0) -> str:
async with httpx.AsyncClient(
follow_redirects=True,
timeout=timeout,
headers={"User-Agent": "SocialParser/1.0"},
) as client:
page = await fetch_page(url, client)
return page.text
def read_file(path: str) -> str:
file_path = Path(path)
if not file_path.exists():
raise FileNotFoundError(f"Файл не найден: {path}")
for encoding in ("utf-8", "cp1251", "latin-1"):
try:
return file_path.read_text(encoding=encoding)
except UnicodeDecodeError:
continue
raise ValueError(f"Не удалось прочитать файл: {path}")
+13
View File
@@ -0,0 +1,13 @@
"""Общие модели данных."""
from dataclasses import dataclass
from datetime import datetime
@dataclass
class TelegramPost:
id: int
text: str
date: datetime
url: str
channel: str
+11
View File
@@ -0,0 +1,11 @@
"""Словари имён и стоп-слов."""
FIRST_NAMES = frozenset(['александр', 'александра', 'алексей', 'алина', 'алиса', 'альбина', 'анастасия', 'анатолий', 'андрей', 'анжела', 'анна', 'антон', 'антонина', 'аркадий', 'арсений', 'артем', 'артём', 'богдан', 'борис', 'вадим', 'валентин', 'валентина', 'валерий', 'валерия', 'василий', 'вера', 'вероника', 'виктор', 'виктория', 'виталий', 'владимир', 'владислав', 'вова', 'вячеслав', 'галина', 'геннадий', 'георгий', 'григорий', 'даниил', 'данила', 'дарья', 'денис', 'диана', 'дима', 'дмитрий', 'евгений', 'евгения', 'екатерина', 'елена', 'елизавета', 'жанна', 'зоя', 'иван', 'игорь', 'инесса', 'инна', 'ира', 'ирина', 'катя', 'кирилл', 'коля', 'константин', 'ксения', 'лариса', 'леонид', 'лидия', 'лилия', 'любовь', 'людмила', 'максим', 'маргарита', 'марина', 'мария', 'матвей', 'маша', 'милана', 'михаил', 'надежда', 'настя', 'наталья', 'никита', 'николай', 'нина', 'олег', 'ольга', 'оля', 'павел', 'паша', 'петр', 'полина', 'пётр', 'раиса', 'регина', 'рената', 'роман', 'руслан', 'саша', 'света', 'светлана', 'сергей', 'серёжа', 'софия', 'софья', 'станислав', 'степан', 'тамара', 'таня', 'татьяна', 'тимофей', 'тимур', 'фаина', 'федор', 'фёдор', 'эдуард', 'эльвира', 'эмма', 'юлия', 'юрий', 'яна', 'ярослав'])
FIRST_NAMES_GENITIVE = frozenset(['александра', 'александры', 'алексея', 'анатолия', 'андрея', 'анны', 'артема', 'артёма', 'бориса', 'вадима', 'валерия', 'василия', 'виктора', 'виталия', 'владимира', 'вячеслава', 'геннадия', 'георгия', 'григория', 'данила', 'дмитрия', 'екатерины', 'елены', 'ефима', 'ивана', 'игоря', 'кирилла', 'константина', 'леонида', 'максима', 'марии', 'михаила', 'натальи', 'николая', 'олега', 'ольги', 'павла', 'петра', 'романа', 'руслана', 'сергея', 'станислава', 'степана', 'татьяны', 'тимофея', 'тимура', 'федора', 'филиппа', 'фёдора', 'эдуарда', 'юрия', 'якова', 'ярослава'])
STOPWORDS = frozenset(['авто', 'архив', 'афера', 'беларуси', 'блогер', 'блогерша', 'блогеры', 'брянской', 'вакансии', 'видео', 'войти', 'время', 'все', 'вчера', 'главная', 'главное', 'год', 'город', 'далее', 'единое', 'журналист', 'журналисты', 'исследования', 'источник', 'источники', 'камчатке', 'комментарии', 'контакты', 'корреспондент', 'криминал', 'круг', 'культура', 'лента', 'лучшее', 'материалы', 'меню', 'место', 'месяц', 'мир', 'москве', 'неделя', 'новости', 'область', 'офис', 'партнеров', 'партнёров', 'подписаться', 'подробнее', 'поиск', 'полиция', 'популярное', 'преступная', 'путешествия', 'район', 'редакция', 'реклама', 'россии', 'россия', 'рубрика', 'рубрики', 'свежее', 'свобода', 'северный', 'сегодня', 'силовые', 'следствие', 'смотреть', 'спецпроекты', 'спорт', 'статьи', 'техподдержка', 'топ', 'украине', 'улица', 'фото', 'ценности', 'читать', 'шурыгина', 'экономика', 'эксклюзивы'])
PREPOSITIONS = frozenset(['без', 'в', 'для', 'до', 'за', 'из', 'к', 'на', 'над', 'о', 'об', 'от', 'по', 'под', 'при', 'про', 'с', 'у'])
ADJECTIVE_ENDINGS = ("ый", "ий", "ая", "яя", "ое", "ее", "ие", "ые", "ой", "ей")
+205
View File
@@ -0,0 +1,205 @@
"""Извлечение ФИО из текста."""
import re
from dataclasses import dataclass
from app.names import ADJECTIVE_ENDINGS, FIRST_NAMES, FIRST_NAMES_GENITIVE, PREPOSITIONS, STOPWORDS
PATRONYMIC_SUFFIXES = (
"ович", "евич", "овна", "евна", "ична", "инична", "оглы", "кызы",
)
SURNAME_SUFFIXES = (
"ов", "ев", "ёв", "ин", "ын", "ский", "ская", "цкий", "цкая",
"енко", "ук", "юк", "ко", "ич", "як", "ова", "ева", "ёва", "ина",
)
NON_NAME_ENDINGS = ("ция", "ство", "ение", "ание", "ура", "ика", "ник", "тель", "изм", "ист")
CYRILLIC_WORD = r"[А-ЯЁ][а-яё]+(?:-[А-ЯЁ][а-яё]+)?"
INITIAL = r"[А-ЯЁ]\."
FULL_FIO_RE = re.compile(
rf"\b({CYRILLIC_WORD})\s+({CYRILLIC_WORD})\s+({CYRILLIC_WORD})\b"
)
SHORT_FIO_RE = re.compile(
rf"\b({CYRILLIC_WORD})\s+({INITIAL})\s*({INITIAL})\b"
)
TWO_WORD_RE = re.compile(
rf"\b({CYRILLIC_WORD})\s+({CYRILLIC_WORD})\b"
)
@dataclass(frozen=True)
class Person:
surname: str
name: str
patronymic: str
source: str
raw: str
@property
def full_name(self) -> str:
parts = [self.surname, self.name]
if self.patronymic:
parts.append(self.patronymic)
return " ".join(parts)
def _norm(word: str) -> str:
return word.lower().replace("ё", "е")
def _looks_like_patronymic(word: str) -> bool:
lower = _norm(word)
return any(lower.endswith(s.replace("ё", "е")) for s in PATRONYMIC_SUFFIXES)
def _looks_like_surname(word: str) -> bool:
lower = _norm(word)
if len(lower) < 4:
return False
return any(lower.endswith(s.replace("ё", "е")) for s in SURNAME_SUFFIXES)
def _looks_like_adjective(word: str) -> bool:
if _is_known_first_name(word):
return False
lower = _norm(word)
return any(lower.endswith(e) for e in ADJECTIVE_ENDINGS)
def _is_known_first_name(word: str) -> bool:
lower = _norm(word)
return lower in FIRST_NAMES or lower in FIRST_NAMES_GENITIVE
def _is_blocked_word(word: str) -> bool:
lower = _norm(word)
if len(lower) < 2:
return True
if lower in STOPWORDS or lower in PREPOSITIONS:
return True
if _looks_like_adjective(word):
return True
return any(lower.endswith(e) for e in NON_NAME_ENDINGS)
def _is_initial(word: str) -> bool:
return len(word) <= 3 and word.rstrip(".").isalpha() and len(word.rstrip(".")) == 1
def _validate_person(surname: str, name: str, patronymic: str) -> bool:
parts = [surname, name, patronymic]
if any(_is_blocked_word(w) for w in parts if w):
return False
if patronymic:
if _is_initial(patronymic) or _is_initial(name):
return _looks_like_surname(surname) and not _is_blocked_word(surname)
if not _looks_like_patronymic(patronymic):
return False
if not _is_known_first_name(name):
return False
if not (_looks_like_surname(surname) or _is_known_first_name(surname)):
return False
return True
if _looks_like_patronymic(name):
return _looks_like_surname(surname) and not _is_blocked_word(name)
if not _is_known_first_name(name):
return False
if not _looks_like_surname(surname):
return False
return True
def _parse_full_fio(w1: str, w2: str, w3: str) -> Person | None:
if _looks_like_patronymic(w3):
person = Person(w1, w2, w3, "", f"{w1} {w2} {w3}")
elif _looks_like_patronymic(w2) and _looks_like_surname(w3):
person = Person(w3, w1, w2, "", f"{w1} {w2} {w3}")
elif _looks_like_surname(w1) and _is_known_first_name(w2) and _looks_like_patronymic(w3):
person = Person(w1, w2, w3, "", f"{w1} {w2} {w3}")
else:
return None
if _validate_person(person.surname, person.name, person.patronymic):
return person
return None
def _parse_two_words(w1: str, w2: str) -> Person | None:
if _looks_like_patronymic(w1) or _looks_like_patronymic(w2):
return None
if _looks_like_surname(w1) and _is_known_first_name(w2):
person = Person(w1, w2, "", "", f"{w1} {w2}")
elif _looks_like_surname(w2) and _is_known_first_name(w1):
person = Person(w2, w1, "", "", f"{w1} {w2}")
else:
return None
if _validate_person(person.surname, person.name, person.patronymic):
return person
return None
def extract_fio(text: str, source: str = "") -> list[Person]:
"""Найти все ФИО в тексте."""
seen: set[str] = set()
results: list[Person] = []
occupied: list[tuple[int, int]] = []
def overlaps(start: int, end: int) -> bool:
return any(not (end <= s or start >= e) for s, e in occupied)
def add(person: Person, start: int, end: int) -> None:
key = person.full_name.lower()
if key not in seen and not overlaps(start, end):
seen.add(key)
occupied.append((start, end))
results.append(
Person(
surname=person.surname,
name=person.name,
patronymic=person.patronymic,
source=source,
raw=person.raw,
)
)
for match in FULL_FIO_RE.finditer(text):
w1, w2, w3 = match.groups()
person = _parse_full_fio(w1, w2, w3)
if person:
add(person, *match.span())
for match in SHORT_FIO_RE.finditer(text):
start, end = match.span()
if overlaps(start, end):
continue
surname, i1, i2 = match.groups()
if not _looks_like_surname(surname) or _is_blocked_word(surname):
continue
person = Person(
surname=surname,
name=i1,
patronymic=i2,
source="",
raw=match.group(),
)
if _validate_person(person.surname, person.name, person.patronymic):
add(person, start, end)
for match in TWO_WORD_RE.finditer(text):
start, end = match.span()
if overlaps(start, end):
continue
w1, w2 = match.groups()
person = _parse_two_words(w1, w2)
if person:
add(person, start, end)
return results
@@ -0,0 +1 @@
@@ -0,0 +1,89 @@
"""Парсинг событий из постов Telegram-канала."""
import re
from dataclasses import dataclass
from datetime import datetime
from workers.models import TelegramPost
DATE_LINE_RE = re.compile(
r"^(\d{2}\.\d{2}\.\d{2,4})\s+(.+)$",
re.MULTILINE,
)
COORDS_RE = re.compile(
r"(-?\d{1,3}\.\d+)\s*,\s*(-?\d{1,3}\.\d+)",
)
HASHTAG_LINE_RE = re.compile(r"^#\w+", re.MULTILINE)
@dataclass
class EventRecord:
event: str
date: str
geolocation: str
locality: str
source_url: str
def _format_message_date(dt: datetime) -> str:
return dt.strftime("%d.%m.%y")
def _clean_event_text(text: str) -> str:
lines: list[str] = []
for line in text.splitlines():
stripped = line.strip()
if not stripped:
continue
if stripped.lower() == "источник":
continue
if stripped.startswith("#"):
continue
if COORDS_RE.search(stripped):
continue
lines.append(stripped)
return "\n".join(lines)
def parse_event_post(post: TelegramPost) -> EventRecord:
text = post.text
date = _format_message_date(post.date)
locality = ""
geolocation = ""
event = text
first_line = text.splitlines()[0].strip() if text.splitlines() else ""
header_match = DATE_LINE_RE.match(first_line)
if header_match:
date = header_match.group(1)
locality = header_match.group(2).strip()
coords_match = COORDS_RE.search(text)
if coords_match:
geolocation = f"{coords_match.group(1)}, {coords_match.group(2)}"
if header_match:
body = text[len(first_line):].strip()
else:
body = text
if coords_match:
before, after = body.split(coords_match.group(0), 1)
event_body = before.strip()
else:
event_body = body
event = _clean_event_text(event_body) or _clean_event_text(text) or text
event = HASHTAG_LINE_RE.sub("", event).strip()
return EventRecord(
event=event,
date=date,
geolocation=geolocation,
locality=locality,
source_url=post.url,
)
def parse_event_posts(posts: list[TelegramPost]) -> list[EventRecord]:
return [parse_event_post(post) for post in posts if post.text.strip()]
@@ -0,0 +1 @@
@@ -0,0 +1,99 @@
"""Telegram Client API (Telethon) для чтения постов канала."""
import os
import re
from pathlib import Path
from telethon.errors import AuthKeyUnregisteredError, SessionPasswordNeededError
from workers.models import TelegramPost
from workers.sources.telegram_settings import create_client, get_api_credentials
DEFAULT_SESSION_PATH = "/data/telegram.session"
MAX_POSTS = 500
class TelegramConfigError(Exception):
pass
class TelegramAuthError(Exception):
pass
def _get_config() -> tuple[int, str, str]:
api_id, api_hash = get_api_credentials()
session_path = os.environ.get("TELEGRAM_SESSION_PATH", DEFAULT_SESSION_PATH)
return api_id, api_hash, session_path
def normalize_channel(raw: str) -> str:
value = raw.strip()
if not value:
raise ValueError("Укажите канал Telegram")
value = value.replace("https://", "").replace("http://", "")
value = re.sub(r"^web\.telegram\.org/k/#@", "", value)
value = re.sub(r"^t\.me/", "", value)
value = value.lstrip("@").split("/")[0].split("?")[0]
if not value:
raise ValueError("Некорректное имя канала")
return value
def _build_post_url(channel: str, message_id: int) -> str:
username = channel.lstrip("@")
return f"https://t.me/{username}/{message_id}"
async def fetch_channel_posts(channel: str, limit: int = 100) -> list[TelegramPost]:
limit = max(1, min(limit, MAX_POSTS))
try:
api_id, api_hash, session_path = _get_config()
except ValueError as exc:
raise TelegramConfigError(str(exc)) from exc
username = normalize_channel(channel)
if not Path(session_path).exists():
raise TelegramAuthError(
f"Файл сессии не найден: {session_path}. "
"Выполните: python scripts/telegram_auth.py"
)
client = create_client(session_path, api_id, api_hash)
posts: list[TelegramPost] = []
try:
await client.connect()
if not await client.is_user_authorized():
raise TelegramAuthError(
"Telegram-сессия не авторизована. Выполните: python scripts/telegram_auth.py"
)
entity = await client.get_entity(username)
async for message in client.iter_messages(entity, limit=limit):
if not message.text:
continue
posts.append(
TelegramPost(
id=message.id,
text=message.text.strip(),
date=message.date,
url=_build_post_url(username, message.id),
channel=username,
)
)
except AuthKeyUnregisteredError as exc:
raise TelegramAuthError(
"Сессия Telegram недействительна. Переавторизуйтесь: python scripts/telegram_auth.py"
) from exc
except SessionPasswordNeededError as exc:
raise TelegramAuthError(
"Для аккаунта включена двухфакторная аутентификация. "
"Авторизуйтесь через scripts/telegram_auth.py с паролем 2FA."
) from exc
finally:
await client.disconnect()
return posts
@@ -0,0 +1,69 @@
"""Общие настройки подключения Telethon."""
import os
from telethon import TelegramClient
DEFAULT_SESSION_PATH = "/data/telegram.session"
def get_session_path() -> str:
return os.environ.get("TELEGRAM_SESSION_PATH", DEFAULT_SESSION_PATH)
def get_api_credentials() -> tuple[int, str]:
api_id_raw = os.environ.get("TELEGRAM_API_ID", "")
api_hash = os.environ.get("TELEGRAM_API_HASH", "")
if not api_id_raw or not api_hash:
raise ValueError(
"Не заданы TELEGRAM_API_ID и TELEGRAM_API_HASH в .env"
)
try:
api_id = int(api_id_raw)
except ValueError as exc:
raise ValueError("TELEGRAM_API_ID должен быть числом") from exc
return api_id, api_hash
def get_proxy() -> tuple | None:
proxy_type = os.environ.get("TELEGRAM_PROXY_TYPE", "").strip().lower()
if not proxy_type or proxy_type == "none":
return None
host = os.environ.get("TELEGRAM_PROXY_HOST", "127.0.0.1").strip()
port = int(os.environ.get("TELEGRAM_PROXY_PORT", "1080"))
user = os.environ.get("TELEGRAM_PROXY_USER", "").strip()
password = os.environ.get("TELEGRAM_PROXY_PASS", "").strip()
if proxy_type not in ("socks5", "socks4", "http"):
raise ValueError(
f"Неподдерживаемый TELEGRAM_PROXY_TYPE={proxy_type!r}. "
"Используйте socks5, socks4 или http."
)
if user:
return proxy_type, host, port, True, user, password
return proxy_type, host, port
def describe_connection() -> str:
proxy = get_proxy()
if not proxy:
return "напрямую (без прокси)"
return f"через {proxy[0]}://{proxy[1]}:{proxy[2]}"
def create_client(session_path: str, api_id: int, api_hash: str) -> TelegramClient:
kwargs = {
"connection_retries": 10,
"retry_delay": 3,
"timeout": 60,
"request_retries": 5,
}
proxy = get_proxy()
if proxy:
kwargs["proxy"] = proxy
return TelegramClient(session_path, api_id, api_hash, **kwargs)