Add CP→CA→PI platform foundation on MapMil monorepo.
Unify parsing workers, analytics API with PostgreSQL, map UI, and PI distribution into centers/ with Docker Compose. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Cursor
@@ -0,0 +1,28 @@
|
||||
import os
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import DeclarativeBase, sessionmaker
|
||||
|
||||
DATABASE_URL = os.getenv(
|
||||
"DATABASE_URL",
|
||||
"postgresql://mapmil:mapmil@ca-db:5432/mapmil",
|
||||
)
|
||||
|
||||
connect_args: dict = {}
|
||||
if DATABASE_URL.startswith("sqlite"):
|
||||
connect_args = {"check_same_thread": False}
|
||||
|
||||
engine = create_engine(DATABASE_URL, connect_args=connect_args)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,11 @@
|
||||
import os
|
||||
|
||||
from fastapi import Header, HTTPException
|
||||
|
||||
|
||||
def verify_internal_token(
|
||||
x_internal_token: str | None = Header(default=None, alias="X-Internal-Token"),
|
||||
) -> None:
|
||||
expected = os.getenv("INTERNAL_TOKEN", "dev-internal-token")
|
||||
if not x_internal_token or x_internal_token != expected:
|
||||
raise HTTPException(status_code=401, detail="Invalid internal token")
|
||||
@@ -0,0 +1,38 @@
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from .database import Base, engine, get_db
|
||||
from .routers import admin, internal, objects, v1
|
||||
from .seed import seed_objects, seed_test_consumer
|
||||
from .storage import ensure_upload_dir
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
ensure_upload_dir()
|
||||
Base.metadata.create_all(bind=engine)
|
||||
db = next(get_db())
|
||||
try:
|
||||
seed_objects(db)
|
||||
seed_test_consumer(db)
|
||||
finally:
|
||||
db.close()
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(title="CA API (Analytics Center)", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(objects.router)
|
||||
app.include_router(internal.router)
|
||||
app.include_router(admin.router)
|
||||
app.include_router(v1.router)
|
||||
@@ -0,0 +1,145 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
Boolean,
|
||||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from .database import Base
|
||||
|
||||
|
||||
class MapObject(Base):
|
||||
__tablename__ = "map_objects"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
description: Mapped[str] = mapped_column(Text, default="")
|
||||
type: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
latitude: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
longitude: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
event_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("events.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
media: Mapped[list["ObjectMedia"]] = relationship(
|
||||
back_populates="object",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
event: Mapped["Event | None"] = relationship(back_populates="map_object")
|
||||
|
||||
|
||||
class ObjectMedia(Base):
|
||||
__tablename__ = "object_media"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
object_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("map_objects.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
stored_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
original_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
content_type: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
size: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
object: Mapped["MapObject"] = relationship(back_populates="media")
|
||||
|
||||
|
||||
class Event(Base):
|
||||
__tablename__ = "events"
|
||||
__table_args__ = (UniqueConstraint("source_url", name="uq_events_source_url"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
source_type: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
||||
source_url: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
raw_text: Mapped[str] = mapped_column(Text, default="")
|
||||
title: Mapped[str] = mapped_column(String(512), default="")
|
||||
description: Mapped[str] = mapped_column(Text, default="")
|
||||
locality: Mapped[str] = mapped_column(String(255), default="")
|
||||
latitude: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
longitude: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
event_date: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
ingested_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
index=True,
|
||||
)
|
||||
region: Mapped[str | None] = mapped_column(String(100), nullable=True, index=True)
|
||||
topic: Mapped[str | None] = mapped_column(String(100), nullable=True, index=True)
|
||||
tags: Mapped[list | None] = mapped_column(JSON, nullable=True)
|
||||
metadata_: Mapped[dict | None] = mapped_column("metadata", JSON, nullable=True)
|
||||
|
||||
map_object: Mapped["MapObject | None"] = relationship(
|
||||
back_populates="event",
|
||||
uselist=False,
|
||||
)
|
||||
|
||||
|
||||
class ParseJob(Base):
|
||||
__tablename__ = "parse_jobs"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
source_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
source_config: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
schedule: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(50), default="pending", index=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
class Consumer(Base):
|
||||
__tablename__ = "consumers"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
api_key_hash: Mapped[str] = mapped_column(String(64), nullable=False, unique=True)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
filter: Mapped["ConsumerFilter | None"] = relationship(
|
||||
back_populates="consumer",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
|
||||
class ConsumerFilter(Base):
|
||||
__tablename__ = "consumer_filters"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, index=True)
|
||||
consumer_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("consumers.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
regions: Mapped[list | None] = mapped_column(JSON, nullable=True)
|
||||
topics: Mapped[list | None] = mapped_column(JSON, nullable=True)
|
||||
min_access_level: Mapped[int] = mapped_column(Integer, default=0)
|
||||
date_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
consumer: Mapped["Consumer"] = relationship(back_populates="filter")
|
||||
@@ -0,0 +1,78 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import get_db
|
||||
from ..models import Event, ParseJob
|
||||
from ..schemas import (
|
||||
ConsumerCreate,
|
||||
ConsumerRead,
|
||||
EventRead,
|
||||
ParseJobCreate,
|
||||
ParseJobRead,
|
||||
)
|
||||
from ..services.filtering import create_consumer
|
||||
from ..services.jobs import enqueue_job
|
||||
|
||||
router = APIRouter(prefix="/admin", tags=["admin"])
|
||||
|
||||
|
||||
@router.post("/jobs", response_model=ParseJobRead, status_code=201)
|
||||
def create_parse_job(payload: ParseJobCreate, db: Session = Depends(get_db)):
|
||||
job = ParseJob(
|
||||
source_type=payload.source_type,
|
||||
source_config=payload.source_config,
|
||||
schedule=payload.schedule,
|
||||
status="queued",
|
||||
)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
db.refresh(job)
|
||||
|
||||
enqueue_job(job.id, job.source_type, job.source_config)
|
||||
return job
|
||||
|
||||
|
||||
@router.get("/jobs", response_model=list[ParseJobRead])
|
||||
def list_parse_jobs(db: Session = Depends(get_db)):
|
||||
return db.query(ParseJob).order_by(ParseJob.id.desc()).all()
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}", response_model=ParseJobRead)
|
||||
def get_parse_job(job_id: int, db: Session = Depends(get_db)):
|
||||
job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
return job
|
||||
|
||||
|
||||
@router.get("/events", response_model=list[EventRead])
|
||||
def list_events(
|
||||
limit: int = Query(default=100, ge=1, le=1000),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
return (
|
||||
db.query(Event)
|
||||
.order_by(Event.ingested_at.desc())
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
@router.post("/consumers", response_model=ConsumerRead, status_code=201)
|
||||
def create_pi_consumer(payload: ConsumerCreate, db: Session = Depends(get_db)):
|
||||
consumer, api_key = create_consumer(
|
||||
db,
|
||||
name=payload.name,
|
||||
regions=payload.regions,
|
||||
topics=payload.topics,
|
||||
date_from=payload.date_from,
|
||||
)
|
||||
return ConsumerRead(
|
||||
id=consumer.id,
|
||||
name=consumer.name,
|
||||
is_active=consumer.is_active,
|
||||
created_at=consumer.created_at,
|
||||
api_key=api_key,
|
||||
)
|
||||
@@ -0,0 +1,45 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import get_db
|
||||
from ..deps import verify_internal_token
|
||||
from ..models import ParseJob
|
||||
from ..schemas import IngestRequest, IngestResponse
|
||||
from ..services.ingest import ingest_events
|
||||
|
||||
router = APIRouter(prefix="/internal", tags=["internal"])
|
||||
|
||||
|
||||
@router.post("/ingest", response_model=IngestResponse)
|
||||
def internal_ingest(
|
||||
payload: IngestRequest,
|
||||
_: None = Depends(verify_internal_token),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
ingested, updated, map_synced = ingest_events(db, payload.events, payload.job_id)
|
||||
return IngestResponse(
|
||||
ingested=ingested,
|
||||
updated=updated,
|
||||
map_objects_synced=map_synced,
|
||||
)
|
||||
|
||||
|
||||
@router.patch("/jobs/{job_id}")
|
||||
def update_job_status(
|
||||
job_id: int,
|
||||
status: str,
|
||||
error: str | None = None,
|
||||
_: None = Depends(verify_internal_token),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
|
||||
if not job:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
|
||||
job.status = status
|
||||
job.last_run_at = datetime.now(timezone.utc)
|
||||
job.last_error = error
|
||||
db.commit()
|
||||
return {"id": job.id, "status": job.status}
|
||||
@@ -0,0 +1,169 @@
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import get_db
|
||||
from ..models import MapObject, ObjectMedia
|
||||
from ..schemas import MapObjectCreate, MapObjectRead, MapObjectUpdate, ObjectMediaRead
|
||||
from ..storage import (
|
||||
ALLOWED_CONTENT_TYPES,
|
||||
MAX_FILE_SIZE,
|
||||
build_stored_name,
|
||||
is_allowed_content_type,
|
||||
media_file_path,
|
||||
remove_media_file,
|
||||
)
|
||||
|
||||
router = APIRouter(tags=["objects"])
|
||||
|
||||
|
||||
def media_to_read(media: ObjectMedia) -> ObjectMediaRead:
|
||||
return ObjectMediaRead(
|
||||
id=media.id,
|
||||
object_id=media.object_id,
|
||||
original_name=media.original_name,
|
||||
content_type=media.content_type,
|
||||
size=media.size,
|
||||
created_at=media.created_at,
|
||||
url=f"/api/media/{media.id}/file",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api/health")
|
||||
def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/api/objects", response_model=list[MapObjectRead])
|
||||
def list_objects(db: Session = Depends(get_db)):
|
||||
return db.query(MapObject).order_by(MapObject.id).all()
|
||||
|
||||
|
||||
@router.get("/api/objects/{object_id}", response_model=MapObjectRead)
|
||||
def get_object(object_id: int, db: Session = Depends(get_db)):
|
||||
obj = db.query(MapObject).filter(MapObject.id == object_id).first()
|
||||
if not obj:
|
||||
raise HTTPException(status_code=404, detail="Объект не найден")
|
||||
return obj
|
||||
|
||||
|
||||
@router.post("/api/objects", response_model=MapObjectRead, status_code=201)
|
||||
def create_object(payload: MapObjectCreate, db: Session = Depends(get_db)):
|
||||
data = payload.model_dump()
|
||||
created_at = data.pop("created_at", None)
|
||||
obj = MapObject(**data)
|
||||
if created_at is not None:
|
||||
obj.created_at = created_at
|
||||
db.add(obj)
|
||||
db.commit()
|
||||
db.refresh(obj)
|
||||
return obj
|
||||
|
||||
|
||||
@router.patch("/api/objects/{object_id}", response_model=MapObjectRead)
|
||||
def update_object(
|
||||
object_id: int,
|
||||
payload: MapObjectUpdate,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
obj = db.query(MapObject).filter(MapObject.id == object_id).first()
|
||||
if not obj:
|
||||
raise HTTPException(status_code=404, detail="Объект не найден")
|
||||
|
||||
for field, value in payload.model_dump(exclude_unset=True).items():
|
||||
setattr(obj, field, value)
|
||||
|
||||
db.commit()
|
||||
db.refresh(obj)
|
||||
return obj
|
||||
|
||||
|
||||
@router.delete("/api/objects/{object_id}", status_code=204)
|
||||
def delete_object(object_id: int, db: Session = Depends(get_db)):
|
||||
obj = db.query(MapObject).filter(MapObject.id == object_id).first()
|
||||
if not obj:
|
||||
raise HTTPException(status_code=404, detail="Объект не найден")
|
||||
|
||||
for media in obj.media:
|
||||
remove_media_file(media.stored_name)
|
||||
|
||||
db.delete(obj)
|
||||
db.commit()
|
||||
|
||||
|
||||
@router.get("/api/objects/{object_id}/media", response_model=list[ObjectMediaRead])
|
||||
def list_object_media(object_id: int, db: Session = Depends(get_db)):
|
||||
obj = db.query(MapObject).filter(MapObject.id == object_id).first()
|
||||
if not obj:
|
||||
raise HTTPException(status_code=404, detail="Объект не найден")
|
||||
|
||||
media_items = (
|
||||
db.query(ObjectMedia)
|
||||
.filter(ObjectMedia.object_id == object_id)
|
||||
.order_by(ObjectMedia.id)
|
||||
.all()
|
||||
)
|
||||
return [media_to_read(item) for item in media_items]
|
||||
|
||||
|
||||
@router.post("/api/objects/{object_id}/media", response_model=ObjectMediaRead, status_code=201)
|
||||
async def upload_object_media(
|
||||
object_id: int,
|
||||
file: UploadFile = File(...),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
obj = db.query(MapObject).filter(MapObject.id == object_id).first()
|
||||
if not obj:
|
||||
raise HTTPException(status_code=404, detail="Объект не найден")
|
||||
|
||||
content_type = file.content_type or "application/octet-stream"
|
||||
if not is_allowed_content_type(content_type):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Неподдерживаемый тип файла. Разрешены: {', '.join(sorted(ALLOWED_CONTENT_TYPES))}",
|
||||
)
|
||||
|
||||
content = await file.read()
|
||||
if len(content) > MAX_FILE_SIZE:
|
||||
raise HTTPException(status_code=400, detail="Файл слишком большой (макс. 20 МБ)")
|
||||
|
||||
original_name = file.filename or "file"
|
||||
stored_name = build_stored_name(original_name)
|
||||
path = media_file_path(stored_name)
|
||||
path.write_bytes(content)
|
||||
|
||||
media = ObjectMedia(
|
||||
object_id=object_id,
|
||||
stored_name=stored_name,
|
||||
original_name=original_name,
|
||||
content_type=content_type,
|
||||
size=len(content),
|
||||
)
|
||||
db.add(media)
|
||||
db.commit()
|
||||
db.refresh(media)
|
||||
return media_to_read(media)
|
||||
|
||||
|
||||
@router.get("/api/media/{media_id}/file")
|
||||
def get_media_file(media_id: int, db: Session = Depends(get_db)):
|
||||
media = db.query(ObjectMedia).filter(ObjectMedia.id == media_id).first()
|
||||
if not media:
|
||||
raise HTTPException(status_code=404, detail="Медиафайл не найден")
|
||||
|
||||
path = media_file_path(media.stored_name)
|
||||
if not path.exists():
|
||||
raise HTTPException(status_code=404, detail="Файл не найден на диске")
|
||||
|
||||
return FileResponse(path, media_type=media.content_type, filename=media.original_name)
|
||||
|
||||
|
||||
@router.delete("/api/media/{media_id}", status_code=204)
|
||||
def delete_media(media_id: int, db: Session = Depends(get_db)):
|
||||
media = db.query(ObjectMedia).filter(ObjectMedia.id == media_id).first()
|
||||
if not media:
|
||||
raise HTTPException(status_code=404, detail="Медиафайл не найден")
|
||||
|
||||
remove_media_file(media.stored_name)
|
||||
db.delete(media)
|
||||
db.commit()
|
||||
@@ -0,0 +1,33 @@
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..database import get_db
|
||||
from ..schemas import EventRead
|
||||
from ..services.filtering import apply_consumer_filter, find_consumer_by_api_key
|
||||
|
||||
router = APIRouter(prefix="/api/v1", tags=["distribution"])
|
||||
|
||||
|
||||
def get_consumer_from_api_key(
|
||||
authorization: str | None = Header(default=None),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
if not authorization or not authorization.startswith("Bearer "):
|
||||
raise HTTPException(status_code=401, detail="Missing or invalid Authorization header")
|
||||
|
||||
api_key = authorization.removeprefix("Bearer ").strip()
|
||||
consumer = find_consumer_by_api_key(db, api_key)
|
||||
if not consumer:
|
||||
raise HTTPException(status_code=401, detail="Invalid API key")
|
||||
return consumer
|
||||
|
||||
|
||||
@router.get("/events", response_model=list[EventRead])
|
||||
def list_filtered_events(
|
||||
limit: int = Query(default=100, ge=1, le=1000),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
consumer=Depends(get_consumer_from_api_key),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
events = apply_consumer_filter(db, consumer)
|
||||
return events[offset : offset + limit]
|
||||
@@ -0,0 +1,132 @@
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
ObjectType = Literal["point", "marker", "zone", "other"]
|
||||
|
||||
|
||||
class MapObjectCreate(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=255)
|
||||
description: str = ""
|
||||
type: ObjectType
|
||||
latitude: float = Field(ge=-90, le=90)
|
||||
longitude: float = Field(ge=-180, le=180)
|
||||
created_at: datetime | None = None
|
||||
|
||||
|
||||
class MapObjectRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: int
|
||||
name: str
|
||||
description: str
|
||||
type: ObjectType
|
||||
latitude: float
|
||||
longitude: float
|
||||
event_id: int | None = None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class MapObjectUpdate(BaseModel):
|
||||
name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
description: str | None = None
|
||||
type: ObjectType | None = None
|
||||
latitude: float | None = Field(default=None, ge=-90, le=90)
|
||||
longitude: float | None = Field(default=None, ge=-180, le=180)
|
||||
created_at: datetime | None = None
|
||||
|
||||
|
||||
class ObjectMediaRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: int
|
||||
object_id: int
|
||||
original_name: str
|
||||
content_type: str
|
||||
size: int
|
||||
created_at: datetime
|
||||
url: str
|
||||
|
||||
|
||||
class EventRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: int
|
||||
source_type: str
|
||||
source_url: str
|
||||
raw_text: str
|
||||
title: str
|
||||
description: str
|
||||
locality: str
|
||||
latitude: float | None
|
||||
longitude: float | None
|
||||
event_date: datetime | None
|
||||
ingested_at: datetime
|
||||
region: str | None
|
||||
topic: str | None
|
||||
tags: list[str] | None
|
||||
metadata: dict[str, Any] | None = Field(validation_alias="metadata_")
|
||||
|
||||
|
||||
class IngestEventItem(BaseModel):
|
||||
source_type: str = "telegram"
|
||||
source_url: str
|
||||
raw_text: str = ""
|
||||
title: str = ""
|
||||
description: str = ""
|
||||
locality: str = ""
|
||||
latitude: float | None = None
|
||||
longitude: float | None = None
|
||||
event_date: datetime | None = None
|
||||
region: str | None = None
|
||||
topic: str | None = None
|
||||
tags: list[str] | None = None
|
||||
metadata: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class IngestRequest(BaseModel):
|
||||
job_id: int | None = None
|
||||
events: list[IngestEventItem] = Field(default_factory=list)
|
||||
|
||||
|
||||
class IngestResponse(BaseModel):
|
||||
ingested: int
|
||||
updated: int
|
||||
map_objects_synced: int
|
||||
|
||||
|
||||
class ParseJobCreate(BaseModel):
|
||||
source_type: str = "telegram"
|
||||
source_config: dict[str, Any] = Field(default_factory=dict)
|
||||
schedule: str | None = None
|
||||
|
||||
|
||||
class ParseJobRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: int
|
||||
source_type: str
|
||||
source_config: dict[str, Any]
|
||||
schedule: str | None
|
||||
status: str
|
||||
last_run_at: datetime | None
|
||||
last_error: str | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ConsumerCreate(BaseModel):
|
||||
name: str
|
||||
regions: list[str] | None = None
|
||||
topics: list[str] | None = None
|
||||
date_from: datetime | None = None
|
||||
|
||||
|
||||
class ConsumerRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: int
|
||||
name: str
|
||||
is_active: bool
|
||||
created_at: datetime
|
||||
api_key: str | None = None
|
||||
@@ -0,0 +1,68 @@
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .models import Consumer, ConsumerFilter, MapObject
|
||||
from .services.filtering import hash_api_key
|
||||
|
||||
SEED_OBJECTS = [
|
||||
{
|
||||
"name": "Красная площадь",
|
||||
"description": "Главная площадь Москвы, исторический центр города.",
|
||||
"type": "marker",
|
||||
"latitude": 55.7539,
|
||||
"longitude": 37.6208,
|
||||
"created_at": datetime(2018, 5, 9, 12, 0, tzinfo=timezone.utc),
|
||||
},
|
||||
{
|
||||
"name": "ВДНХ",
|
||||
"description": "Выставка достижений народного хозяйства — крупный выставочный комплекс.",
|
||||
"type": "zone",
|
||||
"latitude": 55.8298,
|
||||
"longitude": 37.6361,
|
||||
"created_at": datetime(2020, 8, 15, 10, 30, tzinfo=timezone.utc),
|
||||
},
|
||||
{
|
||||
"name": "МГУ",
|
||||
"description": "Московский государственный университет имени М.В. Ломоносова.",
|
||||
"type": "point",
|
||||
"latitude": 55.7033,
|
||||
"longitude": 37.5307,
|
||||
"created_at": datetime(2022, 2, 12, 14, 0, tzinfo=timezone.utc),
|
||||
},
|
||||
{
|
||||
"name": "Парк Горького",
|
||||
"description": "Центральный парк культуры и отдыха имени М. Горького.",
|
||||
"type": "other",
|
||||
"latitude": 55.7312,
|
||||
"longitude": 37.6013,
|
||||
"created_at": datetime(2024, 6, 1, 9, 0, tzinfo=timezone.utc),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def seed_objects(db: Session) -> None:
|
||||
if db.query(MapObject).count() > 0:
|
||||
return
|
||||
|
||||
for item in SEED_OBJECTS:
|
||||
db.add(MapObject(**item))
|
||||
|
||||
db.commit()
|
||||
|
||||
|
||||
def seed_test_consumer(db: Session) -> None:
|
||||
if db.query(Consumer).filter(Consumer.name == "test-pi").first():
|
||||
return
|
||||
|
||||
api_key = os.getenv("TEST_PI_API_KEY", "test-pi-api-key-change-me")
|
||||
consumer = Consumer(
|
||||
name="test-pi",
|
||||
api_key_hash=hash_api_key(api_key),
|
||||
is_active=True,
|
||||
)
|
||||
db.add(consumer)
|
||||
db.flush()
|
||||
db.add(ConsumerFilter(consumer_id=consumer.id, regions=None, topics=None))
|
||||
db.commit()
|
||||
@@ -0,0 +1,109 @@
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Consumer, ConsumerFilter, Event, MapObject
|
||||
|
||||
|
||||
def hash_api_key(api_key: str) -> str:
|
||||
return hashlib.sha256(api_key.encode()).hexdigest()
|
||||
|
||||
|
||||
def generate_api_key() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def create_consumer(
|
||||
db: Session,
|
||||
name: str,
|
||||
regions: list[str] | None = None,
|
||||
topics: list[str] | None = None,
|
||||
date_from: datetime | None = None,
|
||||
) -> tuple[Consumer, str]:
|
||||
api_key = generate_api_key()
|
||||
consumer = Consumer(name=name, api_key_hash=hash_api_key(api_key), is_active=True)
|
||||
db.add(consumer)
|
||||
db.flush()
|
||||
|
||||
consumer_filter = ConsumerFilter(
|
||||
consumer_id=consumer.id,
|
||||
regions=regions,
|
||||
topics=topics,
|
||||
date_from=date_from,
|
||||
)
|
||||
db.add(consumer_filter)
|
||||
db.commit()
|
||||
db.refresh(consumer)
|
||||
return consumer, api_key
|
||||
|
||||
|
||||
def find_consumer_by_api_key(db: Session, api_key: str) -> Consumer | None:
|
||||
key_hash = hash_api_key(api_key)
|
||||
return (
|
||||
db.query(Consumer)
|
||||
.filter(Consumer.api_key_hash == key_hash, Consumer.is_active.is_(True))
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def apply_consumer_filter(db: Session, consumer: Consumer) -> list[Event]:
|
||||
query = db.query(Event).order_by(Event.ingested_at.desc())
|
||||
consumer_filter = consumer.filter
|
||||
|
||||
if consumer_filter:
|
||||
if consumer_filter.regions:
|
||||
query = query.filter(Event.region.in_(consumer_filter.regions))
|
||||
if consumer_filter.topics:
|
||||
query = query.filter(Event.topic.in_(consumer_filter.topics))
|
||||
if consumer_filter.date_from:
|
||||
query = query.filter(Event.event_date >= consumer_filter.date_from)
|
||||
|
||||
return query.all()
|
||||
|
||||
|
||||
def sync_event_to_map_object(db: Session, event: Event) -> MapObject | None:
|
||||
if event.latitude is None or event.longitude is None:
|
||||
return None
|
||||
|
||||
name = event.title or event.locality or f"Событие #{event.id}"
|
||||
description = event.description or event.raw_text or ""
|
||||
|
||||
existing = (
|
||||
db.query(MapObject)
|
||||
.filter(MapObject.event_id == event.id)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
existing.name = name
|
||||
existing.description = description
|
||||
existing.latitude = event.latitude
|
||||
existing.longitude = event.longitude
|
||||
return existing
|
||||
|
||||
by_url = (
|
||||
db.query(MapObject)
|
||||
.join(Event, MapObject.event_id == Event.id)
|
||||
.filter(Event.source_url == event.source_url)
|
||||
.first()
|
||||
)
|
||||
if by_url:
|
||||
by_url.event_id = event.id
|
||||
by_url.name = name
|
||||
by_url.description = description
|
||||
by_url.latitude = event.latitude
|
||||
by_url.longitude = event.longitude
|
||||
return by_url
|
||||
|
||||
obj = MapObject(
|
||||
name=name,
|
||||
description=description,
|
||||
type="marker",
|
||||
latitude=event.latitude,
|
||||
longitude=event.longitude,
|
||||
event_id=event.id,
|
||||
created_at=event.event_date or event.ingested_at,
|
||||
)
|
||||
db.add(obj)
|
||||
return obj
|
||||
@@ -0,0 +1,72 @@
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Event, ParseJob
|
||||
from ..schemas import IngestEventItem
|
||||
from .filtering import sync_event_to_map_object
|
||||
|
||||
|
||||
def ingest_events(
|
||||
db: Session,
|
||||
items: list[IngestEventItem],
|
||||
job_id: int | None = None,
|
||||
) -> tuple[int, int, int]:
|
||||
ingested = 0
|
||||
updated = 0
|
||||
map_synced = 0
|
||||
|
||||
for item in items:
|
||||
existing = (
|
||||
db.query(Event)
|
||||
.filter(Event.source_url == item.source_url)
|
||||
.first()
|
||||
)
|
||||
|
||||
if existing:
|
||||
existing.source_type = item.source_type
|
||||
existing.raw_text = item.raw_text
|
||||
existing.title = item.title
|
||||
existing.description = item.description
|
||||
existing.locality = item.locality
|
||||
existing.latitude = item.latitude
|
||||
existing.longitude = item.longitude
|
||||
existing.event_date = item.event_date
|
||||
existing.region = item.region
|
||||
existing.topic = item.topic
|
||||
existing.tags = item.tags
|
||||
existing.metadata_ = item.metadata
|
||||
event = existing
|
||||
updated += 1
|
||||
else:
|
||||
event = Event(
|
||||
source_type=item.source_type,
|
||||
source_url=item.source_url,
|
||||
raw_text=item.raw_text,
|
||||
title=item.title,
|
||||
description=item.description,
|
||||
locality=item.locality,
|
||||
latitude=item.latitude,
|
||||
longitude=item.longitude,
|
||||
event_date=item.event_date,
|
||||
region=item.region,
|
||||
topic=item.topic,
|
||||
tags=item.tags,
|
||||
metadata_=item.metadata,
|
||||
)
|
||||
db.add(event)
|
||||
ingested += 1
|
||||
|
||||
db.flush()
|
||||
if sync_event_to_map_object(db, event):
|
||||
map_synced += 1
|
||||
|
||||
if job_id is not None:
|
||||
job = db.query(ParseJob).filter(ParseJob.id == job_id).first()
|
||||
if job:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
job.status = "completed"
|
||||
job.last_run_at = datetime.now(timezone.utc)
|
||||
job.last_error = None
|
||||
|
||||
db.commit()
|
||||
return ingested, updated, map_synced
|
||||
@@ -0,0 +1,20 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
import redis
|
||||
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
||||
JOB_QUEUE_KEY = "cp:jobs"
|
||||
|
||||
|
||||
def get_redis() -> redis.Redis:
|
||||
return redis.from_url(REDIS_URL, decode_responses=True)
|
||||
|
||||
|
||||
def enqueue_job(job_id: int, source_type: str, source_config: dict) -> None:
|
||||
payload = {
|
||||
"job_id": job_id,
|
||||
"source_type": source_type,
|
||||
"source_config": source_config,
|
||||
}
|
||||
get_redis().rpush(JOB_QUEUE_KEY, json.dumps(payload))
|
||||
@@ -0,0 +1,39 @@
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
UPLOAD_DIR = Path(os.getenv("UPLOAD_DIR", "/data/uploads"))
|
||||
|
||||
ALLOWED_CONTENT_TYPES = {
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
"image/gif",
|
||||
"image/webp",
|
||||
"video/mp4",
|
||||
"video/webm",
|
||||
}
|
||||
|
||||
MAX_FILE_SIZE = 20 * 1024 * 1024
|
||||
|
||||
|
||||
def ensure_upload_dir() -> None:
|
||||
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def is_allowed_content_type(content_type: str) -> bool:
|
||||
return content_type in ALLOWED_CONTENT_TYPES
|
||||
|
||||
|
||||
def build_stored_name(original_name: str) -> str:
|
||||
suffix = Path(original_name).suffix.lower()
|
||||
return f"{uuid.uuid4().hex}{suffix}"
|
||||
|
||||
|
||||
def media_file_path(stored_name: str) -> Path:
|
||||
return UPLOAD_DIR / stored_name
|
||||
|
||||
|
||||
def remove_media_file(stored_name: str) -> None:
|
||||
path = media_file_path(stored_name)
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
Reference in New Issue
Block a user