gitrusprusandCursor
9dfe713668
Unify parsing workers, analytics API with PostgreSQL, map UI, and PI distribution into centers/ with Docker Compose. Co-authored-by: Cursor <cursoragent@cursor.com>
107 lines
3.0 KiB
Python
107 lines
3.0 KiB
Python
"""Загрузка содержимого из источников данных."""
|
|
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
NOISE_TAGS = ("script", "style", "noscript", "nav", "header", "footer", "aside", "menu")
|
|
CONTENT_SELECTORS = (
|
|
"article",
|
|
"[role='main']",
|
|
"main",
|
|
".article",
|
|
".content",
|
|
".post",
|
|
".entry-content",
|
|
".news-item",
|
|
".card-full-news",
|
|
".box-card",
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class PageContent:
|
|
text: str
|
|
links: list[str]
|
|
|
|
|
|
async def fetch_page(url: str, client: httpx.AsyncClient) -> PageContent:
|
|
response = await client.get(url)
|
|
response.raise_for_status()
|
|
content_type = response.headers.get("content-type", "")
|
|
if "html" not in content_type:
|
|
return PageContent(text=response.text, links=[])
|
|
|
|
return _parse_html(response.text, base_url=str(response.url))
|
|
|
|
|
|
def _parse_html(html: str, base_url: str) -> PageContent:
|
|
soup = BeautifulSoup(html, "lxml")
|
|
links = _extract_links(soup, base_url)
|
|
|
|
for tag in soup.find_all(NOISE_TAGS):
|
|
tag.decompose()
|
|
|
|
chunks: list[str] = []
|
|
for selector in CONTENT_SELECTORS:
|
|
for node in soup.select(selector):
|
|
text = node.get_text(separator=" ", strip=True)
|
|
if len(text) > 80:
|
|
chunks.append(text)
|
|
|
|
if chunks:
|
|
return PageContent(text=" ".join(chunks), links=links)
|
|
|
|
body = soup.body or soup
|
|
return PageContent(text=body.get_text(separator=" ", strip=True), links=links)
|
|
|
|
|
|
def _extract_links(soup: BeautifulSoup, base_url: str) -> list[str]:
|
|
found: list[str] = []
|
|
seen: set[str] = set()
|
|
base_host = urlparse(base_url).netloc.lower()
|
|
|
|
for tag in soup.find_all("a", href=True):
|
|
href = tag["href"].strip()
|
|
if not href or href.startswith(("#", "mailto:", "tel:", "javascript:")):
|
|
continue
|
|
|
|
absolute = urljoin(base_url, href)
|
|
parsed = urlparse(absolute)
|
|
if parsed.scheme not in ("http", "https"):
|
|
continue
|
|
if parsed.netloc.lower() != base_host:
|
|
continue
|
|
|
|
clean = absolute.split("#", 1)[0]
|
|
if clean and clean not in seen:
|
|
seen.add(clean)
|
|
found.append(clean)
|
|
|
|
return found
|
|
|
|
|
|
async def fetch_url(url: str, timeout: float = 30.0) -> str:
|
|
async with httpx.AsyncClient(
|
|
follow_redirects=True,
|
|
timeout=timeout,
|
|
headers={"User-Agent": "SocialParser/1.0"},
|
|
) as client:
|
|
page = await fetch_page(url, client)
|
|
return page.text
|
|
|
|
|
|
def read_file(path: str) -> str:
|
|
file_path = Path(path)
|
|
if not file_path.exists():
|
|
raise FileNotFoundError(f"Файл не найден: {path}")
|
|
for encoding in ("utf-8", "cp1251", "latin-1"):
|
|
try:
|
|
return file_path.read_text(encoding=encoding)
|
|
except UnicodeDecodeError:
|
|
continue
|
|
raise ValueError(f"Не удалось прочитать файл: {path}")
|