diff --git a/centers/analytics/api/app/routers/parser_profiles.py b/centers/analytics/api/app/routers/parser_profiles.py index 1d4db37..fda3f15 100644 --- a/centers/analytics/api/app/routers/parser_profiles.py +++ b/centers/analytics/api/app/routers/parser_profiles.py @@ -20,10 +20,14 @@ router = APIRouter(prefix="/admin/parser-profiles", tags=["parser-profiles"]) class GenerateRequest(BaseModel): sample_post: str = Field(min_length=1) + hint: str | None = None + current_profile: dict[str, Any] | None = None class GenerateResponse(BaseModel): profile: dict[str, Any] + preview: dict[str, str] + empty_fields: list[str] = Field(default_factory=list) class PreviewRequest(BaseModel): @@ -65,7 +69,11 @@ async def generate_profile(payload: GenerateRequest): detail="DEEPSEEK_API_KEY is not set on ca-api. Add it to .env for parser generation.", ) try: - profile = await builder.generate_profile(payload.sample_post) + profile = await builder.generate_profile( + payload.sample_post, + hint=payload.hint, + current_profile=payload.current_profile, + ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc except Exception as exc: @@ -73,7 +81,13 @@ async def generate_profile(payload: GenerateRequest): status_code=502, detail=f"DeepSeek generate failed: {exc}", ) from exc - return GenerateResponse(profile=profile.model_dump()) + dumped = profile.model_dump() + preview = builder.preview_with_profile(payload.sample_post, dumped) + return GenerateResponse( + profile=dumped, + preview=preview, + empty_fields=builder.empty_preview_fields(preview), + ) @router.post("/preview", response_model=PreviewResponse) diff --git a/centers/analytics/api/app/services/parser_builder.py b/centers/analytics/api/app/services/parser_builder.py index b9a4396..e8b4eaa 100644 --- a/centers/analytics/api/app/services/parser_builder.py +++ b/centers/analytics/api/app/services/parser_builder.py @@ -22,7 +22,8 @@ logger = logging.getLogger("ca.parser_builder") GENERATE_SYSTEM = ( "You design a static heuristic parser profile for structured Telegram posts. " "Output valid JSON only, no markdown, no Python code. " - "The profile is applied with regex/line/marker rules at runtime — never with an LLM." + "The profile is applied with regex/line/marker rules at runtime — never with an LLM. " + "Every rule must actually extract a non-empty value from the given sample when applied." ) @@ -46,6 +47,10 @@ def preview_with_profile(sample_post: str, profile: dict[str, Any] | HeuristicPr return apply_profile(sample_post, profile) +def empty_preview_fields(preview: dict[str, str]) -> list[str]: + return [name for name, value in preview.items() if not (value or "").strip()] + + def _extract_json_object(content: str) -> dict[str, Any]: content = (content or "").strip() if content.startswith("```"): @@ -63,21 +68,11 @@ def _extract_json_object(content: str) -> dict[str, Any]: return parsed -async def generate_profile(sample_post: str) -> HeuristicProfile: - settings = deepseek_settings() - if not settings["api_key"]: - raise RuntimeError( - "DEEPSEEK_API_KEY is not set on ca-api. Add it to .env for parser generation." - ) - - sample = (sample_post or "").strip() - if not sample: - raise ValueError("sample_post is required") - +def _base_user_prompt(sample: str) -> str: fields_help = "\n".join( f"- {spec['name']}: {spec['description']}" for spec in target_field_specs() ) - user_prompt = ( + return ( "Build a HeuristicProfile JSON for this sample Telegram post.\n\n" "Schema:\n" '{"version": 1, "notes": "...", "fields": {' @@ -89,14 +84,88 @@ async def generate_profile(sample_post: str) -> HeuristicProfile: "- Only include fields you can extract reliably from the sample.\n" f"- Allowed field names: {', '.join(TARGET_FIELDS)}.\n" "- Prefer regex / line / after_marker / between over literal.\n" - "- Use literal only for constant topic tags.\n" + "- Use literal only for constant topic/region tags.\n" + "- title: usually the first non-empty line (strategy line, line_index 0).\n" + "- description: the multi-line body AFTER the title and BEFORE coords/hashtags. " + "Prefer strategy between or regex with flags including \"s\" (DOTALL). " + "Do NOT leave description empty if the sample has a body paragraph.\n" + "- locality: settlement name from the title line when present.\n" "- coords should capture 'lat, lon' when present.\n" - "- event_date should capture DD.MM.YYYY or similar.\n" - "- Do not invent Python code; only declarative rules.\n\n" + "- event_date should capture DD.MM.YYYY / DD.MM.YY or similar.\n" + "- topic/region: hashtags like #ru → topic \"ru\"; avoid stuffing unrelated tags into region.\n" + "- Do not invent Python code; only declarative rules.\n" + "- Mentally apply each rule to the sample: every included field must yield a non-empty string.\n\n" f"Target fields:\n{fields_help}\n\n" f"Sample post:\n{sample[:12000]}" ) + +def _parse_profile_response(content: str) -> HeuristicProfile: + raw = _extract_json_object(content) + if "fields" not in raw and isinstance(raw.get("profile"), dict): + raw = raw["profile"] + if "version" not in raw: + raw["version"] = 1 + return HeuristicProfile.model_validate(raw) + + +async def generate_profile( + sample_post: str, + *, + hint: str | None = None, + current_profile: dict[str, Any] | HeuristicProfile | None = None, +) -> HeuristicProfile: + settings = deepseek_settings() + if not settings["api_key"]: + raise RuntimeError( + "DEEPSEEK_API_KEY is not set on ca-api. Add it to .env for parser generation." + ) + + sample = (sample_post or "").strip() + if not sample: + raise ValueError("sample_post is required") + + hint_text = (hint or "").strip() + current: HeuristicProfile | None = None + if current_profile is not None: + current = ( + current_profile + if isinstance(current_profile, HeuristicProfile) + else HeuristicProfile.model_validate(current_profile) + ) + + if current is not None or hint_text: + preview = preview_with_profile(sample, current) if current is not None else {} + empty = empty_preview_fields(preview) if preview else [] + parts = [ + "Revise the HeuristicProfile JSON for this sample Telegram post.", + "Keep strategies that already work; fix only what the manager asks for " + "and any fields that preview as empty.", + "Output the full updated profile JSON (same schema), not a patch.", + "", + _base_user_prompt(sample), + ] + if current is not None: + parts.extend( + [ + "", + "Current profile JSON:", + json.dumps(current.model_dump(), ensure_ascii=False, indent=2)[:12000], + ] + ) + if empty: + parts.append( + "Fields currently empty in preview (must become non-empty if present in sample): " + + ", ".join(empty) + ) + else: + parts.append("Current preview fields: " + json.dumps(preview, ensure_ascii=False)) + if hint_text: + parts.extend(["", "Manager hint (follow this):", hint_text[:4000]]) + user_prompt = "\n".join(parts) + else: + user_prompt = _base_user_prompt(sample) + payload = { "model": settings["model"], "messages": [ @@ -121,10 +190,4 @@ async def generate_profile(sample_post: str) -> HeuristicProfile: data = response.json() content = data["choices"][0]["message"]["content"] - raw = _extract_json_object(content) - # Accept either top-level profile or {"profile": {...}} - if "fields" not in raw and isinstance(raw.get("profile"), dict): - raw = raw["profile"] - if "version" not in raw: - raw["version"] = 1 - return HeuristicProfile.model_validate(raw) + return _parse_profile_response(content) diff --git a/centers/analytics/frontend/src/api/entities.ts b/centers/analytics/frontend/src/api/entities.ts index 9efa175..da8dc79 100644 --- a/centers/analytics/frontend/src/api/entities.ts +++ b/centers/analytics/frontend/src/api/entities.ts @@ -64,10 +64,29 @@ export function fetchTargetFields(): Promise<{ fields: TargetField[] }> { return request<{ fields: TargetField[] }>("/parser-profiles/target-fields", undefined, ADMIN_BASE); } -export function generateParserProfile(sample_post: string): Promise<{ profile: HeuristicProfile }> { - return request<{ profile: HeuristicProfile }>( +export function generateParserProfile(payload: { + sample_post: string; + hint?: string; + current_profile?: HeuristicProfile | Record | null; +}): Promise<{ + profile: HeuristicProfile; + preview: Record; + empty_fields: string[]; +}> { + return request<{ + profile: HeuristicProfile; + preview: Record; + empty_fields: string[]; + }>( "/parser-profiles/generate", - { method: "POST", body: JSON.stringify({ sample_post }) }, + { + method: "POST", + body: JSON.stringify({ + sample_post: payload.sample_post, + hint: payload.hint || undefined, + current_profile: payload.current_profile || undefined, + }), + }, ADMIN_BASE, ); } diff --git a/centers/analytics/frontend/src/views/ParserProfilesView.vue b/centers/analytics/frontend/src/views/ParserProfilesView.vue index 9f93581..38255bb 100644 --- a/centers/analytics/frontend/src/views/ParserProfilesView.vue +++ b/centers/analytics/frontend/src/views/ParserProfilesView.vue @@ -20,10 +20,12 @@ const success = ref(""); const submitting = ref(false); const samplePost = ref(""); +const generateHint = ref(""); const profileName = ref(""); const targetFields = ref([]); const profileJson = ref(""); const previewFields = ref | null>(null); +const emptyFields = ref([]); const generating = ref(false); const previewing = ref(false); const loadingFields = ref(true); @@ -73,10 +75,32 @@ async function handleGenerate() { error.value = ""; success.value = ""; previewFields.value = null; + emptyFields.value = []; try { - const data = await generateParserProfile(samplePost.value); + let current: HeuristicProfile | null = null; + if (profileJson.value.trim()) { + current = syncProfileFromJson(); + if (!current) return; + } + const data = await generateParserProfile({ + sample_post: samplePost.value, + hint: generateHint.value.trim() || undefined, + current_profile: current, + }); profileJson.value = JSON.stringify(data.profile, null, 2); - success.value = "Профиль сгенерирован. Preview и runtime — только по правилам, без LLM."; + previewFields.value = data.preview; + emptyFields.value = data.empty_fields || []; + if (emptyFields.value.length) { + success.value = + `Профиль обновлён, но пустые поля: ${emptyFields.value.join(", ")}. ` + + "Допишите подсказку ниже и снова нажмите Generate."; + if (!generateHint.value.trim() && emptyFields.value.includes("description")) { + generateHint.value = + "description должен содержать многострочный текст тела поста после заголовка и до координат (47.…, 36.…), без строки заголовка и без хвоста Источник/Геопривязка/#теги"; + } + } else { + success.value = "Профиль сгенерирован. Preview и runtime — только по правилам, без LLM."; + } } catch (err) { error.value = err instanceof Error ? err.message : "Ошибка генерации"; } finally { @@ -99,6 +123,9 @@ async function handlePreview() { try { const data = await previewParserProfile(samplePost.value, current); previewFields.value = data.fields; + emptyFields.value = Object.entries(data.fields) + .filter(([, v]) => !v) + .map(([k]) => k); } catch (err) { error.value = err instanceof Error ? err.message : "Ошибка preview"; } finally { @@ -110,9 +137,11 @@ function resetEditor() { editingId.value = null; profileName.value = ""; samplePost.value = ""; + generateHint.value = ""; profileJson.value = ""; profileStatus.value = "draft"; previewFields.value = null; + emptyFields.value = []; success.value = ""; } @@ -120,11 +149,13 @@ function openEdit(p: ParserProfile) { editingId.value = p.id; profileName.value = p.name; samplePost.value = p.sample_post || ""; + generateHint.value = ""; profileJson.value = p.heuristic_profile ? JSON.stringify(p.heuristic_profile, null, 2) : ""; profileStatus.value = p.status === "ready" ? "ready" : "draft"; previewFields.value = null; + emptyFields.value = []; success.value = ""; error.value = ""; window.scrollTo({ top: 0, behavior: "smooth" }); @@ -197,8 +228,9 @@ onMounted(() => {

Профили парсера

- Статичный эвристический профиль: Generate (DeepSeek один раз) → Preview → Сохранить. - Дальше профиль подключается к каналу на странице + Generate строит статичные правила по образцу поста. Если Preview пустой по полю — + напишите подсказку и снова Generate (агент правит текущий JSON). Можно править JSON + вручную. Дальше профиль подключается к каналу на Парсеры.

@@ -231,6 +263,16 @@ onMounted(() => { placeholder="15.03.2024 Населённый пункт Описание… 48.123456, 37.654321" /> +