gitrusprus/MapMil
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add parser builder: one-shot DeepSeek profile for Telegram extract_mode=profile.

Browse Source 
Generate static HeuristicProfile in CA admin, preview and run without LLM on each post via shared interpreter in CP batch and listener.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
gitrusprus
2026-08-16 14:14:18 +03:00
co-authored by Cursor
parent 6dff3c1c3d
commit 699e9be503
23 changed files with 1042 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
centers/analytics/frontend/src/api/parserBuilder.ts
+52
View File
@@ -0,0 +1,52 @@
import { ADMIN_BASE, request } from "./client";
import type { ParseJob } from "../types/admin";
export type TargetField = {
name: string;
type: string;
description: string;
};
export type HeuristicProfile = {
version: number;
fields: Record<string, Record<string, unknown>>;
notes?: string;
};
export function fetchTargetFields(): Promise<{ fields: TargetField[] }> {
return request<{ fields: TargetField[] }>("/parser-builder/target-fields", undefined, ADMIN_BASE);
}
export function generateParserProfile(sample_post: string): Promise<{ profile: HeuristicProfile }> {
return request<{ profile: HeuristicProfile }>(
"/parser-builder/generate",
{ method: "POST", body: JSON.stringify({ sample_post }) },
ADMIN_BASE,
);
}
export function previewParserProfile(
sample_post: string,
profile: HeuristicProfile | Record<string, unknown>,
): Promise<{ fields: Record<string, string> }> {
return request<{ fields: Record<string, string> }>(
"/parser-builder/preview",
{ method: "POST", body: JSON.stringify({ sample_post, profile }) },
ADMIN_BASE,
);
}
export function createProfileJob(payload: {
channel: string;
profile: HeuristicProfile | Record<string, unknown>;
sample_post?: string;
limit?: number;
interval_seconds?: number;
is_active?: boolean;
}): Promise<ParseJob> {
return request<ParseJob>(
"/parser-builder/jobs",
{ method: "POST", body: JSON.stringify(payload) },
ADMIN_BASE,
);
}
centers/analytics/frontend/src/layouts/AppLayout.vue
+1
View File
@@ -7,6 +7,7 @@ const route = useRoute();
const navItems = [
{ to: "/", label: "Карта", exact: true },
{ to: "/parsers", label: "Парсеры" },
{ to: "/parser-builder", label: "Конструктор" },
{ to: "/events", label: "События" },
{ to: "/analytics", label: "Аналитика" },
{ to: "/consumers", label: "ПИ" },
centers/analytics/frontend/src/router/index.ts
+2
View File
@@ -5,6 +5,7 @@ import AnalyticsView from "../views/AnalyticsView.vue";
import ConsumersView from "../views/ConsumersView.vue";
import EventsView from "../views/EventsView.vue";
import MapViewPage from "../views/MapViewPage.vue";
import ParserBuilderView from "../views/ParserBuilderView.vue";
import ParsersView from "../views/ParsersView.vue";
const router = createRouter({
@@ -16,6 +17,7 @@ const router = createRouter({
children: [
{ path: "", name: "map", component: MapViewPage },
{ path: "parsers", name: "parsers", component: ParsersView },
{ path: "parser-builder", name: "parser-builder", component: ParserBuilderView },
{ path: "events", name: "events", component: EventsView },
{ path: "analytics", name: "analytics", component: AnalyticsView },
{ path: "consumers", name: "consumers", component: ConsumersView },
centers/analytics/frontend/src/views/ParserBuilderView.vue
+320
View File
@@ -0,0 +1,320 @@
<script setup lang="ts">
import { onMounted, ref } from "vue";
import { useRouter } from "vue-router";
import {
createProfileJob,
fetchTargetFields,
generateParserProfile,
previewParserProfile,
type HeuristicProfile,
type TargetField,
} from "../api/parserBuilder";
const INTERVAL_OPTIONS = [
{ value: 900, label: "15 мин" },
{ value: 1800, label: "30 мин" },
{ value: 3600, label: "1 ч" },
{ value: 10800, label: "3 ч" },
{ value: 21600, label: "6 ч" },
{ value: 86400, label: "24 ч" },
];
const router = useRouter();
const samplePost = ref("");
const targetFields = ref<TargetField[]>([]);
const profile = ref<HeuristicProfile | null>(null);
const profileJson = ref("");
const previewFields = ref<Record<string, string> | null>(null);
const channel = ref("");
const limit = ref(100);
const intervalSeconds = ref(3600);
const loadingFields = ref(true);
const generating = ref(false);
const previewing = ref(false);
const saving = ref(false);
const error = ref("");
const success = ref("");
async function loadTargetFields() {
loadingFields.value = true;
try {
const data = await fetchTargetFields();
targetFields.value = data.fields;
error.value = "";
} catch (err) {
error.value = err instanceof Error ? err.message : "Не удалось загрузить целевые поля";
} finally {
loadingFields.value = false;
}
}
function syncProfileFromJson(): HeuristicProfile | null {
if (!profileJson.value.trim()) return null;
try {
const parsed = JSON.parse(profileJson.value) as HeuristicProfile;
profile.value = parsed;
error.value = "";
return parsed;
} catch {
error.value = "Некорректный JSON профиля";
return null;
}
}
async function handleGenerate() {
if (!samplePost.value.trim()) {
error.value = "Вставьте образец поста";
return;
}
generating.value = true;
error.value = "";
success.value = "";
previewFields.value = null;
try {
const data = await generateParserProfile(samplePost.value);
profile.value = data.profile;
profileJson.value = JSON.stringify(data.profile, null, 2);
success.value = "Профиль сгенерирован. Preview и runtime идут только по правилам — без LLM.";
} catch (err) {
error.value = err instanceof Error ? err.message : "Ошибка генерации";
} finally {
generating.value = false;
}
}
async function handlePreview() {
const current = syncProfileFromJson();
if (!current) {
if (!error.value) error.value = "Сначала сгенерируйте профиль";
return;
}
if (!samplePost.value.trim()) {
error.value = "Нужен образец поста для preview";
return;
}
previewing.value = true;
error.value = "";
try {
const data = await previewParserProfile(samplePost.value, current);
previewFields.value = data.fields;
} catch (err) {
error.value = err instanceof Error ? err.message : "Ошибка preview";
} finally {
previewing.value = false;
}
}
async function handleCreate() {
const current = syncProfileFromJson();
if (!current) {
if (!error.value) error.value = "Нужен профиль";
return;
}
if (!channel.value.trim()) {
error.value = "Укажите канал Telegram";
return;
}
saving.value = true;
error.value = "";
success.value = "";
try {
const job = await createProfileJob({
channel: channel.value.trim(),
profile: current,
sample_post: samplePost.value.trim() || undefined,
limit: limit.value,
interval_seconds: intervalSeconds.value,
is_active: true,
});
success.value = `Парсер #${job.id} создан (extract_mode=profile).`;
await router.push({ name: "parsers" });
} catch (err) {
error.value = err instanceof Error ? err.message : "Не удалось создать парсер";
} finally {
saving.value = false;
}
}
onMounted(loadTargetFields);
</script>
<template>
<div class="page">
<h2 class="page-heading">Конструктор парсера</h2>
<p class="intro">
Вставьте образец структурированного поста → DeepSeek один раз построит статичный профиль правил.
Дальше preview и парсинг канала идут только по профилю, без LLM на каждый пост.
</p>
<section class="card">
<h3>Образец поста</h3>
<textarea
v-model="samplePost"
class="sample-area"
rows="12"
placeholder="15.03.2024 Населённый пункт&#10;&#10;Описание события…&#10;&#10;48.123456, 37.654321&#10;#тема"
/>
<div class="form-actions">
<button
type="button"
class="btn btn-primary"
:disabled="generating || !samplePost.trim()"
@click="handleGenerate"
>
{{ generating ? "Генерация..." : "Сгенерировать парсер" }}
</button>
<button
type="button"
class="btn"
:disabled="previewing || !profileJson.trim()"
@click="handlePreview"
>
{{ previewing ? "Preview..." : "Preview" }}
</button>
</div>
</section>
<div class="grid-2">
<section class="card">
<h3>Целевая таблица <span class="muted">(Event / ingest)</span></h3>
<p v-if="loadingFields" class="muted">Загрузка…</p>
<div v-else class="table-wrap">
<table class="admin-table">
<thead>
<tr>
<th>Поле</th>
<th>Тип</th>
<th>Описание</th>
</tr>
</thead>
<tbody>
<tr v-for="field in targetFields" :key="field.name">
<td><code>{{ field.name }}</code></td>
<td>{{ field.type }}</td>
<td>{{ field.description }}</td>
</tr>
</tbody>
</table>
</div>
</section>
<section class="card">
<h3>Профиль (JSON)</h3>
<p v-if="!profileJson" class="muted">Появится после генерации. Можно слегка поправить правила.</p>
<textarea
v-model="profileJson"
class="profile-area"
rows="16"
:disabled="!profileJson && !profile"
spellcheck="false"
/>
</section>
</div>
<section v-if="previewFields" class="card">
<h3>Preview строки</h3>
<div class="table-wrap">
<table class="admin-table">
<thead>
<tr>
<th v-for="key in Object.keys(previewFields)" :key="key">{{ key }}</th>
</tr>
</thead>
<tbody>
<tr>
<td v-for="(val, key) in previewFields" :key="key" class="preview-cell">
{{ val || "—" }}
</td>
</tr>
</tbody>
</table>
</div>
</section>
<section class="card">
<h3>Создать Telegram-парсер</h3>
<form class="admin-form" @submit.prevent="handleCreate">
<div class="form-row">
<label>
Канал
<input v-model="channel" type="text" placeholder="example_channel" required />
</label>
<label>
Лимит batch
<input v-model.number="limit" type="number" min="1" max="1000" />
</label>
<label>
Интервал
<select v-model.number="intervalSeconds">
<option v-for="opt in INTERVAL_OPTIONS" :key="opt.value" :value="opt.value">
{{ opt.label }}
</option>
</select>
</label>
</div>
<div class="form-actions">
<button
type="submit"
class="btn btn-primary"
:disabled="saving || !profileJson.trim() || !channel.trim()"
>
{{ saving ? "Создание..." : "Создать парсер" }}
</button>
</div>
</form>
</section>
<p v-if="error" class="form-error">{{ error }}</p>
<p v-if="success" class="form-success">{{ success }}</p>
</div>
</template>
<style scoped>
.intro {
margin: 0 0 1rem;
font-size: 0.9rem;
color: #555;
line-height: 1.5;
max-width: 72ch;
}
.sample-area,
.profile-area {
width: 100%;
font: inherit;
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 0.8rem;
padding: 0.6rem 0.75rem;
border: 1px solid #d1d5db;
border-radius: 6px;
resize: vertical;
line-height: 1.45;
}
.grid-2 {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 1rem;
}
@media (max-width: 960px) {
.grid-2 {
grid-template-columns: 1fr;
}
}
.preview-cell {
max-width: 180px;
white-space: pre-wrap;
word-break: break-word;
font-size: 0.8rem;
}
.form-success {
color: #2e7d32;
font-size: 0.875rem;
margin-top: 0.5rem;
}
</style>
centers/analytics/frontend/src/views/ParsersView.vue
+58 -3
View File
@@ -79,9 +79,15 @@ function parseLines(text: string): string[] {
.filter(Boolean);
}
function extractModeLabel(cfg: Record<string, unknown>): string {
if (cfg.extract_mode === "llm") return " [LLM]";
if (cfg.extract_mode === "profile") return " [profile]";
return "";
}
function sourceSummary(job: ParseJob): string {
const cfg = job.source_config;
const mode = cfg.extract_mode === "llm" ? " [LLM]" : "";
const mode = extractModeLabel(cfg);
if (job.source_type === "telegram") {
return (channelFromConfig(cfg) || "—") + mode;
}
@@ -92,6 +98,21 @@ function sourceSummary(job: ParseJob): string {
return "—";
}
function isProfileJob(job: ParseJob | null): boolean {
return !!job && job.source_config.extract_mode === "profile";
}
function profileJson(job: ParseJob | null): string {
if (!job) return "";
const profile = job.source_config.heuristic_profile;
if (!profile || typeof profile !== "object") return "";
try {
return JSON.stringify(profile, null, 2);
} catch {
return "";
}
}
function formatInterval(seconds: number): string {
const opt = INTERVAL_OPTIONS.find((item) => item.value === seconds);
if (opt) return opt.label;
@@ -111,8 +132,19 @@ function buildSourceConfig(
input_mode: "urls" | "texts" | "mixed";
extract_mode: "heuristic" | "llm";
},
existing?: Record<string, unknown>,
): Record<string, unknown> {
if (sourceType === "telegram") {
// Preserve builder-created profile jobs (edit form has no profile editor)
if (existing?.extract_mode === "profile" && existing.heuristic_profile) {
return {
channel: data.channel.trim(),
limit: data.limit,
extract_mode: "profile",
heuristic_profile: existing.heuristic_profile,
sample_post: existing.sample_post ?? null,
};
}
return {
channel: data.channel.trim(),
limit: data.limit,
@@ -242,7 +274,7 @@ async function handleSaveEdit() {
error.value = "";
try {
await updateJob(editingJob.value.id, {
source_config: buildSourceConfig(sourceType, editForm.value),
source_config: buildSourceConfig(sourceType, editForm.value, editingJob.value.source_config),
interval_seconds: editForm.value.interval_seconds,
is_active: editForm.value.is_active,
});
@@ -318,6 +350,8 @@ onUnmounted(() => {
<p class="hint">
{{ formHint }}
Дубликаты по <code>source_url</code> не записываются.
Статичный парсер по образцу поста —
<router-link to="/parser-builder">Конструктор</router-link>.
</p>
<form class="admin-form" @submit.prevent="handleSubmit">
<div class="form-row">
@@ -479,13 +513,21 @@ onUnmounted(() => {
Лимит
<input v-model.number="editForm.limit" type="number" min="1" max="1000" />
</label>
<label>
<label v-if="!isProfileJob(editingJob)">
Извлечение
<select v-model="editForm.extract_mode">
<option value="heuristic">Эвристики</option>
<option value="llm">LLM DeepSeek</option>
</select>
</label>
<p v-else class="hint profile-hint">
Режим: <code>profile</code> (статичный профиль из конструктора).
Сменить правила — через повторную генерацию в «Конструктор».
</p>
<label v-if="isProfileJob(editingJob)" class="span-2">
Профиль (только просмотр)
<textarea class="profile-ro" :value="profileJson(editingJob)" rows="8" readonly />
</label>
</template>
<template v-else-if="editingJob.source_type === 'crawl4ai'">
<label class="span-2">
@@ -611,4 +653,17 @@ onUnmounted(() => {
gap: 0.5rem;
margin-top: 1.5rem;
}
.profile-hint {
flex: 1 1 100%;
margin: 0;
}
.profile-ro {
width: 100%;
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 0.75rem;
background: #f8f8f8;
color: #444;
}
</style>