diff --git a/backend/dataset_generator.py b/backend/dataset_generator.py index 73b3c4e..de60852 100644 --- a/backend/dataset_generator.py +++ b/backend/dataset_generator.py @@ -474,6 +474,35 @@ def resolve_output_dir(output_dir: str | Path = "sonar_dataset") -> Path: return out +def _safe_object_stem(object_name: str | None) -> str: + stem = Path(object_name or "object").stem + safe = "".join(ch if ch.isalnum() or ch in "_-" else "_" for ch in stem).strip("._-") + return (safe or "object")[:80] + + +def make_generation_run_dir(base_dir: Path, object_name: str | None = None) -> Path: + """Create a new run subdirectory: YYYY-MM-DD_HH-MM-SS-. + + Previous runs under base_dir are left untouched. + """ + from datetime import datetime + + base_dir.mkdir(parents=True, exist_ok=True) + stamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + folder = f"{stamp}-{_safe_object_stem(object_name)}" + path = base_dir / folder + if path.exists(): + n = 2 + while True: + candidate = base_dir / f"{folder}_{n}" + if not candidate.exists(): + path = candidate + break + n += 1 + path.mkdir(parents=True, exist_ok=False) + return path + + def _downsample_points(points: list[list[float]], max_points: int) -> list[list[float]]: max_points = max(100, int(max_points)) if len(points) <= max_points: @@ -588,7 +617,7 @@ def write_scene_files( return {"npy": str(npy_path), "obj": str(obj_path), "stem": stem} -def generate_dataset( +def iter_generate_dataset( *, count: int = 5, seed: int = 42, @@ -599,16 +628,13 @@ def generate_dataset( object_scale_is_max: bool = False, beam_count: int = 45, length_count: int | None = None, -) -> dict[str, Any]: - """Generate `count` unique scenes into output_dir with Area_X naming. +): + """Yield NDJSON-friendly progress events, then a final ``done`` payload. - object_points: normalized template vertices from user .obj (class 1 = object). - object_scale: relative size multiplier vs unit-normalized mesh (1.0 = default). - object_scale_is_max: if True, treat object_scale as upper bound and sample - per-scene scale uniformly from [1, object_scale] (AUV altitude variation). - beam_count: across-track beams (width resolution, X). - length_count: along-track pings (length resolution, Y). Defaults to beam_count. - Each scene has exactly beam_count × length_count sonar returns (first-hit casting). + Events: + {"type":"start","total":N,"outputDir":"...","runName":"..."} + {"type":"progress","current":k,"total":N,"entry":{...}} + {"type":"done","result":{...}} """ count = int(count) if count < 1: @@ -636,7 +662,8 @@ def generate_dataset( if length_count > 1024: raise ValueError("length_count (Длина) must be <= 1024") - out = resolve_output_dir(output_dir) + base = resolve_output_dir(output_dir) + run_dir = make_generation_run_dir(base, object_name) template = normalize_object_points(object_points) labels = plan_scene_labels(count, seed) @@ -655,6 +682,14 @@ def generate_dataset( preview_stem: str | None = None preview_has_object = False + yield { + "type": "start", + "total": count, + "outputDir": str(run_dir), + "baseDir": str(base), + "runName": run_dir.name, + } + for i in range(count): visibility = labels[i] scene_seed = int(seed) + i * 10007 + 17 @@ -675,7 +710,7 @@ def generate_dataset( length_count=length_count, ) area, scene_no, stem = scene_index_to_area_name(i) - paths = write_scene_files(scene, out, stem) + paths = write_scene_files(scene, run_dir, stem) entry = { "index": i, @@ -702,6 +737,14 @@ def generate_dataset( preview_stem = stem preview_has_object = bool(scene["hasObject"]) + yield { + "type": "progress", + "current": i + 1, + "total": count, + "entry": entry, + "outputDir": str(run_dir), + } + preview: dict[str, Any] | None = None if preview_points is not None: pts = _downsample_points(preview_points, 25000) @@ -713,8 +756,10 @@ def generate_dataset( "classLabels": {"0": "background", "1": "object"}, } - return { - "outputDir": str(out), + result = { + "outputDir": str(run_dir), + "baseDir": str(base), + "runName": run_dir.name, "count": count, "seed": int(seed), "beamCount": beam_count, @@ -728,3 +773,36 @@ def generate_dataset( "written": written, "preview": preview, } + yield {"type": "done", "result": result} + + +def generate_dataset( + *, + count: int = 5, + seed: int = 42, + output_dir: str | Path = "sonar_dataset", + object_points: list[list[float]], + object_name: str | None = None, + object_scale: float = 1.0, + object_scale_is_max: bool = False, + beam_count: int = 45, + length_count: int | None = None, +) -> dict[str, Any]: + """Generate `count` unique scenes into a new timestamped run folder under output_dir.""" + result: dict[str, Any] | None = None + for event in iter_generate_dataset( + count=count, + seed=seed, + output_dir=output_dir, + object_points=object_points, + object_name=object_name, + object_scale=object_scale, + object_scale_is_max=object_scale_is_max, + beam_count=beam_count, + length_count=length_count, + ): + if event.get("type") == "done": + result = event["result"] + if result is None: + raise RuntimeError("Dataset generation produced no result.") + return result diff --git a/backend/main.py b/backend/main.py index 4174d38..ed6e2fe 100644 --- a/backend/main.py +++ b/backend/main.py @@ -11,14 +11,14 @@ from typing import Any from fastapi import FastAPI, File, Form, HTTPException, UploadFile from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse, Response +from fastapi.responses import FileResponse, Response, StreamingResponse from fastapi.staticfiles import StaticFiles from pydantic import BaseModel from builtin_presets import BUILTIN_PRESETS, get_builtin_preset from demo_generator import DEMO_SURFACE_TYPES, demo_payload from pipeline_insights import compute_insights -from dataset_generator import generate_dataset, load_object_points_from_obj_text, load_scene_preview +from dataset_generator import iter_generate_dataset, load_object_points_from_obj_text, load_scene_preview from scene_generator import ( catalog_payload as generator_catalog_payload, export_npy_float64, @@ -429,7 +429,7 @@ async def dataset_generate( beamCount: int = Form(45), lengthCount: int | None = Form(None), model: UploadFile = File(...), -) -> dict[str, Any]: +) -> StreamingResponse: filename = (model.filename or "").strip() if not filename.lower().endswith(".obj"): raise HTTPException(status_code=400, detail="Upload a .obj 3D model file.") @@ -437,21 +437,32 @@ async def dataset_generate( raw = await model.read() text = raw.decode("utf-8", errors="ignore") object_points = load_object_points_from_obj_text(text) - return generate_dataset( - count=count, - seed=seed, - output_dir=outputDir or "sonar_dataset", - object_points=object_points, - object_name=filename, - object_scale=objectScale, - object_scale_is_max=objectScaleIsMax, - beam_count=beamCount, - length_count=lengthCount, - ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc - except OSError as exc: - raise HTTPException(status_code=500, detail=f"Failed to write dataset: {exc}") from exc + + def event_stream(): + try: + for event in iter_generate_dataset( + count=count, + seed=seed, + output_dir=outputDir or "sonar_dataset", + object_points=object_points, + object_name=filename, + object_scale=objectScale, + object_scale_is_max=objectScaleIsMax, + beam_count=beamCount, + length_count=lengthCount, + ): + yield json.dumps(event, ensure_ascii=False) + "\n" + except ValueError as exc: + yield json.dumps({"type": "error", "detail": str(exc)}, ensure_ascii=False) + "\n" + except OSError as exc: + yield json.dumps( + {"type": "error", "detail": f"Failed to write dataset: {exc}"}, + ensure_ascii=False, + ) + "\n" + + return StreamingResponse(event_stream(), media_type="application/x-ndjson") @app.post("/api/dataset/preview") diff --git a/frontend/web/src/api/client.js b/frontend/web/src/api/client.js index c8605e9..96bb3c9 100644 --- a/frontend/web/src/api/client.js +++ b/frontend/web/src/api/client.js @@ -289,7 +289,7 @@ export const api = { }); triggerDownload(blob, safeName || suggested); }, - datasetGenerate: ({ + datasetGenerate: async ({ count = 5, seed = 42, outputDir = "sonar_dataset", @@ -298,6 +298,7 @@ export const api = { beamCount = 45, lengthCount = 45, modelFile, + onProgress, } = {}) => { if (!modelFile) { return Promise.reject(new Error("Выберите файл модели .obj")); @@ -311,10 +312,74 @@ export const api = { formData.append("beamCount", String(beamCount ?? 45)); formData.append("lengthCount", String(lengthCount ?? 45)); formData.append("model", modelFile, modelFile.name || "model.obj"); - return request("/api/dataset/generate", { + + const response = await fetch("/api/dataset/generate", { method: "POST", body: formData, }); + if (!response.ok) { + const payload = await response.json().catch(() => ({})); + throw new Error(formatApiError(payload.detail, `Request failed: /api/dataset/generate`)); + } + if (!response.body) { + throw new Error("Пустой ответ генерации датасета"); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + let result = null; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + let event; + try { + event = JSON.parse(trimmed); + } catch { + continue; + } + if (event.type === "error") { + throw new Error(event.detail || "Ошибка генерации датасета"); + } + if (event.type === "start" || event.type === "progress") { + if (typeof onProgress === "function") onProgress(event); + } + if (event.type === "done") { + result = event.result; + } + } + } + + if (buffer.trim()) { + try { + const event = JSON.parse(buffer.trim()); + if (event.type === "error") { + throw new Error(event.detail || "Ошибка генерации датасета"); + } + if (event.type === "done") result = event.result; + if ((event.type === "start" || event.type === "progress") && typeof onProgress === "function") { + onProgress(event); + } + } catch (error) { + if (error instanceof SyntaxError) { + /* ignore trailing junk */ + } else { + throw error; + } + } + } + + if (!result) { + throw new Error("Генерация завершилась без результата"); + } + return result; }, datasetPreview: ({ stem, outputDir = "sonar_dataset", maxPoints = 25000 } = {}) => request("/api/dataset/preview", { diff --git a/frontend/web/src/stores/dataset.js b/frontend/web/src/stores/dataset.js index b0b9520..7ba9cc4 100644 --- a/frontend/web/src/stores/dataset.js +++ b/frontend/web/src/stores/dataset.js @@ -16,6 +16,7 @@ export const useDatasetStore = defineStore("dataset", { objectScaleIsMax: false, beamCount: 45, lengthCount: 45, + generateProgress: 0, lastResult: null, selectedStem: null, previewPoints: [], @@ -116,6 +117,7 @@ export const useDatasetStore = defineStore("dataset", { return; } this.busy = true; + this.generateProgress = 0; this.statusText = "Генерация датасета…"; this.pushLog( `Старт: count=${this.count}, seed=${this.seed}, beams=${this.beamCount}, length=${this.lengthCount}, scale=${this.objectScale}${this.objectScaleIsMax ? " (макс.)" : ""}, dir=${this.outputDir}, model=${this.modelFileName}`, @@ -130,7 +132,35 @@ export const useDatasetStore = defineStore("dataset", { beamCount: Number(this.beamCount) || 45, lengthCount: Number(this.lengthCount) || 45, modelFile: this.modelFile, + onProgress: (event) => { + if (event.type === "start") { + this.generateProgress = 0; + if (event.outputDir) { + this.resolvedOutputDir = event.outputDir; + this.pushLog(`Каталог запуска: ${event.outputDir}`); + } + this.statusText = `Генерация… 0/${event.total || "?"}`; + return; + } + if (event.type === "progress") { + const total = Number(event.total) || 1; + const current = Number(event.current) || 0; + this.generateProgress = Math.min(100, Math.round((100 * current) / total)); + this.statusText = `Генерация… ${current}/${total}`; + const entry = event.entry; + if (entry?.stem) { + const scalePart = + entry.objectScale != null + ? `, scale=${Number(entry.objectScale).toFixed(3)}` + : ""; + this.pushLog( + `${entry.stem}: pts=${entry.pointCount}, object=${entry.objectPointCount}, ${entry.visibility}${scalePart}`, + ); + } + } + }, }); + this.generateProgress = 100; this.lastResult = result; this.resolvedOutputDir = result?.outputDir || null; const s = result?.stats || {}; @@ -142,18 +172,11 @@ export const useDatasetStore = defineStore("dataset", { `Модель: ${result.objectName || this.modelFileName} (${result.objectVertexCount || "?"} вершин), ${scaleNote}, ширина=${result.beamCount ?? this.beamCount}, длина=${result.lengthCount ?? this.lengthCount}. class 1 = object.`, ); this.pushLog( - `Записано ${result.count} сцен. С объектом: ${s.withObject}, без: ${s.withoutObject}.`, + `Записано ${result.count} сцен в ${result.runName || result.outputDir}. С объектом: ${s.withObject}, без: ${s.withoutObject}.`, ); this.pushLog( `Видимость: nearly_hidden=${s.nearly_hidden || 0}, partial=${s.partial || 0}, visible=${s.visible || 0}, absent=${s.absent || 0}.`, ); - for (const item of result.written || []) { - const scalePart = - item.objectScale != null ? `, scale=${Number(item.objectScale).toFixed(3)}` : ""; - this.pushLog( - `${item.stem}: pts=${item.pointCount}, object=${item.objectPointCount}, ${item.visibility}${scalePart}`, - ); - } const initialStem = result?.preview?.stem || @@ -190,6 +213,7 @@ export const useDatasetStore = defineStore("dataset", { throw error; } finally { this.busy = false; + this.generateProgress = 0; } }, async selectScene(stem) { diff --git a/frontend/web/src/views/DatasetView.vue b/frontend/web/src/views/DatasetView.vue index 52eeec9..7549120 100644 --- a/frontend/web/src/views/DatasetView.vue +++ b/frontend/web/src/views/DatasetView.vue @@ -193,11 +193,18 @@ function onHighlightClassChange(event) {

{{ store.statusText }}

@@ -400,10 +407,41 @@ function onHighlightClassChange(event) { font-weight: 600; cursor: pointer; } +.generate-btn { + position: relative; + overflow: hidden; + isolation: isolate; +} +.generate-btn-fill { + position: absolute; + left: 0; + top: 0; + bottom: 0; + width: 0; + background: color-mix(in srgb, var(--primary-button-text, #142033) 22%, transparent); + transition: width 0.18s ease; + z-index: 0; + pointer-events: none; +} +.generate-btn.busy .generate-btn-fill { + background: linear-gradient( + 90deg, + color-mix(in srgb, #fff 35%, var(--primary-button-bg, #ffb020)), + color-mix(in srgb, #fff 12%, var(--primary-button-bg, #ffb020)) + ); +} +.generate-btn-label { + position: relative; + z-index: 1; +} .primary:disabled { opacity: 0.6; cursor: wait; } +.generate-btn.busy:disabled { + opacity: 1; + cursor: wait; +} .status { margin: 0; font-size: 12px;