From d01c7276a8a45f4e71807b2a182c2937b2f7b75c Mon Sep 17 00:00:00 2001 From: DawnARC Date: Wed, 6 May 2026 23:05:44 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8D=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E8=BF=9B=E5=BA=A6=E5=B1=95=E7=A4=BA=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test_web_import_manager_payloads.py | 67 ++++++++++++++++++- .../core/utils/web_import_manager.py | 25 ++++--- uv.lock | 8 +-- 3 files changed, 82 insertions(+), 18 deletions(-) diff --git a/pytests/A_memorix_test/test_web_import_manager_payloads.py b/pytests/A_memorix_test/test_web_import_manager_payloads.py index 8fdca65d..f2d78df3 100644 --- a/pytests/A_memorix_test/test_web_import_manager_payloads.py +++ b/pytests/A_memorix_test/test_web_import_manager_payloads.py @@ -4,7 +4,12 @@ import numpy as np import pytest from src.A_memorix.core.strategies.base import ChunkContext, KnowledgeType, ProcessedChunk, SourceInfo -from src.A_memorix.core.utils.web_import_manager import ImportTaskManager +from src.A_memorix.core.utils.web_import_manager import ( + ImportChunkRecord, + ImportFileRecord, + ImportTaskManager, + ImportTaskRecord, +) class _DummyMetadataStore: @@ -76,6 +81,21 @@ def _build_manager() -> tuple[ImportTaskManager, _DummyMetadataStore]: return manager, metadata_store +def _build_progress_task(task_id: str, total_chunks: int = 2) -> ImportTaskRecord: + file_record = ImportFileRecord( + file_id="file-1", + name="demo.txt", + source_kind="paste", + input_mode="text", + total_chunks=total_chunks, + chunks=[ + ImportChunkRecord(chunk_id=f"chunk-{index}", index=index, chunk_type="text") + for index in range(total_chunks) + ], + ) + return ImportTaskRecord(task_id=task_id, source="paste", params={}, files=[file_record]) + + def _build_chunk(data) -> ProcessedChunk: return ProcessedChunk( type=KnowledgeType.FACTUAL, @@ -96,6 +116,51 @@ async def test_persist_processed_chunk_rejects_non_object_before_paragraph_write assert metadata_store.paragraphs == [] +@pytest.mark.asyncio +async def test_chunk_terminal_progress_uses_successful_chunks_only() -> None: + manager, _ = _build_manager() + + task = _build_progress_task("task-fail-then-complete") + manager._tasks[task.task_id] = task + + await manager._set_chunk_failed(task.task_id, "file-1", "chunk-0", "boom") + await manager._set_chunk_completed(task.task_id, "file-1", "chunk-1") + + file_record = task.files[0] + assert file_record.done_chunks == 1 + assert file_record.failed_chunks == 1 + assert file_record.progress == pytest.approx(0.5) + assert task.progress == pytest.approx(0.5) + + reverse_task = _build_progress_task("task-complete-then-fail") + manager._tasks[reverse_task.task_id] = reverse_task + + await manager._set_chunk_completed(reverse_task.task_id, "file-1", "chunk-0") + await manager._set_chunk_failed(reverse_task.task_id, "file-1", "chunk-1", "boom") + + reverse_file = reverse_task.files[0] + assert reverse_file.done_chunks == 1 + assert reverse_file.failed_chunks == 1 + assert reverse_file.progress == pytest.approx(0.5) + assert reverse_task.progress == pytest.approx(0.5) + + +@pytest.mark.asyncio +async def test_cancelled_chunks_do_not_increase_file_progress() -> None: + manager, _ = _build_manager() + task = _build_progress_task("task-cancelled-progress", total_chunks=3) + manager._tasks[task.task_id] = task + + await manager._set_chunk_completed(task.task_id, "file-1", "chunk-0") + await manager._set_chunk_cancelled(task.task_id, "file-1", "chunk-1", "任务已取消") + + file_record = task.files[0] + assert file_record.done_chunks == 1 + assert file_record.cancelled_chunks == 1 + assert file_record.progress == pytest.approx(1 / 3) + assert task.progress == pytest.approx(1 / 3) + + @pytest.mark.asyncio async def test_persist_processed_chunk_skips_invalid_nested_items() -> None: manager, metadata_store = _build_manager() diff --git a/src/A_memorix/core/utils/web_import_manager.py b/src/A_memorix/core/utils/web_import_manager.py index 2b41f9f2..037ddc6a 100644 --- a/src/A_memorix/core/utils/web_import_manager.py +++ b/src/A_memorix/core/utils/web_import_manager.py @@ -2040,7 +2040,7 @@ class ImportTaskManager: if total <= 0: total = max(1, scanned) - progress = max(0.0, min(1.0, float(scanned) / float(total))) if total > 0 else 0.0 + chunk_progress = max(0.0, min(1.0, float(scanned) / float(total))) if total > 0 else 0.0 preview = f"scanned={scanned}/{total}, migrated={migrated}, bad={bad}, last_id={last_id}" async with self._lock: @@ -2055,14 +2055,14 @@ class ImportTaskManager: if c.status not in {"completed", "failed", "cancelled"}: c.status = "writing" c.step = "migrating" - c.progress = progress + c.progress = chunk_progress c.content_preview = preview c.updated_at = _now() f.total_chunks = total f.done_chunks = done f.failed_chunks = bad f.cancelled_chunks = 0 - f.progress = progress + self._recompute_file_progress(f) if f.status not in {"failed", "cancelled"}: f.status = "writing" f.current_step = "migrating" @@ -2209,7 +2209,7 @@ class ImportTaskManager: f.done_chunks = max(0, min(f.done_chunks, f.total_chunks)) f.failed_chunks = max(0, min(f.failed_chunks, f.total_chunks)) f.cancelled_chunks = 0 - f.progress = 1.0 + self._recompute_file_progress(f) f.status = "completed" f.current_step = "completed" if bad_rows > 0 and not f.error: @@ -3578,9 +3578,7 @@ JSON schema: additional_cancelled += 1 if additional_cancelled > 0: f.cancelled_chunks += additional_cancelled - f.progress = self._compute_ratio( - f.done_chunks + f.failed_chunks + f.cancelled_chunks, f.total_chunks - ) + self._recompute_file_progress(f) f.updated_at = _now() task.updated_at = _now() self._recompute_task_progress(task) @@ -3638,7 +3636,7 @@ JSON schema: c.progress = 1.0 c.updated_at = _now() f.done_chunks += 1 - f.progress = self._compute_ratio(f.done_chunks + f.failed_chunks + f.cancelled_chunks, f.total_chunks) + self._recompute_file_progress(f) f.updated_at = _now() self._recompute_task_progress(task) @@ -3666,7 +3664,7 @@ JSON schema: c.progress = 1.0 c.updated_at = _now() f.failed_chunks += 1 - f.progress = self._compute_ratio(f.done_chunks, f.total_chunks) + self._recompute_file_progress(f) if not f.error: f.error = str(error) f.updated_at = _now() @@ -3690,7 +3688,7 @@ JSON schema: c.progress = 1.0 c.updated_at = _now() f.cancelled_chunks += 1 - f.progress = self._compute_ratio(f.done_chunks + f.failed_chunks + f.cancelled_chunks, f.total_chunks) + self._recompute_file_progress(f) f.updated_at = _now() self._recompute_task_progress(task) @@ -3718,6 +3716,9 @@ JSON schema: return 1.0 return max(0.0, min(1.0, float(done) / float(total))) + def _recompute_file_progress(self, file_record: ImportFileRecord) -> None: + file_record.progress = self._compute_ratio(file_record.done_chunks, file_record.total_chunks) + def _recompute_task_progress(self, task: ImportTaskRecord) -> None: total = 0 done = 0 @@ -3765,9 +3766,7 @@ JSON schema: additional_cancelled += 1 if additional_cancelled > 0: f.cancelled_chunks += additional_cancelled - f.progress = self._compute_ratio( - f.done_chunks + f.failed_chunks + f.cancelled_chunks, f.total_chunks - ) + self._recompute_file_progress(f) f.updated_at = _now() task.status = "cancelled" task.current_step = "cancelled" diff --git a/uv.lock b/uv.lock index dba2d9cf..4853ca9a 100644 --- a/uv.lock +++ b/uv.lock @@ -1511,7 +1511,7 @@ requires-dist = [ { name = "httpx", extras = ["socks"] }, { name = "jieba", specifier = ">=0.42.1" }, { name = "json-repair", specifier = ">=0.47.6" }, - { name = "maibot-dashboard", specifier = ">=1.0.5" }, + { name = "maibot-dashboard", specifier = ">=1.0.6" }, { name = "maibot-plugin-sdk", specifier = ">=2.4.0" }, { name = "maim-message", specifier = ">=0.6.2" }, { name = "matplotlib", specifier = ">=3.10.5" }, @@ -1549,11 +1549,11 @@ dev = [ [[package]] name = "maibot-dashboard" -version = "1.0.5" +version = "1.0.6" source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" } -sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b8/a7/eb1032664ea98b58a861412aca19b31066dc3368f1264a2a53970bd9385c/maibot_dashboard-1.0.5.tar.gz", hash = "sha256:3480723e42120defbaf8ebb952c45bc3e0cd9274a04c5acda0331e55e15ebdc1", size = 2477306, upload-time = "2026-05-05T10:40:34.327Z" } +sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/73/76/a2a47f902f20bbaa699584d7fa9676f591503e0d6954de65aa0a90c07000/maibot_dashboard-1.0.6.tar.gz", hash = "sha256:f383d3505a102554a51bf49d1fc56a8ba8c5db60a3c41b7eab4513a6fd0a1f88", size = 2485522, upload-time = "2026-05-06T10:44:36.42Z" } wheels = [ - { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3a/a0/ad4f7c1d381875ca8d1aeedf5ff6e94692f64cf558479f9e845e47bca830/maibot_dashboard-1.0.5-py3-none-any.whl", hash = "sha256:67bfbb82a1ddd666d20cc958864db38df2e5493f77df0cb049ae83987b1dd65d", size = 2542631, upload-time = "2026-05-05T10:40:32.5Z" }, + { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ae/14/a62631e60c9606a793d6740ef61fc0b8868cf8a79c9f192667026874799b/maibot_dashboard-1.0.6-py3-none-any.whl", hash = "sha256:36299d7039fbb98fd8aa1fb31d2bbc040d1018d9d87ebcf09194e4efb0cf9af7", size = 2552642, upload-time = "2026-05-06T10:44:34.216Z" }, ] [[package]]