Losita/mai-bot
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix:修复错误进度展示的问题

Browse Source
This commit is contained in:
DawnARC
2026-05-06 23:05:44 +08:00
parent 9acf03d24b
commit d01c7276a8
3 changed files with 82 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
pytests/A_memorix_test/test_web_import_manager_payloads.py
View File

@@ -4,7 +4,12 @@ import numpy as np
import pytest
from src.A_memorix.core.strategies.base import ChunkContext, KnowledgeType, ProcessedChunk, SourceInfo
from src.A_memorix.core.utils.web_import_manager import ImportTaskManager
from src.A_memorix.core.utils.web_import_manager import (
ImportChunkRecord,
ImportFileRecord,
ImportTaskManager,
ImportTaskRecord,
)
class _DummyMetadataStore:
@@ -76,6 +81,21 @@ def _build_manager() -> tuple[ImportTaskManager, _DummyMetadataStore]:
return manager, metadata_store
def _build_progress_task(task_id: str, total_chunks: int = 2) -> ImportTaskRecord:
file_record = ImportFileRecord(
file_id="file-1",
name="demo.txt",
source_kind="paste",
input_mode="text",
total_chunks=total_chunks,
chunks=[
ImportChunkRecord(chunk_id=f"chunk-{index}", index=index, chunk_type="text")
for index in range(total_chunks)
],
)
return ImportTaskRecord(task_id=task_id, source="paste", params={}, files=[file_record])
def _build_chunk(data) -> ProcessedChunk:
return ProcessedChunk(
type=KnowledgeType.FACTUAL,
@@ -96,6 +116,51 @@ async def test_persist_processed_chunk_rejects_non_object_before_paragraph_write
assert metadata_store.paragraphs == []
@pytest.mark.asyncio
async def test_chunk_terminal_progress_uses_successful_chunks_only() -> None:
manager, _ = _build_manager()
task = _build_progress_task("task-fail-then-complete")
manager._tasks[task.task_id] = task
await manager._set_chunk_failed(task.task_id, "file-1", "chunk-0", "boom")
await manager._set_chunk_completed(task.task_id, "file-1", "chunk-1")
file_record = task.files[0]
assert file_record.done_chunks == 1
assert file_record.failed_chunks == 1
assert file_record.progress == pytest.approx(0.5)
assert task.progress == pytest.approx(0.5)
reverse_task = _build_progress_task("task-complete-then-fail")
manager._tasks[reverse_task.task_id] = reverse_task
await manager._set_chunk_completed(reverse_task.task_id, "file-1", "chunk-0")
await manager._set_chunk_failed(reverse_task.task_id, "file-1", "chunk-1", "boom")
reverse_file = reverse_task.files[0]
assert reverse_file.done_chunks == 1
assert reverse_file.failed_chunks == 1
assert reverse_file.progress == pytest.approx(0.5)
assert reverse_task.progress == pytest.approx(0.5)
@pytest.mark.asyncio
async def test_cancelled_chunks_do_not_increase_file_progress() -> None:
manager, _ = _build_manager()
task = _build_progress_task("task-cancelled-progress", total_chunks=3)
manager._tasks[task.task_id] = task
await manager._set_chunk_completed(task.task_id, "file-1", "chunk-0")
await manager._set_chunk_cancelled(task.task_id, "file-1", "chunk-1", "任务已取消")
file_record = task.files[0]
assert file_record.done_chunks == 1
assert file_record.cancelled_chunks == 1
assert file_record.progress == pytest.approx(1 / 3)
assert task.progress == pytest.approx(1 / 3)
@pytest.mark.asyncio
async def test_persist_processed_chunk_skips_invalid_nested_items() -> None:
manager, metadata_store = _build_manager()

25
src/A_memorix/core/utils/web_import_manager.py
View File

@@ -2040,7 +2040,7 @@ class ImportTaskManager:
if total <= 0:
total = max(1, scanned)
progress = max(0.0, min(1.0, float(scanned) / float(total))) if total > 0 else 0.0
chunk_progress = max(0.0, min(1.0, float(scanned) / float(total))) if total > 0 else 0.0
preview = f"scanned={scanned}/{total}, migrated={migrated}, bad={bad}, last_id={last_id}"
async with self._lock:
@@ -2055,14 +2055,14 @@ class ImportTaskManager:
if c.status not in {"completed", "failed", "cancelled"}:
c.status = "writing"
c.step = "migrating"
c.progress = progress
c.progress = chunk_progress
c.content_preview = preview
c.updated_at = _now()
f.total_chunks = total
f.done_chunks = done
f.failed_chunks = bad
f.cancelled_chunks = 0
f.progress = progress
self._recompute_file_progress(f)
if f.status not in {"failed", "cancelled"}:
f.status = "writing"
f.current_step = "migrating"
@@ -2209,7 +2209,7 @@ class ImportTaskManager:
f.done_chunks = max(0, min(f.done_chunks, f.total_chunks))
f.failed_chunks = max(0, min(f.failed_chunks, f.total_chunks))
f.cancelled_chunks = 0
f.progress = 1.0
self._recompute_file_progress(f)
f.status = "completed"
f.current_step = "completed"
if bad_rows > 0 and not f.error:
@@ -3578,9 +3578,7 @@ JSON schema:
additional_cancelled += 1
if additional_cancelled > 0:
f.cancelled_chunks += additional_cancelled
f.progress = self._compute_ratio(
f.done_chunks + f.failed_chunks + f.cancelled_chunks, f.total_chunks
)
self._recompute_file_progress(f)
f.updated_at = _now()
task.updated_at = _now()
self._recompute_task_progress(task)
@@ -3638,7 +3636,7 @@ JSON schema:
c.progress = 1.0
c.updated_at = _now()
f.done_chunks += 1
f.progress = self._compute_ratio(f.done_chunks + f.failed_chunks + f.cancelled_chunks, f.total_chunks)
self._recompute_file_progress(f)
f.updated_at = _now()
self._recompute_task_progress(task)
@@ -3666,7 +3664,7 @@ JSON schema:
c.progress = 1.0
c.updated_at = _now()
f.failed_chunks += 1
f.progress = self._compute_ratio(f.done_chunks, f.total_chunks)
self._recompute_file_progress(f)
if not f.error:
f.error = str(error)
f.updated_at = _now()
@@ -3690,7 +3688,7 @@ JSON schema:
c.progress = 1.0
c.updated_at = _now()
f.cancelled_chunks += 1
f.progress = self._compute_ratio(f.done_chunks + f.failed_chunks + f.cancelled_chunks, f.total_chunks)
self._recompute_file_progress(f)
f.updated_at = _now()
self._recompute_task_progress(task)
@@ -3718,6 +3716,9 @@ JSON schema:
return 1.0
return max(0.0, min(1.0, float(done) / float(total)))
def _recompute_file_progress(self, file_record: ImportFileRecord) -> None:
file_record.progress = self._compute_ratio(file_record.done_chunks, file_record.total_chunks)
def _recompute_task_progress(self, task: ImportTaskRecord) -> None:
total = 0
done = 0
@@ -3765,9 +3766,7 @@ JSON schema:
additional_cancelled += 1
if additional_cancelled > 0:
f.cancelled_chunks += additional_cancelled
f.progress = self._compute_ratio(
f.done_chunks + f.failed_chunks + f.cancelled_chunks, f.total_chunks
)
self._recompute_file_progress(f)
f.updated_at = _now()
task.status = "cancelled"
task.current_step = "cancelled"

8
uv.lock generated
View File

@@ -1511,7 +1511,7 @@ requires-dist = [
{ name = "httpx", extras = ["socks"] },
{ name = "jieba", specifier = ">=0.42.1" },
{ name = "json-repair", specifier = ">=0.47.6" },
{ name = "maibot-dashboard", specifier = ">=1.0.5" },
{ name = "maibot-dashboard", specifier = ">=1.0.6" },
{ name = "maibot-plugin-sdk", specifier = ">=2.4.0" },
{ name = "maim-message", specifier = ">=0.6.2" },
{ name = "matplotlib", specifier = ">=3.10.5" },
@@ -1549,11 +1549,11 @@ dev = [
[[package]]
name = "maibot-dashboard"
version = "1.0.5"
version = "1.0.6"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b8/a7/eb1032664ea98b58a861412aca19b31066dc3368f1264a2a53970bd9385c/maibot_dashboard-1.0.5.tar.gz", hash = "sha256:3480723e42120defbaf8ebb952c45bc3e0cd9274a04c5acda0331e55e15ebdc1", size = 2477306, upload-time = "2026-05-05T10:40:34.327Z" }
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/73/76/a2a47f902f20bbaa699584d7fa9676f591503e0d6954de65aa0a90c07000/maibot_dashboard-1.0.6.tar.gz", hash = "sha256:f383d3505a102554a51bf49d1fc56a8ba8c5db60a3c41b7eab4513a6fd0a1f88", size = 2485522, upload-time = "2026-05-06T10:44:36.42Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/3a/a0/ad4f7c1d381875ca8d1aeedf5ff6e94692f64cf558479f9e845e47bca830/maibot_dashboard-1.0.5-py3-none-any.whl", hash = "sha256:67bfbb82a1ddd666d20cc958864db38df2e5493f77df0cb049ae83987b1dd65d", size = 2542631, upload-time = "2026-05-05T10:40:32.5Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/ae/14/a62631e60c9606a793d6740ef61fc0b8868cf8a79c9f192667026874799b/maibot_dashboard-1.0.6-py3-none-any.whl", hash = "sha256:36299d7039fbb98fd8aa1fb31d2bbc040d1018d9d87ebcf09194e4efb0cf9af7", size = 2552642, upload-time = "2026-05-06T10:44:34.216Z" },
]
[[package]]